fsspec 2024.5.0__tar.gz → 2024.6.1__tar.gz

{fsspec-2024.5.0 → fsspec-2024.6.1}/.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.3.4
+    rev: v0.4.4
     hooks:
       # Run the linter.
       - id: ruff

{fsspec-2024.5.0 → fsspec-2024.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fsspec
-Version: 2024.5.0
+Version: 2024.6.1
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
@@ -59,6 +59,12 @@ Requires-Dist: distributed; extra == 'dask'
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: doc
+Requires-Dist: numpydoc; extra == 'doc'
+Requires-Dist: sphinx; extra == 'doc'
+Requires-Dist: sphinx-design; extra == 'doc'
+Requires-Dist: sphinx-rtd-theme; extra == 'doc'
+Requires-Dist: yarl; extra == 'doc'
 Provides-Extra: dropbox
 Requires-Dist: dropbox; extra == 'dropbox'
 Requires-Dist: dropboxdrivefs; extra == 'dropbox'
@@ -178,7 +184,6 @@ Description-Content-Type: text/markdown
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
-[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
 
 A specification for pythonic filesystems.
 
@@ -225,13 +230,13 @@ CI runtime. For local use, pick a version suitable for you.
 mamba create -n fsspec -c conda-forge  python=3.9 -y
 conda activate fsspec
 
-# Standard dev test install.
-pip install -e ".[dev,test]"
+# Standard dev install with docs and tests.
+pip install -e ".[dev,doc,test]"
 
 # Full tests except for downstream
 pip install s3fs
 pip uninstall s3fs
-pip install -e .[dev,test_full]
+pip install -e .[dev,doc,test_full]
 pip install s3fs --no-deps
 pytest -v
 

{fsspec-2024.5.0 → fsspec-2024.6.1}/README.md

@@ -4,7 +4,6 @@
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
-[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
 
 A specification for pythonic filesystems.
 
@@ -51,13 +50,13 @@ CI runtime. For local use, pick a version suitable for you.
 mamba create -n fsspec -c conda-forge  python=3.9 -y
 conda activate fsspec
 
-# Standard dev test install.
-pip install -e ".[dev,test]"
+# Standard dev install with docs and tests.
+pip install -e ".[dev,doc,test]"
 
 # Full tests except for downstream
 pip install s3fs
 pip uninstall s3fs
-pip install -e .[dev,test_full]
+pip install -e .[dev,doc,test_full]
 pip install s3fs --no-deps
 pytest -v
 

fsspec-2024.6.1/docs/environment.yml

@@ -0,0 +1,5 @@
+name: fsspec
+channels:
+  - defaults
+dependencies:
+  - python=3.9

{fsspec-2024.5.0 → fsspec-2024.6.1}/docs/source/changelog.rst

@@ -1,6 +1,40 @@
 Changelog
 =========
 
+2024.6.1
+--------
+
+Fixes
+
+- fix appending to non-dict reference sets (#1634)
+- don't let generic edit info dicts (#1633)
+- set https's loop before calling super (#1633)
+- cached write file doesn't need to update it's size on close (#1633)
+- fix JSON serialize for FSs with interior FSs (#1628, 1627)
+- option to strip "password" when pickling (#1625)
+- fix filecache write (#1622)
+
+
+2024.6.0
+--------
+
+Enhancements
+
+- allow dicts (not just bytes) for referenceFS (#1616
+- make filesystems JSON serializeable (#1612)
+- implement multifile cat() for github (#1620)
+
+Fixes
+
+- implement auto_mkdir for SMB (#1604)
+
+Other
+
+- add doc deps to pyproject (#1613)
+- re-remove test from package (#1611)
+- formatting (#1610, 1608, 1602)
+- change monthly download badge (#1607)
+
 2024.5.0
 --------
 

{fsspec-2024.5.0 → fsspec-2024.6.1}/docs/source/developer.rst

@@ -107,9 +107,9 @@ The following can be used to install ``fsspec`` in development mode
 
    git clone https://github.com/fsspec/filesystem_spec
    cd filesystem_spec
-   pip install -e .
+   pip install -e .[dev,doc,test]
 
-A number of additional dependencies are required to run tests, see "ci/environment*.yml", as
+A number of additional dependencies are required to run tests in full, see "ci/environment*.yml", as
 well as Docker. Most implementation-specific tests should skip if their requirements are
 not met.
 

{fsspec-2024.5.0 → fsspec-2024.6.1}/docs/source/index.rst

@@ -69,7 +69,7 @@ The following libraries use ``fsspec`` internally for path and file handling:
 .. _xarray: https://docs.xarray.dev/
 .. _zarr: https://zarr.readthedocs.io/
 .. _DVC: https://dvc.org/
-.. _kedro: https://kedro.readthedocs.io/en/stable/01_introduction/01_introduction.html
+.. _kedro: https://docs.kedro.org/en/stable/tutorial/set_up_data.html#supported-data-locations
 .. _pyxet: https://github.com/xetdata/pyxet
 .. _Huggingface🤗 Datasets: https://github.com/huggingface/datasets
 .. _pyarrow: https://arrow.apache.org/docs/python/

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.5.0'
-__version_tuple__ = version_tuple = (2024, 5, 0)
+__version__ = version = '2024.6.1'
+__version_tuple__ = version_tuple = (2024, 6, 1)

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/caching.py

@@ -15,6 +15,7 @@ from typing import (
     ClassVar,
     Generic,
     NamedTuple,
+    Optional,
     OrderedDict,
     TypeVar,
 )
@@ -574,7 +575,7 @@ class KnownPartsOfAFile(BaseCache):
         blocksize: int,
         fetcher: Fetcher,
         size: int,
-        data: dict[tuple[int, int], bytes] = {},
+        data: Optional[dict[tuple[int, int], bytes]] = None,
         strict: bool = True,
         **_: Any,
     ):
@@ -597,7 +598,7 @@ class KnownPartsOfAFile(BaseCache):
 
             self.data = dict(zip(offsets, blocks))
         else:
-            self.data = data
+            self.data = {}
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
         if start is None:

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/compression.py

@@ -139,7 +139,7 @@ class SnappyFile(AbstractBufferedFile):
 try:
     import snappy
 
-    snappy.compress
+    snappy.compress(b"")
     # Snappy may use the .sz file extension, but this is not part of the
     # standard implementation.
     register_compression("snappy", SnappyFile, [])

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/generic.py

@@ -197,6 +197,7 @@ class GenericFileSystem(AsyncFileSystem):
             )
         result = {}
         for k, v in out.items():
+            v = v.copy()  # don't corrupt target FS dircache
             name = fs.unstrip_protocol(k)
             v["name"] = name
             result[name] = v
@@ -210,6 +211,7 @@ class GenericFileSystem(AsyncFileSystem):
             out = await fs._info(url, **kwargs)
         else:
             out = fs.info(url, **kwargs)
+        out = out.copy()  # don't edit originals
         out["name"] = fs.unstrip_protocol(out["name"])
         return out
 
@@ -224,6 +226,7 @@ class GenericFileSystem(AsyncFileSystem):
             out = await fs._ls(url, detail=True, **kwargs)
         else:
             out = fs.ls(url, detail=True, **kwargs)
+        out = [o.copy() for o in out]  # don't edit originals
         for o in out:
             o["name"] = fs.unstrip_protocol(o["name"])
         if detail:

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/implementations/cached.py

@@ -425,7 +425,6 @@ class CachingFileSystem(AbstractFileSystem):
             "clear_cache",
             "clear_expired_cache",
             "pop_from_cache",
-            "_mkcache",
             "local_file",
             "_paths_from_path",
             "get_mapper",
@@ -435,12 +434,10 @@ class CachingFileSystem(AbstractFileSystem):
             "__hash__",
             "__eq__",
             "to_json",
+            "to_dict",
             "cache_size",
             "pipe_file",
             "pipe",
-            "isdir",
-            "isfile",
-            "exists",
             "start_transaction",
             "end_transaction",
         }:
@@ -510,15 +507,6 @@ class CachingFileSystem(AbstractFileSystem):
             ^ hash(self.target_protocol)
         )
 
-    def to_json(self):
-        """Calculate JSON representation.
-
-        Not implemented yet for CachingFileSystem.
-        """
-        raise NotImplementedError(
-            "CachingFileSystem JSON representation not implemented"
-        )
-
 
 class WholeFileCacheFileSystem(CachingFileSystem):
     """Caches whole remote files on first access
@@ -663,7 +651,8 @@ class WholeFileCacheFileSystem(CachingFileSystem):
     def _open(self, path, mode="rb", **kwargs):
         path = self._strip_protocol(path)
         if "r" not in mode:
-            fn = self._make_local_details(path)
+            hash = self._mapper(path)
+            fn = os.path.join(self.storage[-1], hash)
             user_specified_kwargs = {
                 k: v
                 for k, v in kwargs.items()
@@ -806,7 +795,8 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         if self._intrans:
             f = [_ for _ in self.transaction.files if _.path == path]
             if f:
-                return {"name": path, "size": f[0].size or f[0].tell(), "type": "file"}
+                size = os.path.getsize(f[0].fn) if f[0].closed else f[0].tell()
+                return {"name": path, "size": size, "type": "file"}
             f = any(_.path.startswith(path + "/") for _ in self.transaction.files)
             if f:
                 return {"name": path, "size": 0, "type": "directory"}
@@ -912,7 +902,7 @@ class LocalTempFile:
         self.close()
 
     def close(self):
-        self.size = self.fh.tell()
+        # self.size = self.fh.tell()
         if self.closed:
             return
         self.fh.close()

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/implementations/dirfs.py

@@ -56,6 +56,8 @@ class DirFileSystem(AsyncFileSystem):
             if not path:
                 return self.path
             return self.fs.sep.join((self.path, self._strip_protocol(path)))
+        if isinstance(path, dict):
+            return {self._join(_path): value for _path, value in path.items()}
         return [self._join(_path) for _path in path]
 
     def _relpath(self, path):

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/implementations/github.py

@@ -1,5 +1,7 @@
 import requests
 
+import fsspec
+
 from ..spec import AbstractFileSystem
 from ..utils import infer_storage_options
 from .memory import MemoryFile
@@ -225,3 +227,13 @@ class GithubFileSystem(AbstractFileSystem):
             raise FileNotFoundError(path)
         r.raise_for_status()
         return MemoryFile(None, None, r.content)
+
+    def cat(self, path, recursive=False, on_error="raise", **kwargs):
+        paths = self.expand_path(path, recursive=recursive)
+        urls = [
+            self.rurl.format(org=self.org, repo=self.repo, path=u, sha=self.root)
+            for u, sh in paths
+        ]
+        fs = fsspec.filesystem("http")
+        data = fs.cat(urls, on_error="return")
+        return {u: v for ((k, v), u) in zip(data.items(), urls)}

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/implementations/http.py

@@ -560,6 +560,7 @@ class HTTPFile(AbstractBufferedFile):
         if mode != "rb":
             raise NotImplementedError("File mode not supported")
         self.asynchronous = asynchronous
+        self.loop = loop
         self.url = url
         self.session = session
         self.details = {"name": url, "size": size, "type": "file"}
@@ -572,7 +573,6 @@ class HTTPFile(AbstractBufferedFile):
             cache_options=cache_options,
             **kwargs,
         )
-        self.loop = loop
 
     def read(self, length=-1):
         """Read bytes from file
@@ -736,6 +736,7 @@ class HTTPStreamFile(AbstractBufferedFile):
             return r
 
         self.r = sync(self.loop, cor)
+        self.loop = fs.loop
 
     def seek(self, loc, whence=0):
         if loc == 0 and whence == 1:

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/implementations/reference.py

@@ -935,6 +935,13 @@ class ReferenceFileSystem(AsyncFileSystem):
 
     def _process_references0(self, references):
         """Make reference dict for Spec Version 0"""
+        if isinstance(references, dict):
+            # do not do this for lazy/parquet backend, which will not make dicts,
+            # but must remain writable in the original object
+            references = {
+                key: json.dumps(val) if isinstance(val, dict) else val
+                for key, val in references.items()
+            }
         self.references = references
 
     def _process_references1(self, references, template_overrides=None):
@@ -952,6 +959,8 @@ class ReferenceFileSystem(AsyncFileSystem):
                 if v.startswith("base64:"):
                     self.references[k] = base64.b64decode(v[7:])
                 self.references[k] = v
+            elif isinstance(v, dict):
+                self.references[k] = json.dumps(v)
             elif self.templates:
                 u = v[0]
                 if "{{" in u:

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/implementations/smb.py

@@ -68,6 +68,7 @@ class SMBFileSystem(AbstractFileSystem):
         encrypt=None,
         share_access=None,
         register_session_retries=5,
+        auto_mkdir=False,
         **kwargs,
     ):
         """
@@ -102,6 +103,10 @@ class SMBFileSystem(AbstractFileSystem):
             - 'r': Allow other handles to be opened with read access.
             - 'w': Allow other handles to be opened with write access.
             - 'd': Allow other handles to be opened with delete access.
+        auto_mkdir: bool
+            Whether, when opening a file, the directory containing it should
+            be created (if it doesn't already exist). This is assumed by pyarrow
+            and zarr-python code.
         """
         super().__init__(**kwargs)
         self.host = host
@@ -113,6 +118,7 @@ class SMBFileSystem(AbstractFileSystem):
         self.temppath = kwargs.pop("temppath", "")
         self.share_access = share_access
         self.register_session_retries = register_session_retries
+        self.auto_mkdir = auto_mkdir
         self._connect()
 
     @property
@@ -224,6 +230,8 @@ class SMBFileSystem(AbstractFileSystem):
         By specifying 'share_access' in 'kwargs' it is possible to override the
         default shared access setting applied in the constructor of this object.
         """
+        if self.auto_mkdir and "w" in mode:
+            self.makedirs(self._parent(path), exist_ok=True)
         bls = block_size if block_size is not None and block_size >= 0 else -1
         wpath = _as_unc_path(self.host, path)
         share_access = kwargs.pop("share_access", self.share_access)
@@ -245,6 +253,8 @@ class SMBFileSystem(AbstractFileSystem):
         """Copy within two locations in the same filesystem"""
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
+        if self.auto_mkdir:
+            self.makedirs(self._parent(path2), exist_ok=True)
         smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs)
 
     def _rm(self, path):

fsspec-2024.6.1/fsspec/json.py

@@ -0,0 +1,121 @@
+import json
+from contextlib import suppress
+from pathlib import PurePath
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+)
+
+from .registry import _import_class, get_filesystem_class
+from .spec import AbstractFileSystem
+
+
+class FilesystemJSONEncoder(json.JSONEncoder):
+    include_password: ClassVar[bool] = True
+
+    def default(self, o: Any) -> Any:
+        if isinstance(o, AbstractFileSystem):
+            return o.to_dict(include_password=self.include_password)
+        if isinstance(o, PurePath):
+            cls = type(o)
+            return {"cls": f"{cls.__module__}.{cls.__name__}", "str": str(o)}
+
+        return super().default(o)
+
+    def make_serializable(self, obj: Any) -> Any:
+        """
+        Recursively converts an object so that it can be JSON serialized via
+        :func:`json.dumps` and :func:`json.dump`, without actually calling
+        said functions.
+        """
+        if isinstance(obj, (str, int, float, bool)):
+            return obj
+        if isinstance(obj, Mapping):
+            return {k: self.make_serializable(v) for k, v in obj.items()}
+        if isinstance(obj, Sequence):
+            return [self.make_serializable(v) for v in obj]
+
+        return self.default(obj)
+
+
+class FilesystemJSONDecoder(json.JSONDecoder):
+    def __init__(
+        self,
+        *,
+        object_hook: Optional[Callable[[Dict[str, Any]], Any]] = None,
+        parse_float: Optional[Callable[[str], Any]] = None,
+        parse_int: Optional[Callable[[str], Any]] = None,
+        parse_constant: Optional[Callable[[str], Any]] = None,
+        strict: bool = True,
+        object_pairs_hook: Optional[Callable[[List[Tuple[str, Any]]], Any]] = None,
+    ) -> None:
+        self.original_object_hook = object_hook
+
+        super().__init__(
+            object_hook=self.custom_object_hook,
+            parse_float=parse_float,
+            parse_int=parse_int,
+            parse_constant=parse_constant,
+            strict=strict,
+            object_pairs_hook=object_pairs_hook,
+        )
+
+    @classmethod
+    def try_resolve_path_cls(cls, dct: Dict[str, Any]):
+        with suppress(Exception):
+            fqp = dct["cls"]
+
+            path_cls = _import_class(fqp)
+
+            if issubclass(path_cls, PurePath):
+                return path_cls
+
+        return None
+
+    @classmethod
+    def try_resolve_fs_cls(cls, dct: Dict[str, Any]):
+        with suppress(Exception):
+            if "cls" in dct:
+                try:
+                    fs_cls = _import_class(dct["cls"])
+                    if issubclass(fs_cls, AbstractFileSystem):
+                        return fs_cls
+                except Exception:
+                    if "protocol" in dct:  # Fallback if cls cannot be imported
+                        return get_filesystem_class(dct["protocol"])
+
+                    raise
+
+        return None
+
+    def custom_object_hook(self, dct: Dict[str, Any]):
+        if "cls" in dct:
+            if (obj_cls := self.try_resolve_fs_cls(dct)) is not None:
+                return AbstractFileSystem.from_dict(dct)
+            if (obj_cls := self.try_resolve_path_cls(dct)) is not None:
+                return obj_cls(dct["str"])
+
+        if self.original_object_hook is not None:
+            return self.original_object_hook(dct)
+
+        return dct
+
+    def unmake_serializable(self, obj: Any) -> Any:
+        """
+        Inverse function of :meth:`FilesystemJSONEncoder.make_serializable`.
+        """
+        if isinstance(obj, dict):
+            obj = self.custom_object_hook(obj)
+        if isinstance(obj, dict):
+            return {k: self.unmake_serializable(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [self.unmake_serializable(v) for v in obj]
+
+        return obj

{fsspec-2024.5.0 → fsspec-2024.6.1}/fsspec/registry.py

@@ -257,27 +257,33 @@ update the current installation.
 """
 
 
-def _import_class(cls, minv=None):
-    """Take a string FQP and return the imported class or identifier
+def _import_class(fqp: str):
+    """Take a fully-qualified path and return the imported class or identifier.
 
-    cls is of the form "package.module.klass" or "package.module:subobject.klass"
+    ``fqp`` is of the form "package.module.klass" or
+    "package.module:subobject.klass".
+
+    Warnings
+    --------
+    This can import arbitrary modules. Make sure you haven't installed any modules
+    that may execute malicious code at import time.
     """
-    if ":" in cls:
-        mod, name = cls.rsplit(":", 1)
-        s3 = mod == "s3fs"
-        mod = importlib.import_module(mod)
-        if s3 and mod.__version__.split(".") < ["0", "5"]:
-            warnings.warn(s3_msg)
-        for part in name.split("."):
-            mod = getattr(mod, part)
-        return mod
+    if ":" in fqp:
+        mod, name = fqp.rsplit(":", 1)
     else:
-        mod, name = cls.rsplit(".", 1)
-        s3 = mod == "s3fs"
-        mod = importlib.import_module(mod)
-        if s3 and mod.__version__.split(".") < ["0", "5"]:
-            warnings.warn(s3_msg)
-        return getattr(mod, name)
+        mod, name = fqp.rsplit(".", 1)
+
+    is_s3 = mod == "s3fs"
+    mod = importlib.import_module(mod)
+    if is_s3 and mod.__version__.split(".") < ["0", "5"]:
+        warnings.warn(s3_msg)
+    for part in name.split("."):
+        mod = getattr(mod, part)
+
+    if not isinstance(mod, type):
+        raise TypeError(f"{fqp} is not a class")
+
+    return mod
 
 
 def filesystem(protocol, **storage_options):