fsspec 2024.5.0__tar.gz → 2024.6.0__tar.gz

{fsspec-2024.5.0 → fsspec-2024.6.0}/.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.3.4
+    rev: v0.4.4
     hooks:
       # Run the linter.
       - id: ruff

{fsspec-2024.5.0 → fsspec-2024.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fsspec
-Version: 2024.5.0
+Version: 2024.6.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
@@ -59,6 +59,12 @@ Requires-Dist: distributed; extra == 'dask'
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: doc
+Requires-Dist: numpydoc; extra == 'doc'
+Requires-Dist: sphinx; extra == 'doc'
+Requires-Dist: sphinx-design; extra == 'doc'
+Requires-Dist: sphinx-rtd-theme; extra == 'doc'
+Requires-Dist: yarl; extra == 'doc'
 Provides-Extra: dropbox
 Requires-Dist: dropbox; extra == 'dropbox'
 Requires-Dist: dropboxdrivefs; extra == 'dropbox'
@@ -178,7 +184,7 @@ Description-Content-Type: text/markdown
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
-[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
+[![fsspec Downloads Last Month](https://assets.piptrends.com/get-last-month-downloads-badge/fsspec.svg 'fsspec Downloads Last Month by pip Trends')](https://piptrends.com/package/fsspec)
 
 A specification for pythonic filesystems.
 
@@ -225,13 +231,13 @@ CI runtime. For local use, pick a version suitable for you.
 mamba create -n fsspec -c conda-forge  python=3.9 -y
 conda activate fsspec
 
-# Standard dev test install.
-pip install -e ".[dev,test]"
+# Standard dev install with docs and tests.
+pip install -e ".[dev,doc,test]"
 
 # Full tests except for downstream
 pip install s3fs
 pip uninstall s3fs
-pip install -e .[dev,test_full]
+pip install -e .[dev,doc,test_full]
 pip install s3fs --no-deps
 pytest -v
 

{fsspec-2024.5.0 → fsspec-2024.6.0}/README.md

@@ -4,7 +4,7 @@
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
-[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
+[![fsspec Downloads Last Month](https://assets.piptrends.com/get-last-month-downloads-badge/fsspec.svg 'fsspec Downloads Last Month by pip Trends')](https://piptrends.com/package/fsspec)
 
 A specification for pythonic filesystems.
 
@@ -51,13 +51,13 @@ CI runtime. For local use, pick a version suitable for you.
 mamba create -n fsspec -c conda-forge  python=3.9 -y
 conda activate fsspec
 
-# Standard dev test install.
-pip install -e ".[dev,test]"
+# Standard dev install with docs and tests.
+pip install -e ".[dev,doc,test]"
 
 # Full tests except for downstream
 pip install s3fs
 pip uninstall s3fs
-pip install -e .[dev,test_full]
+pip install -e .[dev,doc,test_full]
 pip install s3fs --no-deps
 pytest -v
 

fsspec-2024.6.0/docs/environment.yml

@@ -0,0 +1,5 @@
+name: fsspec
+channels:
+  - defaults
+dependencies:
+  - python=3.9

{fsspec-2024.5.0 → fsspec-2024.6.0}/docs/source/changelog.rst

@@ -1,6 +1,26 @@
 Changelog
 =========
 
+2024.6.0
+--------
+
+Enhancements
+
+- allow dicts (not just bytes) for referenceFS (#1616
+- make filesystems JSON serializeable (#1612)
+- implement multifile cat() for github (#1620)
+
+Fixes
+
+- implement auto_mkdir for SMB (#1604)
+
+Other
+
+- add doc deps to pyproject (#1613)
+- re-remove test from package (#1611)
+- formatting (#1610, 1608, 1602)
+- change monthly download badge (#1607)
+
 2024.5.0
 --------
 

{fsspec-2024.5.0 → fsspec-2024.6.0}/docs/source/developer.rst

@@ -107,9 +107,9 @@ The following can be used to install ``fsspec`` in development mode
 
    git clone https://github.com/fsspec/filesystem_spec
    cd filesystem_spec
-   pip install -e .
+   pip install -e .[dev,doc,test]
 
-A number of additional dependencies are required to run tests, see "ci/environment*.yml", as
+A number of additional dependencies are required to run tests in full, see "ci/environment*.yml", as
 well as Docker. Most implementation-specific tests should skip if their requirements are
 not met.
 

{fsspec-2024.5.0 → fsspec-2024.6.0}/docs/source/index.rst

@@ -69,7 +69,7 @@ The following libraries use ``fsspec`` internally for path and file handling:
 .. _xarray: https://docs.xarray.dev/
 .. _zarr: https://zarr.readthedocs.io/
 .. _DVC: https://dvc.org/
-.. _kedro: https://kedro.readthedocs.io/en/stable/01_introduction/01_introduction.html
+.. _kedro: https://docs.kedro.org/en/stable/tutorial/set_up_data.html#supported-data-locations
 .. _pyxet: https://github.com/xetdata/pyxet
 .. _Huggingface🤗 Datasets: https://github.com/huggingface/datasets
 .. _pyarrow: https://arrow.apache.org/docs/python/

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.5.0'
-__version_tuple__ = version_tuple = (2024, 5, 0)
+__version__ = version = '2024.6.0'
+__version_tuple__ = version_tuple = (2024, 6, 0)

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/caching.py

@@ -15,6 +15,7 @@ from typing import (
     ClassVar,
     Generic,
     NamedTuple,
+    Optional,
     OrderedDict,
     TypeVar,
 )
@@ -574,7 +575,7 @@ class KnownPartsOfAFile(BaseCache):
         blocksize: int,
         fetcher: Fetcher,
         size: int,
-        data: dict[tuple[int, int], bytes] = {},
+        data: Optional[dict[tuple[int, int], bytes]] = None,
         strict: bool = True,
         **_: Any,
     ):
@@ -597,7 +598,7 @@ class KnownPartsOfAFile(BaseCache):
 
             self.data = dict(zip(offsets, blocks))
         else:
-            self.data = data
+            self.data = {}
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
         if start is None:

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/compression.py

@@ -139,7 +139,7 @@ class SnappyFile(AbstractBufferedFile):
 try:
     import snappy
 
-    snappy.compress
+    snappy.compress(b"")
     # Snappy may use the .sz file extension, but this is not part of the
     # standard implementation.
     register_compression("snappy", SnappyFile, [])

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/implementations/cached.py

@@ -425,7 +425,6 @@ class CachingFileSystem(AbstractFileSystem):
             "clear_cache",
             "clear_expired_cache",
             "pop_from_cache",
-            "_mkcache",
             "local_file",
             "_paths_from_path",
             "get_mapper",
@@ -435,12 +434,10 @@ class CachingFileSystem(AbstractFileSystem):
             "__hash__",
             "__eq__",
             "to_json",
+            "to_dict",
             "cache_size",
             "pipe_file",
             "pipe",
-            "isdir",
-            "isfile",
-            "exists",
             "start_transaction",
             "end_transaction",
         }:
@@ -510,15 +507,6 @@ class CachingFileSystem(AbstractFileSystem):
             ^ hash(self.target_protocol)
         )
 
-    def to_json(self):
-        """Calculate JSON representation.
-
-        Not implemented yet for CachingFileSystem.
-        """
-        raise NotImplementedError(
-            "CachingFileSystem JSON representation not implemented"
-        )
-
 
 class WholeFileCacheFileSystem(CachingFileSystem):
     """Caches whole remote files on first access

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/implementations/github.py

@@ -1,5 +1,7 @@
 import requests
 
+import fsspec
+
 from ..spec import AbstractFileSystem
 from ..utils import infer_storage_options
 from .memory import MemoryFile
@@ -225,3 +227,13 @@ class GithubFileSystem(AbstractFileSystem):
             raise FileNotFoundError(path)
         r.raise_for_status()
         return MemoryFile(None, None, r.content)
+
+    def cat(self, path, recursive=False, on_error="raise", **kwargs):
+        paths = self.expand_path(path, recursive=recursive)
+        urls = [
+            self.rurl.format(org=self.org, repo=self.repo, path=u, sha=self.root)
+            for u, sh in paths
+        ]
+        fs = fsspec.filesystem("http")
+        data = fs.cat(urls, on_error="return")
+        return {u: v for ((k, v), u) in zip(data.items(), urls)}

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/implementations/reference.py

@@ -935,6 +935,10 @@ class ReferenceFileSystem(AsyncFileSystem):
 
     def _process_references0(self, references):
         """Make reference dict for Spec Version 0"""
+        references = {
+            key: json.dumps(val) if isinstance(val, dict) else val
+            for key, val in references.items()
+        }
         self.references = references
 
     def _process_references1(self, references, template_overrides=None):
@@ -952,6 +956,8 @@ class ReferenceFileSystem(AsyncFileSystem):
                 if v.startswith("base64:"):
                     self.references[k] = base64.b64decode(v[7:])
                 self.references[k] = v
+            elif isinstance(v, dict):
+                self.references[k] = json.dumps(v)
             elif self.templates:
                 u = v[0]
                 if "{{" in u:

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/implementations/smb.py

@@ -68,6 +68,7 @@ class SMBFileSystem(AbstractFileSystem):
         encrypt=None,
         share_access=None,
         register_session_retries=5,
+        auto_mkdir=False,
         **kwargs,
     ):
         """
@@ -102,6 +103,10 @@ class SMBFileSystem(AbstractFileSystem):
             - 'r': Allow other handles to be opened with read access.
             - 'w': Allow other handles to be opened with write access.
             - 'd': Allow other handles to be opened with delete access.
+        auto_mkdir: bool
+            Whether, when opening a file, the directory containing it should
+            be created (if it doesn't already exist). This is assumed by pyarrow
+            and zarr-python code.
         """
         super().__init__(**kwargs)
         self.host = host
@@ -113,6 +118,7 @@ class SMBFileSystem(AbstractFileSystem):
         self.temppath = kwargs.pop("temppath", "")
         self.share_access = share_access
         self.register_session_retries = register_session_retries
+        self.auto_mkdir = auto_mkdir
         self._connect()
 
     @property
@@ -224,6 +230,8 @@ class SMBFileSystem(AbstractFileSystem):
         By specifying 'share_access' in 'kwargs' it is possible to override the
         default shared access setting applied in the constructor of this object.
         """
+        if self.auto_mkdir and "w" in mode:
+            self.makedirs(self._parent(path), exist_ok=True)
         bls = block_size if block_size is not None and block_size >= 0 else -1
         wpath = _as_unc_path(self.host, path)
         share_access = kwargs.pop("share_access", self.share_access)
@@ -245,6 +253,8 @@ class SMBFileSystem(AbstractFileSystem):
         """Copy within two locations in the same filesystem"""
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
+        if self.auto_mkdir:
+            self.makedirs(self._parent(path2), exist_ok=True)
         smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs)
 
     def _rm(self, path):

fsspec-2024.6.0/fsspec/json.py

@@ -0,0 +1,81 @@
+import json
+from contextlib import suppress
+from pathlib import PurePath
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from .registry import _import_class, get_filesystem_class
+from .spec import AbstractFileSystem
+
+
+class FilesystemJSONEncoder(json.JSONEncoder):
+    def default(self, o: Any) -> Any:
+        if isinstance(o, AbstractFileSystem):
+            return o.to_dict()
+        if isinstance(o, PurePath):
+            cls = type(o)
+            return {"cls": f"{cls.__module__}.{cls.__name__}", "str": str(o)}
+
+        return super().default(o)
+
+
+class FilesystemJSONDecoder(json.JSONDecoder):
+    def __init__(
+        self,
+        *,
+        object_hook: Optional[Callable[[Dict[str, Any]], Any]] = None,
+        parse_float: Optional[Callable[[str], Any]] = None,
+        parse_int: Optional[Callable[[str], Any]] = None,
+        parse_constant: Optional[Callable[[str], Any]] = None,
+        strict: bool = True,
+        object_pairs_hook: Optional[Callable[[List[Tuple[str, Any]]], Any]] = None,
+    ) -> None:
+        self.original_object_hook = object_hook
+
+        super().__init__(
+            object_hook=self.custom_object_hook,
+            parse_float=parse_float,
+            parse_int=parse_int,
+            parse_constant=parse_constant,
+            strict=strict,
+            object_pairs_hook=object_pairs_hook,
+        )
+
+    @classmethod
+    def try_resolve_path_cls(cls, dct: Dict[str, Any]):
+        with suppress(Exception):
+            fqp = dct["cls"]
+
+            path_cls = _import_class(fqp)
+
+            if issubclass(path_cls, PurePath):
+                return path_cls
+
+        return None
+
+    @classmethod
+    def try_resolve_fs_cls(cls, dct: Dict[str, Any]):
+        with suppress(Exception):
+            if "cls" in dct:
+                try:
+                    fs_cls = _import_class(dct["cls"])
+                    if issubclass(fs_cls, AbstractFileSystem):
+                        return fs_cls
+                except Exception:
+                    if "protocol" in dct:  # Fallback if cls cannot be imported
+                        return get_filesystem_class(dct["protocol"])
+
+                    raise
+
+        return None
+
+    def custom_object_hook(self, dct: Dict[str, Any]):
+        if "cls" in dct:
+            if (obj_cls := self.try_resolve_fs_cls(dct)) is not None:
+                return AbstractFileSystem.from_dict(dct)
+            if (obj_cls := self.try_resolve_path_cls(dct)) is not None:
+                return obj_cls(dct["str"])
+
+        if self.original_object_hook is not None:
+            return self.original_object_hook(dct)
+
+        return dct

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/registry.py

@@ -257,27 +257,33 @@ update the current installation.
 """
 
 
-def _import_class(cls, minv=None):
-    """Take a string FQP and return the imported class or identifier
+def _import_class(fqp: str):
+    """Take a fully-qualified path and return the imported class or identifier.
 
-    cls is of the form "package.module.klass" or "package.module:subobject.klass"
+    ``fqp`` is of the form "package.module.klass" or
+    "package.module:subobject.klass".
+
+    Warnings
+    --------
+    This can import arbitrary modules. Make sure you haven't installed any modules
+    that may execute malicious code at import time.
     """
-    if ":" in cls:
-        mod, name = cls.rsplit(":", 1)
-        s3 = mod == "s3fs"
-        mod = importlib.import_module(mod)
-        if s3 and mod.__version__.split(".") < ["0", "5"]:
-            warnings.warn(s3_msg)
-        for part in name.split("."):
-            mod = getattr(mod, part)
-        return mod
+    if ":" in fqp:
+        mod, name = fqp.rsplit(":", 1)
     else:
-        mod, name = cls.rsplit(".", 1)
-        s3 = mod == "s3fs"
-        mod = importlib.import_module(mod)
-        if s3 and mod.__version__.split(".") < ["0", "5"]:
-            warnings.warn(s3_msg)
-        return getattr(mod, name)
+        mod, name = fqp.rsplit(".", 1)
+
+    is_s3 = mod == "s3fs"
+    mod = importlib.import_module(mod)
+    if is_s3 and mod.__version__.split(".") < ["0", "5"]:
+        warnings.warn(s3_msg)
+    for part in name.split("."):
+        mod = getattr(mod, part)
+
+    if not isinstance(mod, type):
+        raise TypeError(f"{fqp} is not a class")
+
+    return mod
 
 
 def filesystem(protocol, **storage_options):

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/spec.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import io
+import json
 import logging
 import os
 import threading
@@ -9,7 +10,7 @@ import weakref
 from errno import ESPIPE
 from glob import has_magic
 from hashlib import sha256
-from typing import ClassVar
+from typing import Any, ClassVar, Dict, Tuple
 
 from .callbacks import DEFAULT_CALLBACK
 from .config import apply_config, conf
@@ -115,6 +116,10 @@ class AbstractFileSystem(metaclass=_Cached):
     #: Extra *class attributes* that should be considered when hashing.
     _extra_tokenize_attributes = ()
 
+    # Set by _Cached metaclass
+    storage_args: Tuple[Any, ...]
+    storage_options: Dict[str, Any]
+
     def __init__(self, *args, **storage_options):
         """Create and configure file-system instance
 
@@ -1381,61 +1386,98 @@ class AbstractFileSystem(metaclass=_Cached):
                 length = size - offset
             return read_block(f, offset, length, delimiter)
 
-    def to_json(self):
+    def to_json(self) -> str:
         """
-        JSON representation of this filesystem instance
+        JSON representation of this filesystem instance.
 
         Returns
         -------
-        str: JSON structure with keys cls (the python location of this class),
-            protocol (text name of this class's protocol, first one in case of
-            multiple), args (positional args, usually empty), and all other
-            kwargs as their own keys.
+        JSON string with keys ``cls`` (the python location of this class),
+        protocol (text name of this class's protocol, first one in case of
+        multiple), ``args`` (positional args, usually empty), and all other
+        keyword arguments as their own keys.
+        """
+        from .json import FilesystemJSONEncoder
+
+        return json.dumps(self, cls=FilesystemJSONEncoder)
+
+    @staticmethod
+    def from_json(blob: str) -> AbstractFileSystem:
         """
-        import json
+        Recreate a filesystem instance from JSON representation.
+
+        See ``.to_json()`` for the expected structure of the input.
+
+        Parameters
+        ----------
+        blob: str
+
+        Returns
+        -------
+        file system instance, not necessarily of this particular class.
 
+        Warnings
+        --------
+        This can import arbitrary modules (as determined by the ``cls`` key).
+        Make sure you haven't installed any modules that may execute malicious code
+        at import time.
+        """
+        from .json import FilesystemJSONDecoder
+
+        return json.loads(blob, cls=FilesystemJSONDecoder)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        JSON-serializable dictionary representation of this filesystem instance.
+
+        Returns
+        -------
+        Dictionary with keys ``cls`` (the python location of this class),
+        protocol (text name of this class's protocol, first one in case of
+        multiple), ``args`` (positional args, usually empty), and all other
+        keyword arguments as their own keys.
+        """
         cls = type(self)
-        cls = ".".join((cls.__module__, cls.__name__))
-        proto = (
-            self.protocol[0]
-            if isinstance(self.protocol, (tuple, list))
-            else self.protocol
-        )
-        return json.dumps(
-            dict(
-                cls=cls,
-                protocol=proto,
-                args=self.storage_args,
-                **self.storage_options,
-            )
+        proto = self.protocol
+
+        return dict(
+            cls=f"{cls.__module__}:{cls.__name__}",
+            protocol=proto[0] if isinstance(proto, (tuple, list)) else proto,
+            args=self.storage_args,
+            **self.storage_options,
         )
 
     @staticmethod
-    def from_json(blob):
+    def from_dict(dct: Dict[str, Any]) -> AbstractFileSystem:
         """
-        Recreate a filesystem instance from JSON representation
+        Recreate a filesystem instance from dictionary representation.
 
-        See ``.to_json()`` for the expected structure of the input
+        See ``.to_dict()`` for the expected structure of the input.
 
         Parameters
         ----------
-        blob: str
+        dct: Dict[str, Any]
 
         Returns
         -------
         file system instance, not necessarily of this particular class.
+
+        Warnings
+        --------
+        This can import arbitrary modules (as determined by the ``cls`` key).
+        Make sure you haven't installed any modules that may execute malicious code
+        at import time.
         """
-        import json
+        from .json import FilesystemJSONDecoder
 
-        from .registry import _import_class, get_filesystem_class
+        cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct)
+        if cls is None:
+            raise ValueError("Not a serialized AbstractFileSystem")
 
-        dic = json.loads(blob)
-        protocol = dic.pop("protocol")
-        try:
-            cls = _import_class(dic.pop("cls"))
-        except (ImportError, ValueError, RuntimeError, KeyError):
-            cls = get_filesystem_class(protocol)
-        return cls(*dic.pop("args", ()), **dic)
+        dct.pop("cls", None)
+        dct.pop("protocol", None)
+
+        return cls(*dct.pop("args", ()), **dct)
 
     def _get_pyarrow_filesystem(self):
         """

{fsspec-2024.5.0 → fsspec-2024.6.0}/fsspec/utils.py

@@ -435,7 +435,7 @@ def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
 
 def get_protocol(url: str) -> str:
     url = stringify_path(url)
-    parts = re.split(r"(\:\:|\://)", url, 1)
+    parts = re.split(r"(\:\:|\://)", url, maxsplit=1)
     if len(parts) > 1:
         return parts[0]
     return "file"