fsspec 2023.9.2__tar.gz → 2023.12.0__tar.gz

{fsspec-2023.9.2/fsspec.egg-info → fsspec-2023.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fsspec
-Version: 2023.9.2
+Version: 2023.12.0
 Summary: File-system specification
 Home-page: https://github.com/fsspec/filesystem_spec
 Maintainer: Martin Durant
@@ -19,30 +19,74 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Provides-Extra: entrypoints
 Provides-Extra: abfs
+Requires-Dist: adlfs; extra == "abfs"
 Provides-Extra: adl
+Requires-Dist: adlfs; extra == "adl"
 Provides-Extra: dask
+Requires-Dist: dask; extra == "dask"
+Requires-Dist: distributed; extra == "dask"
 Provides-Extra: dropbox
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: requests; extra == "dropbox"
+Requires-Dist: dropbox; extra == "dropbox"
 Provides-Extra: gcs
+Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: git
+Requires-Dist: pygit2; extra == "git"
 Provides-Extra: github
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gs
+Requires-Dist: gcsfs; extra == "gs"
 Provides-Extra: hdfs
+Requires-Dist: pyarrow>=1; extra == "hdfs"
 Provides-Extra: arrow
+Requires-Dist: pyarrow>=1; extra == "arrow"
 Provides-Extra: http
+Requires-Dist: requests; extra == "http"
+Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == "http"
 Provides-Extra: sftp
+Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: s3
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: oci
+Requires-Dist: ocifs; extra == "oci"
 Provides-Extra: smb
+Requires-Dist: smbprotocol; extra == "smb"
 Provides-Extra: ssh
+Requires-Dist: paramiko; extra == "ssh"
 Provides-Extra: fuse
+Requires-Dist: fusepy; extra == "fuse"
 Provides-Extra: libarchive
+Requires-Dist: libarchive-c; extra == "libarchive"
 Provides-Extra: gui
+Requires-Dist: panel; extra == "gui"
 Provides-Extra: tqdm
+Requires-Dist: tqdm; extra == "tqdm"
 Provides-Extra: full
+Requires-Dist: adlfs; extra == "full"
+Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == "full"
+Requires-Dist: dask; extra == "full"
+Requires-Dist: distributed; extra == "full"
+Requires-Dist: dropbox; extra == "full"
+Requires-Dist: dropboxdrivefs; extra == "full"
+Requires-Dist: fusepy; extra == "full"
+Requires-Dist: gcsfs; extra == "full"
+Requires-Dist: libarchive-c; extra == "full"
+Requires-Dist: ocifs; extra == "full"
+Requires-Dist: panel; extra == "full"
+Requires-Dist: paramiko; extra == "full"
+Requires-Dist: pyarrow>=1; extra == "full"
+Requires-Dist: pygit2; extra == "full"
+Requires-Dist: requests; extra == "full"
+Requires-Dist: s3fs; extra == "full"
+Requires-Dist: smbprotocol; extra == "full"
+Requires-Dist: tqdm; extra == "full"
 Provides-Extra: devel
-License-File: LICENSE
+Requires-Dist: pytest; extra == "devel"
+Requires-Dist: pytest-cov; extra == "devel"
 
 # filesystem_spec
 
@@ -50,6 +94,7 @@ License-File: LICENSE
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
+[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
 
 A specification for pythonic filesystems.
 

{fsspec-2023.9.2 → fsspec-2023.12.0}/README.md

@@ -4,6 +4,7 @@
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
+[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
 
 A specification for pythonic filesystems.
 

{fsspec-2023.9.2 → fsspec-2023.12.0}/fsspec/__init__.py

@@ -48,10 +48,15 @@ def process_entries():
                 specs = eps.select(group="fsspec.specs")
             else:
                 specs = eps.get("fsspec.specs", [])
+            registered_names = {}
             for spec in specs:
                 err_msg = f"Unable to load filesystem from {spec}"
+                name = spec.name
+                if name in registered_names:
+                    continue
+                registered_names[name] = True
                 register_implementation(
-                    spec.name,
+                    name,
                     spec.value.replace(":", "."),
                     errtxt=err_msg,
                     # We take our implementations as the ones to overload with if

{fsspec-2023.9.2 → fsspec-2023.12.0}/fsspec/_version.py

@@ -1,5 +1,5 @@
 
-# This file was generated by 'versioneer.py' (0.20) from
+# This file was generated by 'versioneer.py' (0.29) from
 # revision-control system data, or from the parent directory name of an
 # unpacked source archive. Distribution tarballs contain a pre-generated copy
 # of this file.
@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-09-22T14:30:26-0400",
+ "date": "2023-12-02T20:51:30-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "3ad74814c8bd8620b3aa434080218eb658a81a6c",
- "version": "2023.9.2"
+ "full-revisionid": "5cf9cd952c5d276835d3caef9c32fcf69d55b10c",
+ "version": "2023.12.0"
 }
 '''  # END VERSION_JSON
 

{fsspec-2023.9.2 → fsspec-2023.12.0}/fsspec/archive.py

@@ -13,7 +13,7 @@ class AbstractArchiveFileSystem(AbstractFileSystem):
     """
 
     def __str__(self):
-        return "<Archive-like object %s at %s>" % (type(self).__name__, id(self))
+        return f"<Archive-like object {type(self).__name__} at {id(self)}>"
 
     __repr__ = __str__
 

{fsspec-2023.9.2 → fsspec-2023.12.0}/fsspec/asyn.py

@@ -15,7 +15,7 @@ from .callbacks import _DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
 from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep
 from .spec import AbstractBufferedFile, AbstractFileSystem
-from .utils import is_exception, other_paths
+from .utils import glob_translate, is_exception, other_paths
 
 private = re.compile("_[^_]")
 iothread = [None]  # dedicated fsspec IO thread
@@ -106,7 +106,7 @@ def sync(loop, func, *args, timeout=None, **kwargs):
 
 
 def sync_wrapper(func, obj=None):
-    """Given a function, make so can be called in async or blocking contexts
+    """Given a function, make so can be called in blocking contexts
 
     Leave obj=None if defining within a class. Pass the instance if attaching
     as an attribute of the instance.
@@ -426,7 +426,7 @@ class AsyncFileSystem(AbstractFileSystem):
                 end = ""
             if isinstance(end, numbers.Integral):
                 end -= 1  # bytes range is inclusive
-        return "bytes=%s-%s" % (start, end)
+        return f"bytes={start}-{end}"
 
     async def _cat_file(self, path, start=None, end=None, **kwargs):
         raise NotImplementedError
@@ -467,6 +467,16 @@ class AsyncFileSystem(AbstractFileSystem):
         on_error="return",
         **kwargs,
     ):
+        """Get the contents of byte ranges from one or more files
+
+        Parameters
+        ----------
+        paths: list
+            A list of of filepaths on this filesystems
+        starts, ends: int or list
+            Bytes limits of the read. If using a single int, the same value will be
+            used to read all the specified files.
+        """
         # TODO: on_error
         if max_gap is not None:
             # use utils.merge_offset_ranges
@@ -476,7 +486,7 @@ class AsyncFileSystem(AbstractFileSystem):
         if not isinstance(starts, Iterable):
             starts = [starts] * len(paths)
         if not isinstance(ends, Iterable):
-            ends = [starts] * len(paths)
+            ends = [ends] * len(paths)
         if len(starts) != len(paths) or len(ends) != len(paths):
             raise ValueError
         coros = [
@@ -662,9 +672,9 @@ class AsyncFileSystem(AbstractFileSystem):
             [self._size(p) for p in paths], batch_size=batch_size
         )
 
-    async def _exists(self, path):
+    async def _exists(self, path, **kwargs):
         try:
-            await self._info(path)
+            await self._info(path, **kwargs)
             return True
         except FileNotFoundError:
             return False
@@ -735,8 +745,12 @@ class AsyncFileSystem(AbstractFileSystem):
 
         import re
 
-        ends = path.endswith("/")
+        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
+        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_sep or path.endswith(
+            tuple(sep + "**" for sep in seps)
+        )
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
@@ -746,11 +760,11 @@ class AsyncFileSystem(AbstractFileSystem):
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            if await self._exists(path):
+            if await self._exists(path, **kwargs):
                 if not detail:
                     return [path]
                 else:
-                    return {path: await self._info(path)}
+                    return {path: await self._info(path, **kwargs)}
             else:
                 if not detail:
                     return []  # glob of non-existent returns empty
@@ -775,46 +789,22 @@ class AsyncFileSystem(AbstractFileSystem):
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=True, detail=True, **kwargs
         )
-        # Escape characters special to python regex, leaving our supported
-        # special characters in place.
-        # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html
-        # for shell globbing details.
-        pattern = (
-            "^"
-            + (
-                path.replace("\\", r"\\")
-                .replace(".", r"\.")
-                .replace("+", r"\+")
-                .replace("//", "/")
-                .replace("(", r"\(")
-                .replace(")", r"\)")
-                .replace("|", r"\|")
-                .replace("^", r"\^")
-                .replace("$", r"\$")
-                .replace("{", r"\{")
-                .replace("}", r"\}")
-                .rstrip("/")
-                .replace("?", ".")
-            )
-            + "$"
-        )
-        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
-        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+
+        pattern = glob_translate(path + ("/" if ends_with_sep else ""))
         pattern = re.compile(pattern)
+
         out = {
-            p: allpaths[p]
-            for p in sorted(allpaths)
-            if pattern.match(p.replace("//", "/").rstrip("/"))
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                (
+                    p + "/"
+                    if append_slash_to_dirname and info["type"] == "directory"
+                    else p
+                )
+            )
         }
 
-        # Return directories only when the glob end by a slash
-        # This is needed for posix glob compliance
-        if ends:
-            out = {k: v for k, v in out.items() if v["type"] == "directory"}
-
         if detail:
             return out
         else: