fsspec 2024.3.1__py3-none-any.whl → 2024.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fsspec/__init__.py +2 -3
- fsspec/_version.py +14 -19
- fsspec/caching.py +86 -16
- fsspec/compression.py +2 -1
- fsspec/core.py +32 -8
- fsspec/exceptions.py +1 -0
- fsspec/generic.py +1 -1
- fsspec/gui.py +1 -1
- fsspec/implementations/arrow.py +0 -2
- fsspec/implementations/cache_mapper.py +1 -2
- fsspec/implementations/cache_metadata.py +7 -7
- fsspec/implementations/cached.py +1 -13
- fsspec/implementations/dirfs.py +2 -2
- fsspec/implementations/github.py +12 -0
- fsspec/implementations/http.py +9 -9
- fsspec/implementations/local.py +78 -45
- fsspec/implementations/memory.py +9 -0
- fsspec/implementations/reference.py +6 -0
- fsspec/implementations/smb.py +13 -1
- fsspec/implementations/webhdfs.py +1 -3
- fsspec/json.py +81 -0
- fsspec/parquet.py +0 -8
- fsspec/registry.py +28 -18
- fsspec/spec.py +97 -38
- fsspec/tests/abstract/mv.py +57 -0
- fsspec/utils.py +1 -3
- fsspec-2024.6.0.dist-info/METADATA +279 -0
- fsspec-2024.6.0.dist-info/RECORD +55 -0
- {fsspec-2024.3.1.dist-info → fsspec-2024.6.0.dist-info}/WHEEL +1 -2
- fsspec-2024.3.1.dist-info/METADATA +0 -167
- fsspec-2024.3.1.dist-info/RECORD +0 -54
- fsspec-2024.3.1.dist-info/top_level.txt +0 -1
- {fsspec-2024.3.1.dist-info → fsspec-2024.6.0.dist-info/licenses}/LICENSE +0 -0
fsspec/spec.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import io
|
|
4
|
+
import json
|
|
4
5
|
import logging
|
|
5
6
|
import os
|
|
6
7
|
import threading
|
|
@@ -9,7 +10,7 @@ import weakref
|
|
|
9
10
|
from errno import ESPIPE
|
|
10
11
|
from glob import has_magic
|
|
11
12
|
from hashlib import sha256
|
|
12
|
-
from typing import ClassVar
|
|
13
|
+
from typing import Any, ClassVar, Dict, Tuple
|
|
13
14
|
|
|
14
15
|
from .callbacks import DEFAULT_CALLBACK
|
|
15
16
|
from .config import apply_config, conf
|
|
@@ -115,6 +116,10 @@ class AbstractFileSystem(metaclass=_Cached):
|
|
|
115
116
|
#: Extra *class attributes* that should be considered when hashing.
|
|
116
117
|
_extra_tokenize_attributes = ()
|
|
117
118
|
|
|
119
|
+
# Set by _Cached metaclass
|
|
120
|
+
storage_args: Tuple[Any, ...]
|
|
121
|
+
storage_options: Dict[str, Any]
|
|
122
|
+
|
|
118
123
|
def __init__(self, *args, **storage_options):
|
|
119
124
|
"""Create and configure file-system instance
|
|
120
125
|
|
|
@@ -358,8 +363,10 @@ class AbstractFileSystem(metaclass=_Cached):
|
|
|
358
363
|
but contains nothing), None if not in cache.
|
|
359
364
|
"""
|
|
360
365
|
parent = self._parent(path)
|
|
361
|
-
|
|
366
|
+
try:
|
|
362
367
|
return self.dircache[path.rstrip("/")]
|
|
368
|
+
except KeyError:
|
|
369
|
+
pass
|
|
363
370
|
try:
|
|
364
371
|
files = [
|
|
365
372
|
f
|
|
@@ -1175,7 +1182,10 @@ class AbstractFileSystem(metaclass=_Cached):
|
|
|
1175
1182
|
if path1 == path2:
|
|
1176
1183
|
logger.debug("%s mv: The paths are the same, so no files were moved.", self)
|
|
1177
1184
|
else:
|
|
1178
|
-
|
|
1185
|
+
# explicitly raise exception to prevent data corruption
|
|
1186
|
+
self.copy(
|
|
1187
|
+
path1, path2, recursive=recursive, maxdepth=maxdepth, onerror="raise"
|
|
1188
|
+
)
|
|
1179
1189
|
self.rm(path1, recursive=recursive)
|
|
1180
1190
|
|
|
1181
1191
|
def rm_file(self, path):
|
|
@@ -1376,61 +1386,98 @@ class AbstractFileSystem(metaclass=_Cached):
|
|
|
1376
1386
|
length = size - offset
|
|
1377
1387
|
return read_block(f, offset, length, delimiter)
|
|
1378
1388
|
|
|
1379
|
-
def to_json(self):
|
|
1389
|
+
def to_json(self) -> str:
|
|
1380
1390
|
"""
|
|
1381
|
-
JSON representation of this filesystem instance
|
|
1391
|
+
JSON representation of this filesystem instance.
|
|
1382
1392
|
|
|
1383
1393
|
Returns
|
|
1384
1394
|
-------
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1395
|
+
JSON string with keys ``cls`` (the python location of this class),
|
|
1396
|
+
protocol (text name of this class's protocol, first one in case of
|
|
1397
|
+
multiple), ``args`` (positional args, usually empty), and all other
|
|
1398
|
+
keyword arguments as their own keys.
|
|
1389
1399
|
"""
|
|
1390
|
-
import
|
|
1400
|
+
from .json import FilesystemJSONEncoder
|
|
1401
|
+
|
|
1402
|
+
return json.dumps(self, cls=FilesystemJSONEncoder)
|
|
1403
|
+
|
|
1404
|
+
@staticmethod
|
|
1405
|
+
def from_json(blob: str) -> AbstractFileSystem:
|
|
1406
|
+
"""
|
|
1407
|
+
Recreate a filesystem instance from JSON representation.
|
|
1408
|
+
|
|
1409
|
+
See ``.to_json()`` for the expected structure of the input.
|
|
1391
1410
|
|
|
1411
|
+
Parameters
|
|
1412
|
+
----------
|
|
1413
|
+
blob: str
|
|
1414
|
+
|
|
1415
|
+
Returns
|
|
1416
|
+
-------
|
|
1417
|
+
file system instance, not necessarily of this particular class.
|
|
1418
|
+
|
|
1419
|
+
Warnings
|
|
1420
|
+
--------
|
|
1421
|
+
This can import arbitrary modules (as determined by the ``cls`` key).
|
|
1422
|
+
Make sure you haven't installed any modules that may execute malicious code
|
|
1423
|
+
at import time.
|
|
1424
|
+
"""
|
|
1425
|
+
from .json import FilesystemJSONDecoder
|
|
1426
|
+
|
|
1427
|
+
return json.loads(blob, cls=FilesystemJSONDecoder)
|
|
1428
|
+
|
|
1429
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
1430
|
+
"""
|
|
1431
|
+
JSON-serializable dictionary representation of this filesystem instance.
|
|
1432
|
+
|
|
1433
|
+
Returns
|
|
1434
|
+
-------
|
|
1435
|
+
Dictionary with keys ``cls`` (the python location of this class),
|
|
1436
|
+
protocol (text name of this class's protocol, first one in case of
|
|
1437
|
+
multiple), ``args`` (positional args, usually empty), and all other
|
|
1438
|
+
keyword arguments as their own keys.
|
|
1439
|
+
"""
|
|
1392
1440
|
cls = type(self)
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
else
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
dict(
|
|
1401
|
-
cls=cls,
|
|
1402
|
-
protocol=proto,
|
|
1403
|
-
args=self.storage_args,
|
|
1404
|
-
**self.storage_options,
|
|
1405
|
-
)
|
|
1441
|
+
proto = self.protocol
|
|
1442
|
+
|
|
1443
|
+
return dict(
|
|
1444
|
+
cls=f"{cls.__module__}:{cls.__name__}",
|
|
1445
|
+
protocol=proto[0] if isinstance(proto, (tuple, list)) else proto,
|
|
1446
|
+
args=self.storage_args,
|
|
1447
|
+
**self.storage_options,
|
|
1406
1448
|
)
|
|
1407
1449
|
|
|
1408
1450
|
@staticmethod
|
|
1409
|
-
def
|
|
1451
|
+
def from_dict(dct: Dict[str, Any]) -> AbstractFileSystem:
|
|
1410
1452
|
"""
|
|
1411
|
-
Recreate a filesystem instance from
|
|
1453
|
+
Recreate a filesystem instance from dictionary representation.
|
|
1412
1454
|
|
|
1413
|
-
See ``.
|
|
1455
|
+
See ``.to_dict()`` for the expected structure of the input.
|
|
1414
1456
|
|
|
1415
1457
|
Parameters
|
|
1416
1458
|
----------
|
|
1417
|
-
|
|
1459
|
+
dct: Dict[str, Any]
|
|
1418
1460
|
|
|
1419
1461
|
Returns
|
|
1420
1462
|
-------
|
|
1421
1463
|
file system instance, not necessarily of this particular class.
|
|
1464
|
+
|
|
1465
|
+
Warnings
|
|
1466
|
+
--------
|
|
1467
|
+
This can import arbitrary modules (as determined by the ``cls`` key).
|
|
1468
|
+
Make sure you haven't installed any modules that may execute malicious code
|
|
1469
|
+
at import time.
|
|
1422
1470
|
"""
|
|
1423
|
-
import
|
|
1471
|
+
from .json import FilesystemJSONDecoder
|
|
1424
1472
|
|
|
1425
|
-
|
|
1473
|
+
cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct)
|
|
1474
|
+
if cls is None:
|
|
1475
|
+
raise ValueError("Not a serialized AbstractFileSystem")
|
|
1426
1476
|
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
except (ImportError, ValueError, RuntimeError, KeyError):
|
|
1432
|
-
cls = get_filesystem_class(protocol)
|
|
1433
|
-
return cls(*dic.pop("args", ()), **dic)
|
|
1477
|
+
dct.pop("cls", None)
|
|
1478
|
+
dct.pop("protocol", None)
|
|
1479
|
+
|
|
1480
|
+
return cls(*dct.pop("args", ()), **dct)
|
|
1434
1481
|
|
|
1435
1482
|
def _get_pyarrow_filesystem(self):
|
|
1436
1483
|
"""
|
|
@@ -1693,7 +1740,12 @@ class AbstractBufferedFile(io.IOBase):
|
|
|
1693
1740
|
"""Files are equal if they have the same checksum, only in read mode"""
|
|
1694
1741
|
if self is other:
|
|
1695
1742
|
return True
|
|
1696
|
-
return
|
|
1743
|
+
return (
|
|
1744
|
+
isinstance(other, type(self))
|
|
1745
|
+
and self.mode == "rb"
|
|
1746
|
+
and other.mode == "rb"
|
|
1747
|
+
and hash(self) == hash(other)
|
|
1748
|
+
)
|
|
1697
1749
|
|
|
1698
1750
|
def commit(self):
|
|
1699
1751
|
"""Move from temp to final destination"""
|
|
@@ -1839,11 +1891,18 @@ class AbstractBufferedFile(io.IOBase):
|
|
|
1839
1891
|
length = self.size - self.loc
|
|
1840
1892
|
if self.closed:
|
|
1841
1893
|
raise ValueError("I/O operation on closed file.")
|
|
1842
|
-
logger.debug("%s read: %i - %i", self, self.loc, self.loc + length)
|
|
1843
1894
|
if length == 0:
|
|
1844
1895
|
# don't even bother calling fetch
|
|
1845
1896
|
return b""
|
|
1846
1897
|
out = self.cache._fetch(self.loc, self.loc + length)
|
|
1898
|
+
|
|
1899
|
+
logger.debug(
|
|
1900
|
+
"%s read: %i - %i %s",
|
|
1901
|
+
self,
|
|
1902
|
+
self.loc,
|
|
1903
|
+
self.loc + length,
|
|
1904
|
+
self.cache._log_stats(),
|
|
1905
|
+
)
|
|
1847
1906
|
self.loc += len(out)
|
|
1848
1907
|
return out
|
|
1849
1908
|
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
import fsspec
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_move_raises_error_with_tmpdir(tmpdir):
|
|
9
|
+
# Create a file in the temporary directory
|
|
10
|
+
source = tmpdir.join("source_file.txt")
|
|
11
|
+
source.write("content")
|
|
12
|
+
|
|
13
|
+
# Define a destination that simulates a protected or invalid path
|
|
14
|
+
destination = tmpdir.join("non_existent_directory/destination_file.txt")
|
|
15
|
+
|
|
16
|
+
# Instantiate the filesystem (assuming the local file system interface)
|
|
17
|
+
fs = fsspec.filesystem("file")
|
|
18
|
+
|
|
19
|
+
# Use the actual file paths as string
|
|
20
|
+
with pytest.raises(FileNotFoundError):
|
|
21
|
+
fs.mv(str(source), str(destination))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@pytest.mark.parametrize("recursive", (True, False))
|
|
25
|
+
def test_move_raises_error_with_tmpdir_permission(recursive, tmpdir):
|
|
26
|
+
# Create a file in the temporary directory
|
|
27
|
+
source = tmpdir.join("source_file.txt")
|
|
28
|
+
source.write("content")
|
|
29
|
+
|
|
30
|
+
# Create a protected directory (non-writable)
|
|
31
|
+
protected_dir = tmpdir.mkdir("protected_directory")
|
|
32
|
+
protected_path = str(protected_dir)
|
|
33
|
+
|
|
34
|
+
# Set the directory to read-only
|
|
35
|
+
if os.name == "nt":
|
|
36
|
+
os.system(f'icacls "{protected_path}" /deny Everyone:(W)')
|
|
37
|
+
else:
|
|
38
|
+
os.chmod(protected_path, 0o555) # Sets the directory to read-only
|
|
39
|
+
|
|
40
|
+
# Define a destination inside the protected directory
|
|
41
|
+
destination = protected_dir.join("destination_file.txt")
|
|
42
|
+
|
|
43
|
+
# Instantiate the filesystem (assuming the local file system interface)
|
|
44
|
+
fs = fsspec.filesystem("file")
|
|
45
|
+
|
|
46
|
+
# Try to move the file to the read-only directory, expecting a permission error
|
|
47
|
+
with pytest.raises(PermissionError):
|
|
48
|
+
fs.mv(str(source), str(destination), recursive=recursive)
|
|
49
|
+
|
|
50
|
+
# Assert the file was not created in the destination
|
|
51
|
+
assert not os.path.exists(destination)
|
|
52
|
+
|
|
53
|
+
# Cleanup: Restore permissions so the directory can be cleaned up
|
|
54
|
+
if os.name == "nt":
|
|
55
|
+
os.system(f'icacls "{protected_path}" /remove:d Everyone')
|
|
56
|
+
else:
|
|
57
|
+
os.chmod(protected_path, 0o755) # Restore write permission for cleanup
|
fsspec/utils.py
CHANGED
|
@@ -350,8 +350,6 @@ def stringify_path(filepath: str | os.PathLike[str] | pathlib.Path) -> str:
|
|
|
350
350
|
return filepath
|
|
351
351
|
elif hasattr(filepath, "__fspath__"):
|
|
352
352
|
return filepath.__fspath__()
|
|
353
|
-
elif isinstance(filepath, pathlib.Path):
|
|
354
|
-
return str(filepath)
|
|
355
353
|
elif hasattr(filepath, "path"):
|
|
356
354
|
return filepath.path
|
|
357
355
|
else:
|
|
@@ -437,7 +435,7 @@ def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
|
|
|
437
435
|
|
|
438
436
|
def get_protocol(url: str) -> str:
|
|
439
437
|
url = stringify_path(url)
|
|
440
|
-
parts = re.split(r"(\:\:|\://)", url, 1)
|
|
438
|
+
parts = re.split(r"(\:\:|\://)", url, maxsplit=1)
|
|
441
439
|
if len(parts) > 1:
|
|
442
440
|
return parts[0]
|
|
443
441
|
return "file"
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: fsspec
|
|
3
|
+
Version: 2024.6.0
|
|
4
|
+
Summary: File-system specification
|
|
5
|
+
Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
|
|
6
|
+
Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
|
|
7
|
+
Project-URL: Homepage, https://github.com/fsspec/filesystem_spec
|
|
8
|
+
Maintainer-email: Martin Durant <mdurant@anaconda.com>
|
|
9
|
+
License: BSD 3-Clause License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2018, Martin Durant
|
|
12
|
+
All rights reserved.
|
|
13
|
+
|
|
14
|
+
Redistribution and use in source and binary forms, with or without
|
|
15
|
+
modification, are permitted provided that the following conditions are met:
|
|
16
|
+
|
|
17
|
+
* Redistributions of source code must retain the above copyright notice, this
|
|
18
|
+
list of conditions and the following disclaimer.
|
|
19
|
+
|
|
20
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
|
21
|
+
this list of conditions and the following disclaimer in the documentation
|
|
22
|
+
and/or other materials provided with the distribution.
|
|
23
|
+
|
|
24
|
+
* Neither the name of the copyright holder nor the names of its
|
|
25
|
+
contributors may be used to endorse or promote products derived from
|
|
26
|
+
this software without specific prior written permission.
|
|
27
|
+
|
|
28
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
29
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
30
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
31
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
32
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
33
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
34
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
35
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
36
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
37
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
38
|
+
License-File: LICENSE
|
|
39
|
+
Keywords: file
|
|
40
|
+
Classifier: Development Status :: 4 - Beta
|
|
41
|
+
Classifier: Intended Audience :: Developers
|
|
42
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
43
|
+
Classifier: Operating System :: OS Independent
|
|
44
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
45
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
46
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
47
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
48
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
49
|
+
Requires-Python: >=3.8
|
|
50
|
+
Provides-Extra: abfs
|
|
51
|
+
Requires-Dist: adlfs; extra == 'abfs'
|
|
52
|
+
Provides-Extra: adl
|
|
53
|
+
Requires-Dist: adlfs; extra == 'adl'
|
|
54
|
+
Provides-Extra: arrow
|
|
55
|
+
Requires-Dist: pyarrow>=1; extra == 'arrow'
|
|
56
|
+
Provides-Extra: dask
|
|
57
|
+
Requires-Dist: dask; extra == 'dask'
|
|
58
|
+
Requires-Dist: distributed; extra == 'dask'
|
|
59
|
+
Provides-Extra: dev
|
|
60
|
+
Requires-Dist: pre-commit; extra == 'dev'
|
|
61
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
62
|
+
Provides-Extra: doc
|
|
63
|
+
Requires-Dist: numpydoc; extra == 'doc'
|
|
64
|
+
Requires-Dist: sphinx; extra == 'doc'
|
|
65
|
+
Requires-Dist: sphinx-design; extra == 'doc'
|
|
66
|
+
Requires-Dist: sphinx-rtd-theme; extra == 'doc'
|
|
67
|
+
Requires-Dist: yarl; extra == 'doc'
|
|
68
|
+
Provides-Extra: dropbox
|
|
69
|
+
Requires-Dist: dropbox; extra == 'dropbox'
|
|
70
|
+
Requires-Dist: dropboxdrivefs; extra == 'dropbox'
|
|
71
|
+
Requires-Dist: requests; extra == 'dropbox'
|
|
72
|
+
Provides-Extra: entrypoints
|
|
73
|
+
Provides-Extra: full
|
|
74
|
+
Requires-Dist: adlfs; extra == 'full'
|
|
75
|
+
Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'full'
|
|
76
|
+
Requires-Dist: dask; extra == 'full'
|
|
77
|
+
Requires-Dist: distributed; extra == 'full'
|
|
78
|
+
Requires-Dist: dropbox; extra == 'full'
|
|
79
|
+
Requires-Dist: dropboxdrivefs; extra == 'full'
|
|
80
|
+
Requires-Dist: fusepy; extra == 'full'
|
|
81
|
+
Requires-Dist: gcsfs; extra == 'full'
|
|
82
|
+
Requires-Dist: libarchive-c; extra == 'full'
|
|
83
|
+
Requires-Dist: ocifs; extra == 'full'
|
|
84
|
+
Requires-Dist: panel; extra == 'full'
|
|
85
|
+
Requires-Dist: paramiko; extra == 'full'
|
|
86
|
+
Requires-Dist: pyarrow>=1; extra == 'full'
|
|
87
|
+
Requires-Dist: pygit2; extra == 'full'
|
|
88
|
+
Requires-Dist: requests; extra == 'full'
|
|
89
|
+
Requires-Dist: s3fs; extra == 'full'
|
|
90
|
+
Requires-Dist: smbprotocol; extra == 'full'
|
|
91
|
+
Requires-Dist: tqdm; extra == 'full'
|
|
92
|
+
Provides-Extra: fuse
|
|
93
|
+
Requires-Dist: fusepy; extra == 'fuse'
|
|
94
|
+
Provides-Extra: gcs
|
|
95
|
+
Requires-Dist: gcsfs; extra == 'gcs'
|
|
96
|
+
Provides-Extra: git
|
|
97
|
+
Requires-Dist: pygit2; extra == 'git'
|
|
98
|
+
Provides-Extra: github
|
|
99
|
+
Requires-Dist: requests; extra == 'github'
|
|
100
|
+
Provides-Extra: gs
|
|
101
|
+
Requires-Dist: gcsfs; extra == 'gs'
|
|
102
|
+
Provides-Extra: gui
|
|
103
|
+
Requires-Dist: panel; extra == 'gui'
|
|
104
|
+
Provides-Extra: hdfs
|
|
105
|
+
Requires-Dist: pyarrow>=1; extra == 'hdfs'
|
|
106
|
+
Provides-Extra: http
|
|
107
|
+
Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'http'
|
|
108
|
+
Provides-Extra: libarchive
|
|
109
|
+
Requires-Dist: libarchive-c; extra == 'libarchive'
|
|
110
|
+
Provides-Extra: oci
|
|
111
|
+
Requires-Dist: ocifs; extra == 'oci'
|
|
112
|
+
Provides-Extra: s3
|
|
113
|
+
Requires-Dist: s3fs; extra == 's3'
|
|
114
|
+
Provides-Extra: sftp
|
|
115
|
+
Requires-Dist: paramiko; extra == 'sftp'
|
|
116
|
+
Provides-Extra: smb
|
|
117
|
+
Requires-Dist: smbprotocol; extra == 'smb'
|
|
118
|
+
Provides-Extra: ssh
|
|
119
|
+
Requires-Dist: paramiko; extra == 'ssh'
|
|
120
|
+
Provides-Extra: test
|
|
121
|
+
Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test'
|
|
122
|
+
Requires-Dist: numpy; extra == 'test'
|
|
123
|
+
Requires-Dist: pytest; extra == 'test'
|
|
124
|
+
Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test'
|
|
125
|
+
Requires-Dist: pytest-benchmark; extra == 'test'
|
|
126
|
+
Requires-Dist: pytest-cov; extra == 'test'
|
|
127
|
+
Requires-Dist: pytest-mock; extra == 'test'
|
|
128
|
+
Requires-Dist: pytest-recording; extra == 'test'
|
|
129
|
+
Requires-Dist: pytest-rerunfailures; extra == 'test'
|
|
130
|
+
Requires-Dist: requests; extra == 'test'
|
|
131
|
+
Provides-Extra: test-downstream
|
|
132
|
+
Requires-Dist: aiobotocore<3.0.0,>=2.5.4; extra == 'test-downstream'
|
|
133
|
+
Requires-Dist: dask-expr; extra == 'test-downstream'
|
|
134
|
+
Requires-Dist: dask[dataframe,test]; extra == 'test-downstream'
|
|
135
|
+
Requires-Dist: moto[server]<5,>4; extra == 'test-downstream'
|
|
136
|
+
Requires-Dist: pytest-timeout; extra == 'test-downstream'
|
|
137
|
+
Requires-Dist: xarray; extra == 'test-downstream'
|
|
138
|
+
Provides-Extra: test-full
|
|
139
|
+
Requires-Dist: adlfs; extra == 'test-full'
|
|
140
|
+
Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test-full'
|
|
141
|
+
Requires-Dist: cloudpickle; extra == 'test-full'
|
|
142
|
+
Requires-Dist: dask; extra == 'test-full'
|
|
143
|
+
Requires-Dist: distributed; extra == 'test-full'
|
|
144
|
+
Requires-Dist: dropbox; extra == 'test-full'
|
|
145
|
+
Requires-Dist: dropboxdrivefs; extra == 'test-full'
|
|
146
|
+
Requires-Dist: fastparquet; extra == 'test-full'
|
|
147
|
+
Requires-Dist: fusepy; extra == 'test-full'
|
|
148
|
+
Requires-Dist: gcsfs; extra == 'test-full'
|
|
149
|
+
Requires-Dist: jinja2; extra == 'test-full'
|
|
150
|
+
Requires-Dist: kerchunk; extra == 'test-full'
|
|
151
|
+
Requires-Dist: libarchive-c; extra == 'test-full'
|
|
152
|
+
Requires-Dist: lz4; extra == 'test-full'
|
|
153
|
+
Requires-Dist: notebook; extra == 'test-full'
|
|
154
|
+
Requires-Dist: numpy; extra == 'test-full'
|
|
155
|
+
Requires-Dist: ocifs; extra == 'test-full'
|
|
156
|
+
Requires-Dist: pandas; extra == 'test-full'
|
|
157
|
+
Requires-Dist: panel; extra == 'test-full'
|
|
158
|
+
Requires-Dist: paramiko; extra == 'test-full'
|
|
159
|
+
Requires-Dist: pyarrow; extra == 'test-full'
|
|
160
|
+
Requires-Dist: pyarrow>=1; extra == 'test-full'
|
|
161
|
+
Requires-Dist: pyftpdlib; extra == 'test-full'
|
|
162
|
+
Requires-Dist: pygit2; extra == 'test-full'
|
|
163
|
+
Requires-Dist: pytest; extra == 'test-full'
|
|
164
|
+
Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test-full'
|
|
165
|
+
Requires-Dist: pytest-benchmark; extra == 'test-full'
|
|
166
|
+
Requires-Dist: pytest-cov; extra == 'test-full'
|
|
167
|
+
Requires-Dist: pytest-mock; extra == 'test-full'
|
|
168
|
+
Requires-Dist: pytest-recording; extra == 'test-full'
|
|
169
|
+
Requires-Dist: pytest-rerunfailures; extra == 'test-full'
|
|
170
|
+
Requires-Dist: python-snappy; extra == 'test-full'
|
|
171
|
+
Requires-Dist: requests; extra == 'test-full'
|
|
172
|
+
Requires-Dist: smbprotocol; extra == 'test-full'
|
|
173
|
+
Requires-Dist: tqdm; extra == 'test-full'
|
|
174
|
+
Requires-Dist: urllib3; extra == 'test-full'
|
|
175
|
+
Requires-Dist: zarr; extra == 'test-full'
|
|
176
|
+
Requires-Dist: zstandard; extra == 'test-full'
|
|
177
|
+
Provides-Extra: tqdm
|
|
178
|
+
Requires-Dist: tqdm; extra == 'tqdm'
|
|
179
|
+
Description-Content-Type: text/markdown
|
|
180
|
+
|
|
181
|
+
# filesystem_spec
|
|
182
|
+
|
|
183
|
+
[](https://pypi.python.org/pypi/fsspec/)
|
|
184
|
+
[](https://anaconda.org/conda-forge/fsspec)
|
|
185
|
+

|
|
186
|
+
[](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
|
|
187
|
+
[](https://piptrends.com/package/fsspec)
|
|
188
|
+
|
|
189
|
+
A specification for pythonic filesystems.
|
|
190
|
+
|
|
191
|
+
## Install
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
pip install fsspec
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
would install the base fsspec. Various optionally supported features might require specification of custom
|
|
198
|
+
extra require, e.g. `pip install fsspec[ssh]` will install dependencies for `ssh` backends support.
|
|
199
|
+
Use `pip install fsspec[full]` for installation of all known extra dependencies.
|
|
200
|
+
|
|
201
|
+
Up-to-date package also provided through conda-forge distribution:
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
conda install -c conda-forge fsspec
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
## Purpose
|
|
209
|
+
|
|
210
|
+
To produce a template or specification for a file-system interface, that specific implementations should follow,
|
|
211
|
+
so that applications making use of them can rely on a common behaviour and not have to worry about the specific
|
|
212
|
+
internal implementation decisions with any given backend. Many such implementations are included in this package,
|
|
213
|
+
or in sister projects such as `s3fs` and `gcsfs`.
|
|
214
|
+
|
|
215
|
+
In addition, if this is well-designed, then additional functionality, such as a key-value store or FUSE
|
|
216
|
+
mounting of the file-system implementation may be available for all implementations "for free".
|
|
217
|
+
|
|
218
|
+
## Documentation
|
|
219
|
+
|
|
220
|
+
Please refer to [RTD](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
|
|
221
|
+
|
|
222
|
+
## Develop
|
|
223
|
+
|
|
224
|
+
fsspec uses GitHub Actions for CI. Environment files can be found
|
|
225
|
+
in the "ci/" directory. Note that the main environment is called "py38",
|
|
226
|
+
but it is expected that the version of python installed be adjustable at
|
|
227
|
+
CI runtime. For local use, pick a version suitable for you.
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
# For a new environment (mamba / conda).
|
|
231
|
+
mamba create -n fsspec -c conda-forge python=3.9 -y
|
|
232
|
+
conda activate fsspec
|
|
233
|
+
|
|
234
|
+
# Standard dev install with docs and tests.
|
|
235
|
+
pip install -e ".[dev,doc,test]"
|
|
236
|
+
|
|
237
|
+
# Full tests except for downstream
|
|
238
|
+
pip install s3fs
|
|
239
|
+
pip uninstall s3fs
|
|
240
|
+
pip install -e .[dev,doc,test_full]
|
|
241
|
+
pip install s3fs --no-deps
|
|
242
|
+
pytest -v
|
|
243
|
+
|
|
244
|
+
# Downstream tests.
|
|
245
|
+
sh install_s3fs.sh
|
|
246
|
+
# Windows powershell.
|
|
247
|
+
install_s3fs.sh
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
### Testing
|
|
251
|
+
|
|
252
|
+
Tests can be run in the dev environment, if activated, via ``pytest fsspec``.
|
|
253
|
+
|
|
254
|
+
The full fsspec suite requires a system-level docker, docker-compose, and fuse
|
|
255
|
+
installation. If only making changes to one backend implementation, it is
|
|
256
|
+
not generally necessary to run all tests locally.
|
|
257
|
+
|
|
258
|
+
It is expected that contributors ensure that any change to fsspec does not
|
|
259
|
+
cause issues or regressions for either other fsspec-related packages such
|
|
260
|
+
as gcsfs and s3fs, nor for downstream users of fsspec. The "downstream" CI
|
|
261
|
+
run and corresponding environment file run a set of tests from the dask
|
|
262
|
+
test suite, and very minimal tests against pandas and zarr from the
|
|
263
|
+
test_downstream.py module in this repo.
|
|
264
|
+
|
|
265
|
+
### Code Formatting
|
|
266
|
+
|
|
267
|
+
fsspec uses [Black](https://black.readthedocs.io/en/stable) to ensure
|
|
268
|
+
a consistent code format throughout the project.
|
|
269
|
+
Run ``black fsspec`` from the root of the filesystem_spec repository to
|
|
270
|
+
auto-format your code. Additionally, many editors have plugins that will apply
|
|
271
|
+
``black`` as you edit files. ``black`` is included in the ``tox`` environments.
|
|
272
|
+
|
|
273
|
+
Optionally, you may wish to setup [pre-commit hooks](https://pre-commit.com) to
|
|
274
|
+
automatically run ``black`` when you make a git commit.
|
|
275
|
+
Run ``pre-commit install --install-hooks`` from the root of the
|
|
276
|
+
filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run
|
|
277
|
+
before you commit, reformatting any changed files. You can format without
|
|
278
|
+
committing via ``pre-commit run`` or skip these checks with ``git commit
|
|
279
|
+
--no-verify``.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
fsspec/__init__.py,sha256=l9MJaNNV2d4wKpCtMvXDr55n92DkdrAayGy3F9ICjzk,1998
|
|
2
|
+
fsspec/_version.py,sha256=-UnK4Q32q89vhPrzKnhQueNoH7tvQyAGqnmmbqXpb74,417
|
|
3
|
+
fsspec/archive.py,sha256=S__DzfZj-urAN3tp2W6jJ6YDiXG1fAl7FjvWUN73qIE,2386
|
|
4
|
+
fsspec/asyn.py,sha256=AOd2SXH2YPCaQL5jA6IegYevdMFkAnGD7Seh9DC2gSE,36404
|
|
5
|
+
fsspec/caching.py,sha256=x6IEdxtR3cMDjy40sNHyawR2SLtNSahVuP5i_TImdso,31600
|
|
6
|
+
fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210
|
|
7
|
+
fsspec/compression.py,sha256=jCSUMJu-zSNyrusnHT0wKXgOd1tTJR6vM126i5SR5Zc,4865
|
|
8
|
+
fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
|
|
9
|
+
fsspec/conftest.py,sha256=fVfx-NLrH_OZS1TIpYNoPzM7efEcMoL62reHOdYeFCA,1245
|
|
10
|
+
fsspec/core.py,sha256=Iln37fNZqjjk5vaDGU_0WWuwOxN1iVsQ6sDmCmuEvrs,23681
|
|
11
|
+
fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717
|
|
12
|
+
fsspec/exceptions.py,sha256=pauSLDMxzTJMOjvX1WEUK0cMyFkrFxpWJsyFywav7A8,331
|
|
13
|
+
fsspec/fuse.py,sha256=66amOa6wdIbS0DMhhfAPUoOB37HPorfXD1izV0prmTY,10145
|
|
14
|
+
fsspec/generic.py,sha256=lg7kRrxRG0qTJH349TVhBdRl-7nRH6xYe7LVOIqfcfA,13589
|
|
15
|
+
fsspec/gui.py,sha256=k46F11VGBLlrliPj3XbxHKlVGByWoX67Ofmu9ijaPBQ,13929
|
|
16
|
+
fsspec/json.py,sha256=5bOjXnB5nI4fwPoOXuCoKXKT7XQzYOHalLbG0ZS2bvo,2633
|
|
17
|
+
fsspec/mapping.py,sha256=hSsiRo-dgAOj6oHf67bF3i11U4xREglXToHGUX4GhRY,8261
|
|
18
|
+
fsspec/parquet.py,sha256=ONG29Enesp0ToCH2bQ7zkpimnVIsZ2S4xCLj35-fY78,19455
|
|
19
|
+
fsspec/registry.py,sha256=HVC-4HWDZnA6rycJwAu8F8ZXzON_85MTQVIyS6LOHxo,11320
|
|
20
|
+
fsspec/spec.py,sha256=dckmPmqZpLCiplUOPr_K5zeeDsg2nUxKrNLDCVb0Ek4,68058
|
|
21
|
+
fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398
|
|
22
|
+
fsspec/utils.py,sha256=8czEIoX4GpcC42WLGoy3t_EMeZjJE8e5rTpOT_nEPo0,22987
|
|
23
|
+
fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
fsspec/implementations/arrow.py,sha256=Y4F_IwWXuJI1mRO_c0_PI5o-Wp58RLmoiH_s-x88w4M,8631
|
|
25
|
+
fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421
|
|
26
|
+
fsspec/implementations/cache_metadata.py,sha256=pcOJYcBQY5OaC7Yhw0F3wjg08QLYApGmoISCrbs59ks,8511
|
|
27
|
+
fsspec/implementations/cached.py,sha256=AhgGa-XRu8rxxT6rusN3yEmayebP_F7hc6cYQlcyHo4,32720
|
|
28
|
+
fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
|
|
29
|
+
fsspec/implementations/data.py,sha256=LDLczxRh8h7x39Zjrd-GgzdQHr78yYxDlrv2C9Uxb5E,1658
|
|
30
|
+
fsspec/implementations/dbfs.py,sha256=cix9OYUveuSOx5UO5uRUwNUkYqjzyY0fkKnca1kTgZ0,15014
|
|
31
|
+
fsspec/implementations/dirfs.py,sha256=qrGXraUBsXUUGneLuhVddC6EfMaPBPA4iAb8M965yRA,11374
|
|
32
|
+
fsspec/implementations/ftp.py,sha256=rp6cTog8xqjDPlKdSLKcsyP7K593_ByMabxGbNSEpTo,11655
|
|
33
|
+
fsspec/implementations/git.py,sha256=vKGI-Vd5q4H2RrvhebkPc9NwlfkZ980OUGhebeCw-M0,4034
|
|
34
|
+
fsspec/implementations/github.py,sha256=eAn1kJ7VeWR6gVoVRLBYclF_rQDXSJU-xzMXpvPQWqs,8002
|
|
35
|
+
fsspec/implementations/http.py,sha256=uRmhzPzmLV6yCLJtc9XvK-aiTX48k67ZakCmh4msCzE,29637
|
|
36
|
+
fsspec/implementations/jupyter.py,sha256=B2uj7OEm7yIk-vRSsO37_ND0t0EBvn4B-Su43ibN4Pg,3811
|
|
37
|
+
fsspec/implementations/libarchive.py,sha256=5_I2DiLXwQ1JC8x-K7jXu-tBwhO9dj7tFLnb0bTnVMQ,7102
|
|
38
|
+
fsspec/implementations/local.py,sha256=qc68w69-I7zqVO8njv_s-THVImwICOqxyt-_2EK1VLg,15042
|
|
39
|
+
fsspec/implementations/memory.py,sha256=-BpOVwaWyW2rDvxWIIcrZTNFAhvuG66VWeIM6vLwhkc,10134
|
|
40
|
+
fsspec/implementations/reference.py,sha256=CpkKnWswsJl6XVExuoSY87147NPK_GLarxhRq2ec3vA,44107
|
|
41
|
+
fsspec/implementations/sftp.py,sha256=fMY9XZcmpjszQ2tCqO_TPaJesaeD_Dv7ptYzgUPGoO0,5631
|
|
42
|
+
fsspec/implementations/smb.py,sha256=RcqCvVBPD3U0I0Rc31ns6HRhqKVDugjPQMDPVpvZSNg,11408
|
|
43
|
+
fsspec/implementations/tar.py,sha256=dam78Tp_CozybNqCY2JYgGBS3Uc9FuJUAT9oB0lolOs,4111
|
|
44
|
+
fsspec/implementations/webhdfs.py,sha256=Wm7zr0iX3SZx5LtWfJIo-5rkIaoEoWq_Ev87NWbUgug,16721
|
|
45
|
+
fsspec/implementations/zip.py,sha256=vc1fNz-yO8uWQ9bQUqBFYpTcgsfZQq9vDwwg4Aufs9Y,4417
|
|
46
|
+
fsspec/tests/abstract/__init__.py,sha256=i1wcFixV6QhOwdoB24c8oXjzobISNqiKVz9kl2DvAY8,10028
|
|
47
|
+
fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973
|
|
48
|
+
fsspec/tests/abstract/copy.py,sha256=gU5-d97U3RSde35Vp4RxPY4rWwL744HiSrJ8IBOp9-8,19967
|
|
49
|
+
fsspec/tests/abstract/get.py,sha256=vNR4HztvTR7Cj56AMo7_tx7TeYz1Jgr_2Wb8Lv-UiBY,20755
|
|
50
|
+
fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1982
|
|
51
|
+
fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201
|
|
52
|
+
fsspec-2024.6.0.dist-info/METADATA,sha256=EUU8Nzg29TeSOhaN-GOUCpPN1KAiPid6UIm1P0x27Eg,11936
|
|
53
|
+
fsspec-2024.6.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
|
|
54
|
+
fsspec-2024.6.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
|
|
55
|
+
fsspec-2024.6.0.dist-info/RECORD,,
|