fsspec 2024.3.1__py3-none-any.whl → 2024.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fsspec/spec.py CHANGED
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import io
4
+ import json
4
5
  import logging
5
6
  import os
6
7
  import threading
@@ -9,7 +10,7 @@ import weakref
9
10
  from errno import ESPIPE
10
11
  from glob import has_magic
11
12
  from hashlib import sha256
12
- from typing import ClassVar
13
+ from typing import Any, ClassVar, Dict, Tuple
13
14
 
14
15
  from .callbacks import DEFAULT_CALLBACK
15
16
  from .config import apply_config, conf
@@ -115,6 +116,10 @@ class AbstractFileSystem(metaclass=_Cached):
115
116
  #: Extra *class attributes* that should be considered when hashing.
116
117
  _extra_tokenize_attributes = ()
117
118
 
119
+ # Set by _Cached metaclass
120
+ storage_args: Tuple[Any, ...]
121
+ storage_options: Dict[str, Any]
122
+
118
123
  def __init__(self, *args, **storage_options):
119
124
  """Create and configure file-system instance
120
125
 
@@ -358,8 +363,10 @@ class AbstractFileSystem(metaclass=_Cached):
358
363
  but contains nothing), None if not in cache.
359
364
  """
360
365
  parent = self._parent(path)
361
- if path.rstrip("/") in self.dircache:
366
+ try:
362
367
  return self.dircache[path.rstrip("/")]
368
+ except KeyError:
369
+ pass
363
370
  try:
364
371
  files = [
365
372
  f
@@ -1175,7 +1182,10 @@ class AbstractFileSystem(metaclass=_Cached):
1175
1182
  if path1 == path2:
1176
1183
  logger.debug("%s mv: The paths are the same, so no files were moved.", self)
1177
1184
  else:
1178
- self.copy(path1, path2, recursive=recursive, maxdepth=maxdepth)
1185
+ # explicitly raise exception to prevent data corruption
1186
+ self.copy(
1187
+ path1, path2, recursive=recursive, maxdepth=maxdepth, onerror="raise"
1188
+ )
1179
1189
  self.rm(path1, recursive=recursive)
1180
1190
 
1181
1191
  def rm_file(self, path):
@@ -1376,61 +1386,98 @@ class AbstractFileSystem(metaclass=_Cached):
1376
1386
  length = size - offset
1377
1387
  return read_block(f, offset, length, delimiter)
1378
1388
 
1379
- def to_json(self):
1389
+ def to_json(self) -> str:
1380
1390
  """
1381
- JSON representation of this filesystem instance
1391
+ JSON representation of this filesystem instance.
1382
1392
 
1383
1393
  Returns
1384
1394
  -------
1385
- str: JSON structure with keys cls (the python location of this class),
1386
- protocol (text name of this class's protocol, first one in case of
1387
- multiple), args (positional args, usually empty), and all other
1388
- kwargs as their own keys.
1395
+ JSON string with keys ``cls`` (the python location of this class),
1396
+ protocol (text name of this class's protocol, first one in case of
1397
+ multiple), ``args`` (positional args, usually empty), and all other
1398
+ keyword arguments as their own keys.
1389
1399
  """
1390
- import json
1400
+ from .json import FilesystemJSONEncoder
1401
+
1402
+ return json.dumps(self, cls=FilesystemJSONEncoder)
1403
+
1404
+ @staticmethod
1405
+ def from_json(blob: str) -> AbstractFileSystem:
1406
+ """
1407
+ Recreate a filesystem instance from JSON representation.
1408
+
1409
+ See ``.to_json()`` for the expected structure of the input.
1391
1410
 
1411
+ Parameters
1412
+ ----------
1413
+ blob: str
1414
+
1415
+ Returns
1416
+ -------
1417
+ file system instance, not necessarily of this particular class.
1418
+
1419
+ Warnings
1420
+ --------
1421
+ This can import arbitrary modules (as determined by the ``cls`` key).
1422
+ Make sure you haven't installed any modules that may execute malicious code
1423
+ at import time.
1424
+ """
1425
+ from .json import FilesystemJSONDecoder
1426
+
1427
+ return json.loads(blob, cls=FilesystemJSONDecoder)
1428
+
1429
+ def to_dict(self) -> Dict[str, Any]:
1430
+ """
1431
+ JSON-serializable dictionary representation of this filesystem instance.
1432
+
1433
+ Returns
1434
+ -------
1435
+ Dictionary with keys ``cls`` (the python location of this class),
1436
+ protocol (text name of this class's protocol, first one in case of
1437
+ multiple), ``args`` (positional args, usually empty), and all other
1438
+ keyword arguments as their own keys.
1439
+ """
1392
1440
  cls = type(self)
1393
- cls = ".".join((cls.__module__, cls.__name__))
1394
- proto = (
1395
- self.protocol[0]
1396
- if isinstance(self.protocol, (tuple, list))
1397
- else self.protocol
1398
- )
1399
- return json.dumps(
1400
- dict(
1401
- cls=cls,
1402
- protocol=proto,
1403
- args=self.storage_args,
1404
- **self.storage_options,
1405
- )
1441
+ proto = self.protocol
1442
+
1443
+ return dict(
1444
+ cls=f"{cls.__module__}:{cls.__name__}",
1445
+ protocol=proto[0] if isinstance(proto, (tuple, list)) else proto,
1446
+ args=self.storage_args,
1447
+ **self.storage_options,
1406
1448
  )
1407
1449
 
1408
1450
  @staticmethod
1409
- def from_json(blob):
1451
+ def from_dict(dct: Dict[str, Any]) -> AbstractFileSystem:
1410
1452
  """
1411
- Recreate a filesystem instance from JSON representation
1453
+ Recreate a filesystem instance from dictionary representation.
1412
1454
 
1413
- See ``.to_json()`` for the expected structure of the input
1455
+ See ``.to_dict()`` for the expected structure of the input.
1414
1456
 
1415
1457
  Parameters
1416
1458
  ----------
1417
- blob: str
1459
+ dct: Dict[str, Any]
1418
1460
 
1419
1461
  Returns
1420
1462
  -------
1421
1463
  file system instance, not necessarily of this particular class.
1464
+
1465
+ Warnings
1466
+ --------
1467
+ This can import arbitrary modules (as determined by the ``cls`` key).
1468
+ Make sure you haven't installed any modules that may execute malicious code
1469
+ at import time.
1422
1470
  """
1423
- import json
1471
+ from .json import FilesystemJSONDecoder
1424
1472
 
1425
- from .registry import _import_class, get_filesystem_class
1473
+ cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct)
1474
+ if cls is None:
1475
+ raise ValueError("Not a serialized AbstractFileSystem")
1426
1476
 
1427
- dic = json.loads(blob)
1428
- protocol = dic.pop("protocol")
1429
- try:
1430
- cls = _import_class(dic.pop("cls"))
1431
- except (ImportError, ValueError, RuntimeError, KeyError):
1432
- cls = get_filesystem_class(protocol)
1433
- return cls(*dic.pop("args", ()), **dic)
1477
+ dct.pop("cls", None)
1478
+ dct.pop("protocol", None)
1479
+
1480
+ return cls(*dct.pop("args", ()), **dct)
1434
1481
 
1435
1482
  def _get_pyarrow_filesystem(self):
1436
1483
  """
@@ -1693,7 +1740,12 @@ class AbstractBufferedFile(io.IOBase):
1693
1740
  """Files are equal if they have the same checksum, only in read mode"""
1694
1741
  if self is other:
1695
1742
  return True
1696
- return self.mode == "rb" and other.mode == "rb" and hash(self) == hash(other)
1743
+ return (
1744
+ isinstance(other, type(self))
1745
+ and self.mode == "rb"
1746
+ and other.mode == "rb"
1747
+ and hash(self) == hash(other)
1748
+ )
1697
1749
 
1698
1750
  def commit(self):
1699
1751
  """Move from temp to final destination"""
@@ -1839,11 +1891,18 @@ class AbstractBufferedFile(io.IOBase):
1839
1891
  length = self.size - self.loc
1840
1892
  if self.closed:
1841
1893
  raise ValueError("I/O operation on closed file.")
1842
- logger.debug("%s read: %i - %i", self, self.loc, self.loc + length)
1843
1894
  if length == 0:
1844
1895
  # don't even bother calling fetch
1845
1896
  return b""
1846
1897
  out = self.cache._fetch(self.loc, self.loc + length)
1898
+
1899
+ logger.debug(
1900
+ "%s read: %i - %i %s",
1901
+ self,
1902
+ self.loc,
1903
+ self.loc + length,
1904
+ self.cache._log_stats(),
1905
+ )
1847
1906
  self.loc += len(out)
1848
1907
  return out
1849
1908
 
@@ -0,0 +1,57 @@
1
+ import os
2
+
3
+ import pytest
4
+
5
+ import fsspec
6
+
7
+
8
+ def test_move_raises_error_with_tmpdir(tmpdir):
9
+ # Create a file in the temporary directory
10
+ source = tmpdir.join("source_file.txt")
11
+ source.write("content")
12
+
13
+ # Define a destination that simulates a protected or invalid path
14
+ destination = tmpdir.join("non_existent_directory/destination_file.txt")
15
+
16
+ # Instantiate the filesystem (assuming the local file system interface)
17
+ fs = fsspec.filesystem("file")
18
+
19
+ # Use the actual file paths as string
20
+ with pytest.raises(FileNotFoundError):
21
+ fs.mv(str(source), str(destination))
22
+
23
+
24
+ @pytest.mark.parametrize("recursive", (True, False))
25
+ def test_move_raises_error_with_tmpdir_permission(recursive, tmpdir):
26
+ # Create a file in the temporary directory
27
+ source = tmpdir.join("source_file.txt")
28
+ source.write("content")
29
+
30
+ # Create a protected directory (non-writable)
31
+ protected_dir = tmpdir.mkdir("protected_directory")
32
+ protected_path = str(protected_dir)
33
+
34
+ # Set the directory to read-only
35
+ if os.name == "nt":
36
+ os.system(f'icacls "{protected_path}" /deny Everyone:(W)')
37
+ else:
38
+ os.chmod(protected_path, 0o555) # Sets the directory to read-only
39
+
40
+ # Define a destination inside the protected directory
41
+ destination = protected_dir.join("destination_file.txt")
42
+
43
+ # Instantiate the filesystem (assuming the local file system interface)
44
+ fs = fsspec.filesystem("file")
45
+
46
+ # Try to move the file to the read-only directory, expecting a permission error
47
+ with pytest.raises(PermissionError):
48
+ fs.mv(str(source), str(destination), recursive=recursive)
49
+
50
+ # Assert the file was not created in the destination
51
+ assert not os.path.exists(destination)
52
+
53
+ # Cleanup: Restore permissions so the directory can be cleaned up
54
+ if os.name == "nt":
55
+ os.system(f'icacls "{protected_path}" /remove:d Everyone')
56
+ else:
57
+ os.chmod(protected_path, 0o755) # Restore write permission for cleanup
fsspec/utils.py CHANGED
@@ -350,8 +350,6 @@ def stringify_path(filepath: str | os.PathLike[str] | pathlib.Path) -> str:
350
350
  return filepath
351
351
  elif hasattr(filepath, "__fspath__"):
352
352
  return filepath.__fspath__()
353
- elif isinstance(filepath, pathlib.Path):
354
- return str(filepath)
355
353
  elif hasattr(filepath, "path"):
356
354
  return filepath.path
357
355
  else:
@@ -437,7 +435,7 @@ def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
437
435
 
438
436
  def get_protocol(url: str) -> str:
439
437
  url = stringify_path(url)
440
- parts = re.split(r"(\:\:|\://)", url, 1)
438
+ parts = re.split(r"(\:\:|\://)", url, maxsplit=1)
441
439
  if len(parts) > 1:
442
440
  return parts[0]
443
441
  return "file"
@@ -0,0 +1,279 @@
1
+ Metadata-Version: 2.3
2
+ Name: fsspec
3
+ Version: 2024.6.0
4
+ Summary: File-system specification
5
+ Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
6
+ Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
7
+ Project-URL: Homepage, https://github.com/fsspec/filesystem_spec
8
+ Maintainer-email: Martin Durant <mdurant@anaconda.com>
9
+ License: BSD 3-Clause License
10
+
11
+ Copyright (c) 2018, Martin Durant
12
+ All rights reserved.
13
+
14
+ Redistribution and use in source and binary forms, with or without
15
+ modification, are permitted provided that the following conditions are met:
16
+
17
+ * Redistributions of source code must retain the above copyright notice, this
18
+ list of conditions and the following disclaimer.
19
+
20
+ * Redistributions in binary form must reproduce the above copyright notice,
21
+ this list of conditions and the following disclaimer in the documentation
22
+ and/or other materials provided with the distribution.
23
+
24
+ * Neither the name of the copyright holder nor the names of its
25
+ contributors may be used to endorse or promote products derived from
26
+ this software without specific prior written permission.
27
+
28
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
31
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
32
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
34
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
35
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38
+ License-File: LICENSE
39
+ Keywords: file
40
+ Classifier: Development Status :: 4 - Beta
41
+ Classifier: Intended Audience :: Developers
42
+ Classifier: License :: OSI Approved :: BSD License
43
+ Classifier: Operating System :: OS Independent
44
+ Classifier: Programming Language :: Python :: 3.8
45
+ Classifier: Programming Language :: Python :: 3.9
46
+ Classifier: Programming Language :: Python :: 3.10
47
+ Classifier: Programming Language :: Python :: 3.11
48
+ Classifier: Programming Language :: Python :: 3.12
49
+ Requires-Python: >=3.8
50
+ Provides-Extra: abfs
51
+ Requires-Dist: adlfs; extra == 'abfs'
52
+ Provides-Extra: adl
53
+ Requires-Dist: adlfs; extra == 'adl'
54
+ Provides-Extra: arrow
55
+ Requires-Dist: pyarrow>=1; extra == 'arrow'
56
+ Provides-Extra: dask
57
+ Requires-Dist: dask; extra == 'dask'
58
+ Requires-Dist: distributed; extra == 'dask'
59
+ Provides-Extra: dev
60
+ Requires-Dist: pre-commit; extra == 'dev'
61
+ Requires-Dist: ruff; extra == 'dev'
62
+ Provides-Extra: doc
63
+ Requires-Dist: numpydoc; extra == 'doc'
64
+ Requires-Dist: sphinx; extra == 'doc'
65
+ Requires-Dist: sphinx-design; extra == 'doc'
66
+ Requires-Dist: sphinx-rtd-theme; extra == 'doc'
67
+ Requires-Dist: yarl; extra == 'doc'
68
+ Provides-Extra: dropbox
69
+ Requires-Dist: dropbox; extra == 'dropbox'
70
+ Requires-Dist: dropboxdrivefs; extra == 'dropbox'
71
+ Requires-Dist: requests; extra == 'dropbox'
72
+ Provides-Extra: entrypoints
73
+ Provides-Extra: full
74
+ Requires-Dist: adlfs; extra == 'full'
75
+ Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'full'
76
+ Requires-Dist: dask; extra == 'full'
77
+ Requires-Dist: distributed; extra == 'full'
78
+ Requires-Dist: dropbox; extra == 'full'
79
+ Requires-Dist: dropboxdrivefs; extra == 'full'
80
+ Requires-Dist: fusepy; extra == 'full'
81
+ Requires-Dist: gcsfs; extra == 'full'
82
+ Requires-Dist: libarchive-c; extra == 'full'
83
+ Requires-Dist: ocifs; extra == 'full'
84
+ Requires-Dist: panel; extra == 'full'
85
+ Requires-Dist: paramiko; extra == 'full'
86
+ Requires-Dist: pyarrow>=1; extra == 'full'
87
+ Requires-Dist: pygit2; extra == 'full'
88
+ Requires-Dist: requests; extra == 'full'
89
+ Requires-Dist: s3fs; extra == 'full'
90
+ Requires-Dist: smbprotocol; extra == 'full'
91
+ Requires-Dist: tqdm; extra == 'full'
92
+ Provides-Extra: fuse
93
+ Requires-Dist: fusepy; extra == 'fuse'
94
+ Provides-Extra: gcs
95
+ Requires-Dist: gcsfs; extra == 'gcs'
96
+ Provides-Extra: git
97
+ Requires-Dist: pygit2; extra == 'git'
98
+ Provides-Extra: github
99
+ Requires-Dist: requests; extra == 'github'
100
+ Provides-Extra: gs
101
+ Requires-Dist: gcsfs; extra == 'gs'
102
+ Provides-Extra: gui
103
+ Requires-Dist: panel; extra == 'gui'
104
+ Provides-Extra: hdfs
105
+ Requires-Dist: pyarrow>=1; extra == 'hdfs'
106
+ Provides-Extra: http
107
+ Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'http'
108
+ Provides-Extra: libarchive
109
+ Requires-Dist: libarchive-c; extra == 'libarchive'
110
+ Provides-Extra: oci
111
+ Requires-Dist: ocifs; extra == 'oci'
112
+ Provides-Extra: s3
113
+ Requires-Dist: s3fs; extra == 's3'
114
+ Provides-Extra: sftp
115
+ Requires-Dist: paramiko; extra == 'sftp'
116
+ Provides-Extra: smb
117
+ Requires-Dist: smbprotocol; extra == 'smb'
118
+ Provides-Extra: ssh
119
+ Requires-Dist: paramiko; extra == 'ssh'
120
+ Provides-Extra: test
121
+ Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test'
122
+ Requires-Dist: numpy; extra == 'test'
123
+ Requires-Dist: pytest; extra == 'test'
124
+ Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test'
125
+ Requires-Dist: pytest-benchmark; extra == 'test'
126
+ Requires-Dist: pytest-cov; extra == 'test'
127
+ Requires-Dist: pytest-mock; extra == 'test'
128
+ Requires-Dist: pytest-recording; extra == 'test'
129
+ Requires-Dist: pytest-rerunfailures; extra == 'test'
130
+ Requires-Dist: requests; extra == 'test'
131
+ Provides-Extra: test-downstream
132
+ Requires-Dist: aiobotocore<3.0.0,>=2.5.4; extra == 'test-downstream'
133
+ Requires-Dist: dask-expr; extra == 'test-downstream'
134
+ Requires-Dist: dask[dataframe,test]; extra == 'test-downstream'
135
+ Requires-Dist: moto[server]<5,>4; extra == 'test-downstream'
136
+ Requires-Dist: pytest-timeout; extra == 'test-downstream'
137
+ Requires-Dist: xarray; extra == 'test-downstream'
138
+ Provides-Extra: test-full
139
+ Requires-Dist: adlfs; extra == 'test-full'
140
+ Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test-full'
141
+ Requires-Dist: cloudpickle; extra == 'test-full'
142
+ Requires-Dist: dask; extra == 'test-full'
143
+ Requires-Dist: distributed; extra == 'test-full'
144
+ Requires-Dist: dropbox; extra == 'test-full'
145
+ Requires-Dist: dropboxdrivefs; extra == 'test-full'
146
+ Requires-Dist: fastparquet; extra == 'test-full'
147
+ Requires-Dist: fusepy; extra == 'test-full'
148
+ Requires-Dist: gcsfs; extra == 'test-full'
149
+ Requires-Dist: jinja2; extra == 'test-full'
150
+ Requires-Dist: kerchunk; extra == 'test-full'
151
+ Requires-Dist: libarchive-c; extra == 'test-full'
152
+ Requires-Dist: lz4; extra == 'test-full'
153
+ Requires-Dist: notebook; extra == 'test-full'
154
+ Requires-Dist: numpy; extra == 'test-full'
155
+ Requires-Dist: ocifs; extra == 'test-full'
156
+ Requires-Dist: pandas; extra == 'test-full'
157
+ Requires-Dist: panel; extra == 'test-full'
158
+ Requires-Dist: paramiko; extra == 'test-full'
159
+ Requires-Dist: pyarrow; extra == 'test-full'
160
+ Requires-Dist: pyarrow>=1; extra == 'test-full'
161
+ Requires-Dist: pyftpdlib; extra == 'test-full'
162
+ Requires-Dist: pygit2; extra == 'test-full'
163
+ Requires-Dist: pytest; extra == 'test-full'
164
+ Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test-full'
165
+ Requires-Dist: pytest-benchmark; extra == 'test-full'
166
+ Requires-Dist: pytest-cov; extra == 'test-full'
167
+ Requires-Dist: pytest-mock; extra == 'test-full'
168
+ Requires-Dist: pytest-recording; extra == 'test-full'
169
+ Requires-Dist: pytest-rerunfailures; extra == 'test-full'
170
+ Requires-Dist: python-snappy; extra == 'test-full'
171
+ Requires-Dist: requests; extra == 'test-full'
172
+ Requires-Dist: smbprotocol; extra == 'test-full'
173
+ Requires-Dist: tqdm; extra == 'test-full'
174
+ Requires-Dist: urllib3; extra == 'test-full'
175
+ Requires-Dist: zarr; extra == 'test-full'
176
+ Requires-Dist: zstandard; extra == 'test-full'
177
+ Provides-Extra: tqdm
178
+ Requires-Dist: tqdm; extra == 'tqdm'
179
+ Description-Content-Type: text/markdown
180
+
181
+ # filesystem_spec
182
+
183
+ [![PyPI version](https://badge.fury.io/py/fsspec.svg)](https://pypi.python.org/pypi/fsspec/)
184
+ [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
185
+ ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
186
+ [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
187
+ [![fsspec Downloads Last Month](https://assets.piptrends.com/get-last-month-downloads-badge/fsspec.svg 'fsspec Downloads Last Month by pip Trends')](https://piptrends.com/package/fsspec)
188
+
189
+ A specification for pythonic filesystems.
190
+
191
+ ## Install
192
+
193
+ ```bash
194
+ pip install fsspec
195
+ ```
196
+
197
+ would install the base fsspec. Various optionally supported features might require specification of custom
198
+ extra require, e.g. `pip install fsspec[ssh]` will install dependencies for `ssh` backends support.
199
+ Use `pip install fsspec[full]` for installation of all known extra dependencies.
200
+
201
+ Up-to-date package also provided through conda-forge distribution:
202
+
203
+ ```bash
204
+ conda install -c conda-forge fsspec
205
+ ```
206
+
207
+
208
+ ## Purpose
209
+
210
+ To produce a template or specification for a file-system interface, that specific implementations should follow,
211
+ so that applications making use of them can rely on a common behaviour and not have to worry about the specific
212
+ internal implementation decisions with any given backend. Many such implementations are included in this package,
213
+ or in sister projects such as `s3fs` and `gcsfs`.
214
+
215
+ In addition, if this is well-designed, then additional functionality, such as a key-value store or FUSE
216
+ mounting of the file-system implementation may be available for all implementations "for free".
217
+
218
+ ## Documentation
219
+
220
+ Please refer to [RTD](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
221
+
222
+ ## Develop
223
+
224
+ fsspec uses GitHub Actions for CI. Environment files can be found
225
+ in the "ci/" directory. Note that the main environment is called "py38",
226
+ but it is expected that the version of python installed be adjustable at
227
+ CI runtime. For local use, pick a version suitable for you.
228
+
229
+ ```bash
230
+ # For a new environment (mamba / conda).
231
+ mamba create -n fsspec -c conda-forge python=3.9 -y
232
+ conda activate fsspec
233
+
234
+ # Standard dev install with docs and tests.
235
+ pip install -e ".[dev,doc,test]"
236
+
237
+ # Full tests except for downstream
238
+ pip install s3fs
239
+ pip uninstall s3fs
240
+ pip install -e .[dev,doc,test_full]
241
+ pip install s3fs --no-deps
242
+ pytest -v
243
+
244
+ # Downstream tests.
245
+ sh install_s3fs.sh
246
+ # Windows powershell.
247
+ install_s3fs.sh
248
+ ```
249
+
250
+ ### Testing
251
+
252
+ Tests can be run in the dev environment, if activated, via ``pytest fsspec``.
253
+
254
+ The full fsspec suite requires a system-level docker, docker-compose, and fuse
255
+ installation. If only making changes to one backend implementation, it is
256
+ not generally necessary to run all tests locally.
257
+
258
+ It is expected that contributors ensure that any change to fsspec does not
259
+ cause issues or regressions for either other fsspec-related packages such
260
+ as gcsfs and s3fs, nor for downstream users of fsspec. The "downstream" CI
261
+ run and corresponding environment file run a set of tests from the dask
262
+ test suite, and very minimal tests against pandas and zarr from the
263
+ test_downstream.py module in this repo.
264
+
265
+ ### Code Formatting
266
+
267
+ fsspec uses [Black](https://black.readthedocs.io/en/stable) to ensure
268
+ a consistent code format throughout the project.
269
+ Run ``black fsspec`` from the root of the filesystem_spec repository to
270
+ auto-format your code. Additionally, many editors have plugins that will apply
271
+ ``black`` as you edit files. ``black`` is included in the ``tox`` environments.
272
+
273
+ Optionally, you may wish to setup [pre-commit hooks](https://pre-commit.com) to
274
+ automatically run ``black`` when you make a git commit.
275
+ Run ``pre-commit install --install-hooks`` from the root of the
276
+ filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run
277
+ before you commit, reformatting any changed files. You can format without
278
+ committing via ``pre-commit run`` or skip these checks with ``git commit
279
+ --no-verify``.
@@ -0,0 +1,55 @@
1
+ fsspec/__init__.py,sha256=l9MJaNNV2d4wKpCtMvXDr55n92DkdrAayGy3F9ICjzk,1998
2
+ fsspec/_version.py,sha256=-UnK4Q32q89vhPrzKnhQueNoH7tvQyAGqnmmbqXpb74,417
3
+ fsspec/archive.py,sha256=S__DzfZj-urAN3tp2W6jJ6YDiXG1fAl7FjvWUN73qIE,2386
4
+ fsspec/asyn.py,sha256=AOd2SXH2YPCaQL5jA6IegYevdMFkAnGD7Seh9DC2gSE,36404
5
+ fsspec/caching.py,sha256=x6IEdxtR3cMDjy40sNHyawR2SLtNSahVuP5i_TImdso,31600
6
+ fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210
7
+ fsspec/compression.py,sha256=jCSUMJu-zSNyrusnHT0wKXgOd1tTJR6vM126i5SR5Zc,4865
8
+ fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
9
+ fsspec/conftest.py,sha256=fVfx-NLrH_OZS1TIpYNoPzM7efEcMoL62reHOdYeFCA,1245
10
+ fsspec/core.py,sha256=Iln37fNZqjjk5vaDGU_0WWuwOxN1iVsQ6sDmCmuEvrs,23681
11
+ fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717
12
+ fsspec/exceptions.py,sha256=pauSLDMxzTJMOjvX1WEUK0cMyFkrFxpWJsyFywav7A8,331
13
+ fsspec/fuse.py,sha256=66amOa6wdIbS0DMhhfAPUoOB37HPorfXD1izV0prmTY,10145
14
+ fsspec/generic.py,sha256=lg7kRrxRG0qTJH349TVhBdRl-7nRH6xYe7LVOIqfcfA,13589
15
+ fsspec/gui.py,sha256=k46F11VGBLlrliPj3XbxHKlVGByWoX67Ofmu9ijaPBQ,13929
16
+ fsspec/json.py,sha256=5bOjXnB5nI4fwPoOXuCoKXKT7XQzYOHalLbG0ZS2bvo,2633
17
+ fsspec/mapping.py,sha256=hSsiRo-dgAOj6oHf67bF3i11U4xREglXToHGUX4GhRY,8261
18
+ fsspec/parquet.py,sha256=ONG29Enesp0ToCH2bQ7zkpimnVIsZ2S4xCLj35-fY78,19455
19
+ fsspec/registry.py,sha256=HVC-4HWDZnA6rycJwAu8F8ZXzON_85MTQVIyS6LOHxo,11320
20
+ fsspec/spec.py,sha256=dckmPmqZpLCiplUOPr_K5zeeDsg2nUxKrNLDCVb0Ek4,68058
21
+ fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398
22
+ fsspec/utils.py,sha256=8czEIoX4GpcC42WLGoy3t_EMeZjJE8e5rTpOT_nEPo0,22987
23
+ fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
24
+ fsspec/implementations/arrow.py,sha256=Y4F_IwWXuJI1mRO_c0_PI5o-Wp58RLmoiH_s-x88w4M,8631
25
+ fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421
26
+ fsspec/implementations/cache_metadata.py,sha256=pcOJYcBQY5OaC7Yhw0F3wjg08QLYApGmoISCrbs59ks,8511
27
+ fsspec/implementations/cached.py,sha256=AhgGa-XRu8rxxT6rusN3yEmayebP_F7hc6cYQlcyHo4,32720
28
+ fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
29
+ fsspec/implementations/data.py,sha256=LDLczxRh8h7x39Zjrd-GgzdQHr78yYxDlrv2C9Uxb5E,1658
30
+ fsspec/implementations/dbfs.py,sha256=cix9OYUveuSOx5UO5uRUwNUkYqjzyY0fkKnca1kTgZ0,15014
31
+ fsspec/implementations/dirfs.py,sha256=qrGXraUBsXUUGneLuhVddC6EfMaPBPA4iAb8M965yRA,11374
32
+ fsspec/implementations/ftp.py,sha256=rp6cTog8xqjDPlKdSLKcsyP7K593_ByMabxGbNSEpTo,11655
33
+ fsspec/implementations/git.py,sha256=vKGI-Vd5q4H2RrvhebkPc9NwlfkZ980OUGhebeCw-M0,4034
34
+ fsspec/implementations/github.py,sha256=eAn1kJ7VeWR6gVoVRLBYclF_rQDXSJU-xzMXpvPQWqs,8002
35
+ fsspec/implementations/http.py,sha256=uRmhzPzmLV6yCLJtc9XvK-aiTX48k67ZakCmh4msCzE,29637
36
+ fsspec/implementations/jupyter.py,sha256=B2uj7OEm7yIk-vRSsO37_ND0t0EBvn4B-Su43ibN4Pg,3811
37
+ fsspec/implementations/libarchive.py,sha256=5_I2DiLXwQ1JC8x-K7jXu-tBwhO9dj7tFLnb0bTnVMQ,7102
38
+ fsspec/implementations/local.py,sha256=qc68w69-I7zqVO8njv_s-THVImwICOqxyt-_2EK1VLg,15042
39
+ fsspec/implementations/memory.py,sha256=-BpOVwaWyW2rDvxWIIcrZTNFAhvuG66VWeIM6vLwhkc,10134
40
+ fsspec/implementations/reference.py,sha256=CpkKnWswsJl6XVExuoSY87147NPK_GLarxhRq2ec3vA,44107
41
+ fsspec/implementations/sftp.py,sha256=fMY9XZcmpjszQ2tCqO_TPaJesaeD_Dv7ptYzgUPGoO0,5631
42
+ fsspec/implementations/smb.py,sha256=RcqCvVBPD3U0I0Rc31ns6HRhqKVDugjPQMDPVpvZSNg,11408
43
+ fsspec/implementations/tar.py,sha256=dam78Tp_CozybNqCY2JYgGBS3Uc9FuJUAT9oB0lolOs,4111
44
+ fsspec/implementations/webhdfs.py,sha256=Wm7zr0iX3SZx5LtWfJIo-5rkIaoEoWq_Ev87NWbUgug,16721
45
+ fsspec/implementations/zip.py,sha256=vc1fNz-yO8uWQ9bQUqBFYpTcgsfZQq9vDwwg4Aufs9Y,4417
46
+ fsspec/tests/abstract/__init__.py,sha256=i1wcFixV6QhOwdoB24c8oXjzobISNqiKVz9kl2DvAY8,10028
47
+ fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973
48
+ fsspec/tests/abstract/copy.py,sha256=gU5-d97U3RSde35Vp4RxPY4rWwL744HiSrJ8IBOp9-8,19967
49
+ fsspec/tests/abstract/get.py,sha256=vNR4HztvTR7Cj56AMo7_tx7TeYz1Jgr_2Wb8Lv-UiBY,20755
50
+ fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1982
51
+ fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201
52
+ fsspec-2024.6.0.dist-info/METADATA,sha256=EUU8Nzg29TeSOhaN-GOUCpPN1KAiPid6UIm1P0x27Eg,11936
53
+ fsspec-2024.6.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
54
+ fsspec-2024.6.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
55
+ fsspec-2024.6.0.dist-info/RECORD,,
@@ -1,5 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.43.0)
2
+ Generator: hatchling 1.24.2
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
-