fsspec 2025.10.0__tar.gz → 2026.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fsspec-2025.10.0 → fsspec-2026.1.0}/.github/workflows/main.yaml +6 -6
- {fsspec-2025.10.0 → fsspec-2026.1.0}/.github/workflows/pypipublish.yaml +2 -2
- {fsspec-2025.10.0 → fsspec-2026.1.0}/.pre-commit-config.yaml +1 -2
- {fsspec-2025.10.0 → fsspec-2026.1.0}/PKG-INFO +7 -7
- {fsspec-2025.10.0 → fsspec-2026.1.0}/README.md +1 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-friends.yml +1 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/environment.yml +1 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/api.rst +8 -2
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/changelog.rst +40 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/features.rst +1 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/_version.py +2 -2
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/asyn.py +7 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/caching.py +52 -45
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/compression.py +7 -18
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/core.py +20 -3
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/arrow.py +6 -3
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/asyn_wrapper.py +3 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/cache_metadata.py +1 -3
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/cached.py +2 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/data.py +1 -2
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/dirfs.py +2 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/ftp.py +54 -4
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/http.py +7 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/http_sync.py +7 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/libarchive.py +1 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/local.py +4 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/reference.py +1 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/json.py +7 -12
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/parquet.py +100 -61
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/registry.py +4 -1
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/utils.py +3 -10
- {fsspec-2025.10.0 → fsspec-2026.1.0}/pyproject.toml +7 -7
- {fsspec-2025.10.0 → fsspec-2026.1.0}/.codespellrc +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/.coveragerc +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/.gitattributes +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/.gitignore +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/LICENSE +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-downstream.yml +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-linux.yml +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-win.yml +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/Makefile +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/README.md +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/make.bat +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/_static/custom.css +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/async.rst +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/code-of-conduct.rst +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/conf.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/copying.rst +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/developer.rst +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/img/gui.png +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/index.rst +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/intro.rst +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/usage.rst +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/__init__.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/archive.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/callbacks.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/config.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/conftest.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/dircache.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/exceptions.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/fuse.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/generic.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/gui.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/__init__.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/cache_mapper.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/chained.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/dask.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/dbfs.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/gist.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/git.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/github.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/jupyter.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/memory.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/sftp.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/smb.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/tar.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/webhdfs.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/zip.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/mapping.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/spec.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/__init__.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/common.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/copy.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/get.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/mv.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/open.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/pipe.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/put.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/transaction.py +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/install_s3fs.sh +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/readthedocs.yml +0 -0
- {fsspec-2025.10.0 → fsspec-2026.1.0}/setup.cfg +0 -0
|
@@ -14,18 +14,18 @@ jobs:
|
|
|
14
14
|
fail-fast: false
|
|
15
15
|
matrix:
|
|
16
16
|
PY:
|
|
17
|
-
- "3.9"
|
|
18
17
|
- "3.10"
|
|
19
18
|
- "3.11"
|
|
20
19
|
- "3.12"
|
|
21
20
|
- "3.13"
|
|
21
|
+
- "3.14"
|
|
22
22
|
|
|
23
23
|
env:
|
|
24
24
|
CIRUN: true
|
|
25
25
|
|
|
26
26
|
steps:
|
|
27
27
|
- name: Checkout
|
|
28
|
-
uses: actions/checkout@
|
|
28
|
+
uses: actions/checkout@v5
|
|
29
29
|
with:
|
|
30
30
|
fetch-depth: 0
|
|
31
31
|
|
|
@@ -50,7 +50,7 @@ jobs:
|
|
|
50
50
|
|
|
51
51
|
steps:
|
|
52
52
|
- name: Checkout
|
|
53
|
-
uses: actions/checkout@
|
|
53
|
+
uses: actions/checkout@v5
|
|
54
54
|
with:
|
|
55
55
|
fetch-depth: 0
|
|
56
56
|
|
|
@@ -81,7 +81,7 @@ jobs:
|
|
|
81
81
|
|
|
82
82
|
steps:
|
|
83
83
|
- name: Checkout
|
|
84
|
-
uses: actions/checkout@
|
|
84
|
+
uses: actions/checkout@v5
|
|
85
85
|
with:
|
|
86
86
|
fetch-depth: 0
|
|
87
87
|
|
|
@@ -124,7 +124,7 @@ jobs:
|
|
|
124
124
|
|
|
125
125
|
steps:
|
|
126
126
|
- name: Checkout
|
|
127
|
-
uses: actions/checkout@
|
|
127
|
+
uses: actions/checkout@v5
|
|
128
128
|
|
|
129
129
|
- name: Setup conda
|
|
130
130
|
uses: conda-incubator/setup-miniconda@v3
|
|
@@ -145,5 +145,5 @@ jobs:
|
|
|
145
145
|
shell: bash -l {0}
|
|
146
146
|
run: |
|
|
147
147
|
cd ${{ matrix.FRIEND }}
|
|
148
|
-
pytest -v
|
|
148
|
+
pytest -v -W ignore::pytest.PytestRemovedIn9Warning
|
|
149
149
|
cd ..
|
|
@@ -8,9 +8,9 @@ jobs:
|
|
|
8
8
|
deploy:
|
|
9
9
|
runs-on: ubuntu-latest
|
|
10
10
|
steps:
|
|
11
|
-
- uses: actions/checkout@
|
|
11
|
+
- uses: actions/checkout@v5
|
|
12
12
|
- name: Set up Python
|
|
13
|
-
uses: actions/setup-python@
|
|
13
|
+
uses: actions/setup-python@v6
|
|
14
14
|
with:
|
|
15
15
|
python-version: "3.x"
|
|
16
16
|
- name: Install dependencies
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fsspec
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2026.1.0
|
|
4
4
|
Summary: File-system specification
|
|
5
5
|
Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
|
|
6
6
|
Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
|
|
@@ -12,12 +12,12 @@ Keywords: file
|
|
|
12
12
|
Classifier: Development Status :: 4 - Beta
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.10
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
-
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
21
|
Provides-Extra: abfs
|
|
22
22
|
Requires-Dist: adlfs; extra == 'abfs'
|
|
23
23
|
Provides-Extra: adl
|
|
@@ -49,7 +49,7 @@ Requires-Dist: distributed; extra == 'full'
|
|
|
49
49
|
Requires-Dist: dropbox; extra == 'full'
|
|
50
50
|
Requires-Dist: dropboxdrivefs; extra == 'full'
|
|
51
51
|
Requires-Dist: fusepy; extra == 'full'
|
|
52
|
-
Requires-Dist: gcsfs; extra == 'full'
|
|
52
|
+
Requires-Dist: gcsfs>2024.2.0; extra == 'full'
|
|
53
53
|
Requires-Dist: libarchive-c; extra == 'full'
|
|
54
54
|
Requires-Dist: ocifs; extra == 'full'
|
|
55
55
|
Requires-Dist: panel; extra == 'full'
|
|
@@ -57,7 +57,7 @@ Requires-Dist: paramiko; extra == 'full'
|
|
|
57
57
|
Requires-Dist: pyarrow>=1; extra == 'full'
|
|
58
58
|
Requires-Dist: pygit2; extra == 'full'
|
|
59
59
|
Requires-Dist: requests; extra == 'full'
|
|
60
|
-
Requires-Dist: s3fs; extra == 'full'
|
|
60
|
+
Requires-Dist: s3fs>2024.2.0; extra == 'full'
|
|
61
61
|
Requires-Dist: smbprotocol; extra == 'full'
|
|
62
62
|
Requires-Dist: tqdm; extra == 'full'
|
|
63
63
|
Provides-Extra: fuse
|
|
@@ -108,6 +108,7 @@ Requires-Dist: xarray; extra == 'test-downstream'
|
|
|
108
108
|
Provides-Extra: test-full
|
|
109
109
|
Requires-Dist: adlfs; extra == 'test-full'
|
|
110
110
|
Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test-full'
|
|
111
|
+
Requires-Dist: backports-zstd; (python_version < '3.14') and extra == 'test-full'
|
|
111
112
|
Requires-Dist: cloudpickle; extra == 'test-full'
|
|
112
113
|
Requires-Dist: dask; extra == 'test-full'
|
|
113
114
|
Requires-Dist: distributed; extra == 'test-full'
|
|
@@ -143,7 +144,6 @@ Requires-Dist: smbprotocol; extra == 'test-full'
|
|
|
143
144
|
Requires-Dist: tqdm; extra == 'test-full'
|
|
144
145
|
Requires-Dist: urllib3; extra == 'test-full'
|
|
145
146
|
Requires-Dist: zarr; extra == 'test-full'
|
|
146
|
-
Requires-Dist: zstandard; (python_version < '3.14') and extra == 'test-full'
|
|
147
147
|
Provides-Extra: tqdm
|
|
148
148
|
Requires-Dist: tqdm; extra == 'tqdm'
|
|
149
149
|
Description-Content-Type: text/markdown
|
|
@@ -197,7 +197,7 @@ CI runtime. For local use, pick a version suitable for you.
|
|
|
197
197
|
|
|
198
198
|
```bash
|
|
199
199
|
# For a new environment (mamba / conda).
|
|
200
|
-
mamba create -n fsspec -c conda-forge python=3.
|
|
200
|
+
mamba create -n fsspec -c conda-forge python=3.10 -y
|
|
201
201
|
conda activate fsspec
|
|
202
202
|
|
|
203
203
|
# Standard dev install with docs and tests.
|
|
@@ -47,7 +47,7 @@ CI runtime. For local use, pick a version suitable for you.
|
|
|
47
47
|
|
|
48
48
|
```bash
|
|
49
49
|
# For a new environment (mamba / conda).
|
|
50
|
-
mamba create -n fsspec -c conda-forge python=3.
|
|
50
|
+
mamba create -n fsspec -c conda-forge python=3.10 -y
|
|
51
51
|
conda activate fsspec
|
|
52
52
|
|
|
53
53
|
# Standard dev install with docs and tests.
|
|
@@ -235,23 +235,26 @@ documentation carefully before using any particular package.
|
|
|
235
235
|
- `irods`_ for access to iRODS servers, with protocol "irods://"
|
|
236
236
|
- `lakefs`_ for lakeFS data lakes, with protocol "lakefs://"
|
|
237
237
|
- `morefs`_ for `OverlayFileSystem`, `DictFileSystem`, and others
|
|
238
|
+
- `msgraphfs`_ for Microsoft storage (ie Sharepoint) using the drive API through Microsoft Graph, with protocol "msgd://"
|
|
238
239
|
- `obstore`_: zero-dependency access to Amazon S3, Google Cloud Storage, and Azure Blob Storage using the underlying Rust `object_store`_ library, with protocols "s3://", "gs://", and "abfs://".
|
|
239
240
|
- `ocifs`_ for access to Oracle Cloud Object Storage, with protocol "oci://"
|
|
240
241
|
- `ocilake`_ for OCI Data Lake storage
|
|
241
242
|
- `ossfs`_ for Alibaba Cloud (Aliyun) Object Storage System (OSS)
|
|
242
243
|
- `p9fs`_ for 9P (Plan 9 Filesystem Protocol) servers
|
|
243
244
|
- `PyAthena`_ for S3 access to Amazon Athena, with protocol "s3://" or "s3a://"
|
|
245
|
+
- `fsspec-pydantic`_: Pydantic models to represent file and directory types
|
|
244
246
|
- `PyDrive2`_ for Google Drive access
|
|
247
|
+
- `fsspec-python`_: A chained filesystem that connects to Python's import system, to allow for importing from an fsspec backend
|
|
245
248
|
- `fsspec-proxy`_ for "pyscript:" URLs via a proxy server
|
|
246
249
|
- `s3fs`_ for Amazon S3 and other compatible stores, with protocol "s3://"
|
|
247
250
|
- `sshfs`_ for access to SSH servers, with protocol "ssh://" or "sftp://"
|
|
248
251
|
- `swiftspec`_ for OpenStack SWIFT, with protocol "swift://"
|
|
249
252
|
- `tosfs`_ for ByteDance volcano engine Tinder Object Storage (TOS)
|
|
253
|
+
- `fsspec-union`_: A chained filesystem that unions multiple fsspec backends as a read-through cache
|
|
250
254
|
- `wandbfs`_ to access Wandb run data (experimental)
|
|
251
255
|
- `wandbfsspec`_ to access Weights & Biases (experimental)
|
|
252
256
|
- `webdav4`_ for WebDAV, with protocol "webdav://" or "dav://"
|
|
253
257
|
- `xrootd`_ for xrootd, with protocol "root://"
|
|
254
|
-
- `msgraphfs`_ for Microsoft storage (ie Sharepoint) using the drive API through Microsoft Graph, with protocol "msgd://"
|
|
255
258
|
|
|
256
259
|
.. _abfs: https://github.com/dask/adlfs
|
|
257
260
|
.. _adl: https://github.com/dask/adlfs
|
|
@@ -272,6 +275,7 @@ documentation carefully before using any particular package.
|
|
|
272
275
|
.. _irods: https://github.com/xwcl/irods_fsspec
|
|
273
276
|
.. _lakefs: https://github.com/aai-institute/lakefs-spec
|
|
274
277
|
.. _morefs: https://github.com/iterative/morefs
|
|
278
|
+
.. _msgraphfs: https://github.com/acsone/msgraphfs
|
|
275
279
|
.. _object_store: https://docs.rs/object_store/latest/object_store/
|
|
276
280
|
.. _obstore: https://developmentseed.org/obstore/latest/
|
|
277
281
|
.. _ocifs: https://ocifs.readthedocs.io/en/latest/
|
|
@@ -279,16 +283,18 @@ documentation carefully before using any particular package.
|
|
|
279
283
|
.. _ossfs: https://github.com/fsspec/ossfs
|
|
280
284
|
.. _p9fs: https://github.com/pbchekin/p9fs-py
|
|
281
285
|
.. _PyAthena: https://github.com/laughingman7743/PyAthena
|
|
286
|
+
.. _fsspec-pydantic: https://github.com/1kbgz/fsspec-pydantic
|
|
282
287
|
.. _PyDrive2: https://github.com/iterative/PyDrive2
|
|
288
|
+
.. _fsspec-python: https://github.com/1kbgz/fsspec-python
|
|
283
289
|
.. _s3fs: https://s3fs.readthedocs.io/en/latest/
|
|
284
290
|
.. _sshfs: https://github.com/fsspec/sshfs
|
|
285
291
|
.. _swiftspec: https://github.com/fsspec/swiftspec
|
|
286
292
|
.. _tosfs: https://tosfs.readthedocs.io/en/latest/
|
|
293
|
+
.. _fsspec-union: https://github.com/1kbgz/fsspec-union
|
|
287
294
|
.. _wandbfs: https://github.com/jkulhanek/wandbfs
|
|
288
295
|
.. _wandbfsspec: https://github.com/alvarobartt/wandbfsspec
|
|
289
296
|
.. _webdav4: https://github.com/skshetry/webdav4
|
|
290
297
|
.. _xrootd: https://github.com/CoffeaTeam/fsspec-xrootd
|
|
291
|
-
.. _msgraphfs: https://github.com/acsone/msgraphfs
|
|
292
298
|
|
|
293
299
|
.. _readbuffering:
|
|
294
300
|
|
|
@@ -1,6 +1,46 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
+
2026.1.0
|
|
5
|
+
--------
|
|
6
|
+
|
|
7
|
+
Put limits on install extras for s3fs/gcsfs to prevent installing
|
|
8
|
+
ancient versions via pip.
|
|
9
|
+
|
|
10
|
+
Enhancements
|
|
11
|
+
|
|
12
|
+
- configure TLS protocol version for FTP (#1958)
|
|
13
|
+
|
|
14
|
+
Fixes
|
|
15
|
+
|
|
16
|
+
- Respect auto_mkdir in mv for local (#1957)
|
|
17
|
+
|
|
18
|
+
Other
|
|
19
|
+
|
|
20
|
+
- link to pydantic, code and union implementations ($1956)
|
|
21
|
+
- use backport.zstd or builtin (#1962)
|
|
22
|
+
- docs typo (#1963)
|
|
23
|
+
|
|
24
|
+
2025.12.0
|
|
25
|
+
---------
|
|
26
|
+
|
|
27
|
+
Enhancements
|
|
28
|
+
|
|
29
|
+
- fsspec.parquet to support filters and multiple files (#1945)
|
|
30
|
+
|
|
31
|
+
Fixes
|
|
32
|
+
|
|
33
|
+
- passing withdirs in aync _glob() (#1953)
|
|
34
|
+
- fix _rm_file/_rm redirection in async (#1951)
|
|
35
|
+
- allow arrowFile to be seekable (#1950)
|
|
36
|
+
- add size attribute to arrowFile (#1944)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
Other
|
|
40
|
+
|
|
41
|
+
- support py3.14 and drop 3.9 (#1946)
|
|
42
|
+
- avoid ruff warning (#1942)
|
|
43
|
+
|
|
4
44
|
2025.10.0
|
|
5
45
|
---------
|
|
6
46
|
|
|
@@ -161,7 +161,7 @@ be possible to reach all paths on the remote.
|
|
|
161
161
|
Instance Caching
|
|
162
162
|
----------------
|
|
163
163
|
|
|
164
|
-
|
|
164
|
+
If a file-system implementation class is marked as *cachable* (attribute ``.cachable``),
|
|
165
165
|
then its instances will
|
|
166
166
|
get stored in a class attribute, to enable quick look-up instead of needing to regenerate
|
|
167
167
|
potentially expensive connections and sessions. They key in the cache is a tokenisation of
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '
|
|
32
|
-
__version_tuple__ = version_tuple = (
|
|
31
|
+
__version__ = version = '2026.1.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (2026, 1, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -328,6 +328,11 @@ class AsyncFileSystem(AbstractFileSystem):
|
|
|
328
328
|
return self._loop
|
|
329
329
|
|
|
330
330
|
async def _rm_file(self, path, **kwargs):
|
|
331
|
+
if (
|
|
332
|
+
inspect.iscoroutinefunction(self._rm)
|
|
333
|
+
and type(self)._rm is not AsyncFileSystem._rm
|
|
334
|
+
):
|
|
335
|
+
return await self._rm(path, recursive=False, batch_size=1, **kwargs)
|
|
331
336
|
raise NotImplementedError
|
|
332
337
|
|
|
333
338
|
async def _rm(self, path, recursive=False, batch_size=None, **kwargs):
|
|
@@ -776,6 +781,7 @@ class AsyncFileSystem(AbstractFileSystem):
|
|
|
776
781
|
min_idx = min(idx_star, idx_qmark, idx_brace)
|
|
777
782
|
|
|
778
783
|
detail = kwargs.pop("detail", False)
|
|
784
|
+
withdirs = kwargs.pop("withdirs", True)
|
|
779
785
|
|
|
780
786
|
if not has_magic(path):
|
|
781
787
|
if await self._exists(path, **kwargs):
|
|
@@ -805,7 +811,7 @@ class AsyncFileSystem(AbstractFileSystem):
|
|
|
805
811
|
depth = None
|
|
806
812
|
|
|
807
813
|
allpaths = await self._find(
|
|
808
|
-
root, maxdepth=depth, withdirs=
|
|
814
|
+
root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs
|
|
809
815
|
)
|
|
810
816
|
|
|
811
817
|
pattern = glob_translate(path + ("/" if ends_with_sep else ""))
|
|
@@ -6,20 +6,12 @@ import logging
|
|
|
6
6
|
import math
|
|
7
7
|
import os
|
|
8
8
|
import threading
|
|
9
|
-
import warnings
|
|
10
9
|
from collections import OrderedDict
|
|
10
|
+
from collections.abc import Callable
|
|
11
11
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
12
12
|
from itertools import groupby
|
|
13
13
|
from operator import itemgetter
|
|
14
|
-
from typing import
|
|
15
|
-
TYPE_CHECKING,
|
|
16
|
-
Any,
|
|
17
|
-
Callable,
|
|
18
|
-
ClassVar,
|
|
19
|
-
Generic,
|
|
20
|
-
NamedTuple,
|
|
21
|
-
TypeVar,
|
|
22
|
-
)
|
|
14
|
+
from typing import TYPE_CHECKING, Any, ClassVar, Generic, NamedTuple, TypeVar
|
|
23
15
|
|
|
24
16
|
if TYPE_CHECKING:
|
|
25
17
|
import mmap
|
|
@@ -629,7 +621,7 @@ class KnownPartsOfAFile(BaseCache):
|
|
|
629
621
|
fetcher: Fetcher,
|
|
630
622
|
size: int,
|
|
631
623
|
data: dict[tuple[int, int], bytes] | None = None,
|
|
632
|
-
strict: bool =
|
|
624
|
+
strict: bool = False,
|
|
633
625
|
**_: Any,
|
|
634
626
|
):
|
|
635
627
|
super().__init__(blocksize, fetcher, size)
|
|
@@ -653,50 +645,65 @@ class KnownPartsOfAFile(BaseCache):
|
|
|
653
645
|
else:
|
|
654
646
|
self.data = {}
|
|
655
647
|
|
|
648
|
+
@property
|
|
649
|
+
def size(self):
|
|
650
|
+
return sum(_[1] - _[0] for _ in self.data)
|
|
651
|
+
|
|
652
|
+
@size.setter
|
|
653
|
+
def size(self, value):
|
|
654
|
+
pass
|
|
655
|
+
|
|
656
|
+
@property
|
|
657
|
+
def nblocks(self):
|
|
658
|
+
return len(self.data)
|
|
659
|
+
|
|
660
|
+
@nblocks.setter
|
|
661
|
+
def nblocks(self, value):
|
|
662
|
+
pass
|
|
663
|
+
|
|
656
664
|
def _fetch(self, start: int | None, stop: int | None) -> bytes:
|
|
657
665
|
if start is None:
|
|
658
666
|
start = 0
|
|
659
667
|
if stop is None:
|
|
660
668
|
stop = self.size
|
|
669
|
+
self.total_requested_bytes += stop - start
|
|
661
670
|
|
|
662
671
|
out = b""
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
672
|
+
started = False
|
|
673
|
+
loc_old = 0
|
|
674
|
+
for loc0, loc1 in sorted(self.data):
|
|
675
|
+
if (loc0 <= start < loc1) and (loc0 <= stop <= loc1):
|
|
676
|
+
# entirely within the block
|
|
677
|
+
off = start - loc0
|
|
678
|
+
self.hit_count += 1
|
|
679
|
+
return self.data[(loc0, loc1)][off : off + stop - start]
|
|
680
|
+
if stop <= loc0:
|
|
681
|
+
break
|
|
682
|
+
if started and loc0 > loc_old:
|
|
683
|
+
# a gap where we need data
|
|
684
|
+
self.miss_count += 1
|
|
685
|
+
if self.strict:
|
|
686
|
+
raise ValueError
|
|
687
|
+
out += b"\x00" * (loc0 - loc_old)
|
|
666
688
|
if loc0 <= start < loc1:
|
|
689
|
+
# found the start
|
|
690
|
+
self.hit_count += 1
|
|
667
691
|
off = start - loc0
|
|
668
|
-
out = data[off : off + stop - start]
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
# and we are being "strict" about reads
|
|
680
|
-
# beyond the buffer
|
|
681
|
-
start = loc1
|
|
682
|
-
break
|
|
683
|
-
|
|
684
|
-
# We only get here if there is a request outside the
|
|
685
|
-
# known parts of the file. In an ideal world, this
|
|
686
|
-
# should never happen
|
|
687
|
-
if self.fetcher is None:
|
|
688
|
-
# We cannot fetch the data, so raise an error
|
|
689
|
-
raise ValueError(f"Read is outside the known file parts: {(start, stop)}. ")
|
|
690
|
-
# We can fetch the data, but should warn the user
|
|
691
|
-
# that this may be slow
|
|
692
|
-
warnings.warn(
|
|
693
|
-
f"Read is outside the known file parts: {(start, stop)}. "
|
|
694
|
-
f"IO/caching performance may be poor!"
|
|
695
|
-
)
|
|
696
|
-
logger.debug(f"KnownPartsOfAFile cache fetching {start}-{stop}")
|
|
697
|
-
self.total_requested_bytes += stop - start
|
|
692
|
+
out = self.data[(loc0, loc1)][off : off + stop - start]
|
|
693
|
+
started = True
|
|
694
|
+
elif start < loc0 and stop > loc1:
|
|
695
|
+
# the whole block
|
|
696
|
+
self.hit_count += 1
|
|
697
|
+
out += self.data[(loc0, loc1)]
|
|
698
|
+
elif loc0 <= stop <= loc1:
|
|
699
|
+
# end block
|
|
700
|
+
self.hit_count += 1
|
|
701
|
+
return out + self.data[(loc0, loc1)][: stop - loc0]
|
|
702
|
+
loc_old = loc1
|
|
698
703
|
self.miss_count += 1
|
|
699
|
-
|
|
704
|
+
if started and not self.strict:
|
|
705
|
+
return out + b"\x00" * (stop - loc_old)
|
|
706
|
+
raise ValueError
|
|
700
707
|
|
|
701
708
|
|
|
702
709
|
class UpdatableLRU(Generic[P, T]):
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Helper functions for a standard streaming compression API"""
|
|
2
2
|
|
|
3
|
+
import sys
|
|
3
4
|
from zipfile import ZipFile
|
|
4
5
|
|
|
5
6
|
import fsspec.utils
|
|
@@ -155,26 +156,14 @@ except ImportError:
|
|
|
155
156
|
pass
|
|
156
157
|
|
|
157
158
|
try:
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
159
|
+
if sys.version_info >= (3, 14):
|
|
160
|
+
from compression import zstd
|
|
161
|
+
else:
|
|
162
|
+
from backports import zstd
|
|
162
163
|
|
|
164
|
+
register_compression("zstd", zstd.ZstdFile, "zst")
|
|
163
165
|
except ImportError:
|
|
164
|
-
|
|
165
|
-
import zstandard as zstd
|
|
166
|
-
|
|
167
|
-
def zstandard_file(infile, mode="rb"):
|
|
168
|
-
if "r" in mode:
|
|
169
|
-
cctx = zstd.ZstdDecompressor()
|
|
170
|
-
return cctx.stream_reader(infile)
|
|
171
|
-
else:
|
|
172
|
-
cctx = zstd.ZstdCompressor(level=10)
|
|
173
|
-
return cctx.stream_writer(infile)
|
|
174
|
-
|
|
175
|
-
register_compression("zstd", zstandard_file, "zst")
|
|
176
|
-
except ImportError:
|
|
177
|
-
pass
|
|
166
|
+
pass
|
|
178
167
|
|
|
179
168
|
|
|
180
169
|
def available_compressions():
|
|
@@ -18,7 +18,7 @@ from fsspec.caching import ( # noqa: F401
|
|
|
18
18
|
)
|
|
19
19
|
from fsspec.compression import compr
|
|
20
20
|
from fsspec.config import conf
|
|
21
|
-
from fsspec.registry import filesystem, get_filesystem_class
|
|
21
|
+
from fsspec.registry import available_protocols, filesystem, get_filesystem_class
|
|
22
22
|
from fsspec.utils import (
|
|
23
23
|
_unstrip_protocol,
|
|
24
24
|
build_name_function,
|
|
@@ -334,34 +334,51 @@ def _un_chain(path, kwargs):
|
|
|
334
334
|
|
|
335
335
|
if "::" in path:
|
|
336
336
|
x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word
|
|
337
|
+
known_protocols = set(available_protocols())
|
|
337
338
|
bits = []
|
|
339
|
+
|
|
340
|
+
# split on '::', then ensure each bit has a protocol
|
|
338
341
|
for p in path.split("::"):
|
|
339
|
-
if
|
|
342
|
+
if p in known_protocols:
|
|
343
|
+
bits.append(p + "://")
|
|
344
|
+
elif "://" in p or x.match(p):
|
|
340
345
|
bits.append(p)
|
|
341
346
|
else:
|
|
342
347
|
bits.append(p + "://")
|
|
343
348
|
else:
|
|
344
349
|
bits = [path]
|
|
350
|
+
|
|
345
351
|
# [[url, protocol, kwargs], ...]
|
|
346
352
|
out = []
|
|
347
353
|
previous_bit = None
|
|
348
354
|
kwargs = kwargs.copy()
|
|
355
|
+
|
|
349
356
|
for bit in reversed(bits):
|
|
350
357
|
protocol = kwargs.pop("protocol", None) or split_protocol(bit)[0] or "file"
|
|
351
358
|
cls = get_filesystem_class(protocol)
|
|
352
359
|
extra_kwargs = cls._get_kwargs_from_urls(bit)
|
|
353
360
|
kws = kwargs.pop(protocol, {})
|
|
361
|
+
|
|
354
362
|
if bit is bits[0]:
|
|
355
363
|
kws.update(kwargs)
|
|
364
|
+
|
|
356
365
|
kw = dict(
|
|
357
366
|
**{k: v for k, v in extra_kwargs.items() if k not in kws or v != kws[k]},
|
|
358
367
|
**kws,
|
|
359
368
|
)
|
|
360
369
|
bit = cls._strip_protocol(bit)
|
|
361
|
-
|
|
370
|
+
|
|
371
|
+
if (
|
|
372
|
+
"target_protocol" not in kw
|
|
373
|
+
and issubclass(cls, ChainedFileSystem)
|
|
374
|
+
and not bit
|
|
375
|
+
):
|
|
376
|
+
# replace bit if we are chaining and no path given
|
|
362
377
|
bit = previous_bit
|
|
378
|
+
|
|
363
379
|
out.append((bit, protocol, kw))
|
|
364
380
|
previous_bit = bit
|
|
381
|
+
|
|
365
382
|
out.reverse()
|
|
366
383
|
return out
|
|
367
384
|
|
|
@@ -205,11 +205,11 @@ class ArrowFSWrapper(AbstractFileSystem):
|
|
|
205
205
|
return self.fs.get_file_info(path).mtime
|
|
206
206
|
|
|
207
207
|
def cat_file(self, path, start=None, end=None, **kwargs):
|
|
208
|
-
kwargs
|
|
208
|
+
kwargs.setdefault("seekable", start not in [None, 0])
|
|
209
209
|
return super().cat_file(path, start=None, end=None, **kwargs)
|
|
210
210
|
|
|
211
211
|
def get_file(self, rpath, lpath, **kwargs):
|
|
212
|
-
kwargs
|
|
212
|
+
kwargs.setdefault("seekable", False)
|
|
213
213
|
super().get_file(rpath, lpath, **kwargs)
|
|
214
214
|
|
|
215
215
|
|
|
@@ -223,7 +223,6 @@ class ArrowFSWrapper(AbstractFileSystem):
|
|
|
223
223
|
"readable",
|
|
224
224
|
"writable",
|
|
225
225
|
"close",
|
|
226
|
-
"size",
|
|
227
226
|
"seekable",
|
|
228
227
|
],
|
|
229
228
|
)
|
|
@@ -241,6 +240,10 @@ class ArrowFile(io.IOBase):
|
|
|
241
240
|
def __enter__(self):
|
|
242
241
|
return self
|
|
243
242
|
|
|
243
|
+
@property
|
|
244
|
+
def size(self):
|
|
245
|
+
return self.stream.size()
|
|
246
|
+
|
|
244
247
|
def __exit__(self, *args):
|
|
245
248
|
return self.close()
|
|
246
249
|
|
|
@@ -5,6 +5,8 @@ import inspect
|
|
|
5
5
|
import fsspec
|
|
6
6
|
from fsspec.asyn import AsyncFileSystem, running_async
|
|
7
7
|
|
|
8
|
+
from .chained import ChainedFileSystem
|
|
9
|
+
|
|
8
10
|
|
|
9
11
|
def async_wrapper(func, obj=None, semaphore=None):
|
|
10
12
|
"""
|
|
@@ -35,7 +37,7 @@ def async_wrapper(func, obj=None, semaphore=None):
|
|
|
35
37
|
return wrapper
|
|
36
38
|
|
|
37
39
|
|
|
38
|
-
class AsyncFileSystemWrapper(AsyncFileSystem):
|
|
40
|
+
class AsyncFileSystemWrapper(AsyncFileSystem, ChainedFileSystem):
|
|
39
41
|
"""
|
|
40
42
|
A wrapper class to convert a synchronous filesystem into an asynchronous one.
|
|
41
43
|
|
|
@@ -15,9 +15,7 @@ except ImportError:
|
|
|
15
15
|
|
|
16
16
|
if TYPE_CHECKING:
|
|
17
17
|
from collections.abc import Iterator
|
|
18
|
-
from typing import Any, Literal
|
|
19
|
-
|
|
20
|
-
from typing_extensions import TypeAlias
|
|
18
|
+
from typing import Any, Literal, TypeAlias
|
|
21
19
|
|
|
22
20
|
from .cached import CachingFileSystem
|
|
23
21
|
|
|
@@ -6,8 +6,9 @@ import os
|
|
|
6
6
|
import tempfile
|
|
7
7
|
import time
|
|
8
8
|
import weakref
|
|
9
|
+
from collections.abc import Callable
|
|
9
10
|
from shutil import rmtree
|
|
10
|
-
from typing import TYPE_CHECKING, Any,
|
|
11
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
11
12
|
|
|
12
13
|
from fsspec import filesystem
|
|
13
14
|
from fsspec.callbacks import DEFAULT_CALLBACK
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import base64
|
|
2
2
|
import io
|
|
3
|
-
from typing import Optional
|
|
4
3
|
from urllib.parse import unquote
|
|
5
4
|
|
|
6
5
|
from fsspec import AbstractFileSystem
|
|
@@ -50,7 +49,7 @@ class DataFileSystem(AbstractFileSystem):
|
|
|
50
49
|
return io.BytesIO(self.cat_file(path))
|
|
51
50
|
|
|
52
51
|
@staticmethod
|
|
53
|
-
def encode(data: bytes, mime:
|
|
52
|
+
def encode(data: bytes, mime: str | None = None):
|
|
54
53
|
"""Format the given data into data-URL syntax
|
|
55
54
|
|
|
56
55
|
This version always base64 encodes, even when the data is ascii/url-safe.
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
from .. import filesystem
|
|
2
2
|
from ..asyn import AsyncFileSystem
|
|
3
|
+
from .chained import ChainedFileSystem
|
|
3
4
|
|
|
4
5
|
|
|
5
|
-
class DirFileSystem(AsyncFileSystem):
|
|
6
|
+
class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
|
|
6
7
|
"""Directory prefix filesystem
|
|
7
8
|
|
|
8
9
|
The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with
|