fsspec 2025.10.0__tar.gz → 2026.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {fsspec-2025.10.0 → fsspec-2026.1.0}/.github/workflows/main.yaml +6 -6
  2. {fsspec-2025.10.0 → fsspec-2026.1.0}/.github/workflows/pypipublish.yaml +2 -2
  3. {fsspec-2025.10.0 → fsspec-2026.1.0}/.pre-commit-config.yaml +1 -2
  4. {fsspec-2025.10.0 → fsspec-2026.1.0}/PKG-INFO +7 -7
  5. {fsspec-2025.10.0 → fsspec-2026.1.0}/README.md +1 -1
  6. {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-friends.yml +1 -0
  7. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/environment.yml +1 -1
  8. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/api.rst +8 -2
  9. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/changelog.rst +40 -0
  10. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/features.rst +1 -1
  11. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/_version.py +2 -2
  12. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/asyn.py +7 -1
  13. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/caching.py +52 -45
  14. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/compression.py +7 -18
  15. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/core.py +20 -3
  16. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/arrow.py +6 -3
  17. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/asyn_wrapper.py +3 -1
  18. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/cache_metadata.py +1 -3
  19. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/cached.py +2 -1
  20. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/data.py +1 -2
  21. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/dirfs.py +2 -1
  22. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/ftp.py +54 -4
  23. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/http.py +7 -1
  24. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/http_sync.py +7 -1
  25. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/libarchive.py +1 -1
  26. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/local.py +4 -0
  27. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/reference.py +1 -1
  28. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/json.py +7 -12
  29. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/parquet.py +100 -61
  30. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/registry.py +4 -1
  31. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/utils.py +3 -10
  32. {fsspec-2025.10.0 → fsspec-2026.1.0}/pyproject.toml +7 -7
  33. {fsspec-2025.10.0 → fsspec-2026.1.0}/.codespellrc +0 -0
  34. {fsspec-2025.10.0 → fsspec-2026.1.0}/.coveragerc +0 -0
  35. {fsspec-2025.10.0 → fsspec-2026.1.0}/.gitattributes +0 -0
  36. {fsspec-2025.10.0 → fsspec-2026.1.0}/.gitignore +0 -0
  37. {fsspec-2025.10.0 → fsspec-2026.1.0}/LICENSE +0 -0
  38. {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-downstream.yml +0 -0
  39. {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-linux.yml +0 -0
  40. {fsspec-2025.10.0 → fsspec-2026.1.0}/ci/environment-win.yml +0 -0
  41. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/Makefile +0 -0
  42. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/README.md +0 -0
  43. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/make.bat +0 -0
  44. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/_static/custom.css +0 -0
  45. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/async.rst +0 -0
  46. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/code-of-conduct.rst +0 -0
  47. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/conf.py +0 -0
  48. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/copying.rst +0 -0
  49. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/developer.rst +0 -0
  50. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/img/gui.png +0 -0
  51. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/index.rst +0 -0
  52. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/intro.rst +0 -0
  53. {fsspec-2025.10.0 → fsspec-2026.1.0}/docs/source/usage.rst +0 -0
  54. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/__init__.py +0 -0
  55. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/archive.py +0 -0
  56. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/callbacks.py +0 -0
  57. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/config.py +0 -0
  58. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/conftest.py +0 -0
  59. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/dircache.py +0 -0
  60. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/exceptions.py +0 -0
  61. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/fuse.py +0 -0
  62. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/generic.py +0 -0
  63. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/gui.py +0 -0
  64. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/__init__.py +0 -0
  65. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/cache_mapper.py +0 -0
  66. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/chained.py +0 -0
  67. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/dask.py +0 -0
  68. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/dbfs.py +0 -0
  69. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/gist.py +0 -0
  70. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/git.py +0 -0
  71. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/github.py +0 -0
  72. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/jupyter.py +0 -0
  73. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/memory.py +0 -0
  74. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/sftp.py +0 -0
  75. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/smb.py +0 -0
  76. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/tar.py +0 -0
  77. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/webhdfs.py +0 -0
  78. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/implementations/zip.py +0 -0
  79. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/mapping.py +0 -0
  80. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/spec.py +0 -0
  81. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/__init__.py +0 -0
  82. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/common.py +0 -0
  83. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/copy.py +0 -0
  84. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/get.py +0 -0
  85. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/mv.py +0 -0
  86. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/open.py +0 -0
  87. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/pipe.py +0 -0
  88. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/tests/abstract/put.py +0 -0
  89. {fsspec-2025.10.0 → fsspec-2026.1.0}/fsspec/transaction.py +0 -0
  90. {fsspec-2025.10.0 → fsspec-2026.1.0}/install_s3fs.sh +0 -0
  91. {fsspec-2025.10.0 → fsspec-2026.1.0}/readthedocs.yml +0 -0
  92. {fsspec-2025.10.0 → fsspec-2026.1.0}/setup.cfg +0 -0
@@ -14,18 +14,18 @@ jobs:
14
14
  fail-fast: false
15
15
  matrix:
16
16
  PY:
17
- - "3.9"
18
17
  - "3.10"
19
18
  - "3.11"
20
19
  - "3.12"
21
20
  - "3.13"
21
+ - "3.14"
22
22
 
23
23
  env:
24
24
  CIRUN: true
25
25
 
26
26
  steps:
27
27
  - name: Checkout
28
- uses: actions/checkout@v4
28
+ uses: actions/checkout@v5
29
29
  with:
30
30
  fetch-depth: 0
31
31
 
@@ -50,7 +50,7 @@ jobs:
50
50
 
51
51
  steps:
52
52
  - name: Checkout
53
- uses: actions/checkout@v4
53
+ uses: actions/checkout@v5
54
54
  with:
55
55
  fetch-depth: 0
56
56
 
@@ -81,7 +81,7 @@ jobs:
81
81
 
82
82
  steps:
83
83
  - name: Checkout
84
- uses: actions/checkout@v4
84
+ uses: actions/checkout@v5
85
85
  with:
86
86
  fetch-depth: 0
87
87
 
@@ -124,7 +124,7 @@ jobs:
124
124
 
125
125
  steps:
126
126
  - name: Checkout
127
- uses: actions/checkout@v4
127
+ uses: actions/checkout@v5
128
128
 
129
129
  - name: Setup conda
130
130
  uses: conda-incubator/setup-miniconda@v3
@@ -145,5 +145,5 @@ jobs:
145
145
  shell: bash -l {0}
146
146
  run: |
147
147
  cd ${{ matrix.FRIEND }}
148
- pytest -v
148
+ pytest -v -W ignore::pytest.PytestRemovedIn9Warning
149
149
  cd ..
@@ -8,9 +8,9 @@ jobs:
8
8
  deploy:
9
9
  runs-on: ubuntu-latest
10
10
  steps:
11
- - uses: actions/checkout@v4
11
+ - uses: actions/checkout@v5
12
12
  - name: Set up Python
13
- uses: actions/setup-python@v4
13
+ uses: actions/setup-python@v6
14
14
  with:
15
15
  python-version: "3.x"
16
16
  - name: Install dependencies
@@ -13,9 +13,8 @@ repos:
13
13
  - id: check-json
14
14
  - id: check-yaml
15
15
  - repo: https://github.com/astral-sh/ruff-pre-commit
16
- rev: v0.12.2
16
+ rev: v0.14.3
17
17
  hooks:
18
- # Run the linter.
19
18
  - id: ruff-check
20
19
  args: [ --fix, "--show-fixes"]
21
20
  - id: ruff-format
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fsspec
3
- Version: 2025.10.0
3
+ Version: 2026.1.0
4
4
  Summary: File-system specification
5
5
  Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
6
6
  Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
@@ -12,12 +12,12 @@ Keywords: file
12
12
  Classifier: Development Status :: 4 - Beta
13
13
  Classifier: Intended Audience :: Developers
14
14
  Classifier: Operating System :: OS Independent
15
- Classifier: Programming Language :: Python :: 3.9
16
15
  Classifier: Programming Language :: Python :: 3.10
17
16
  Classifier: Programming Language :: Python :: 3.11
18
17
  Classifier: Programming Language :: Python :: 3.12
19
18
  Classifier: Programming Language :: Python :: 3.13
20
- Requires-Python: >=3.9
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Requires-Python: >=3.10
21
21
  Provides-Extra: abfs
22
22
  Requires-Dist: adlfs; extra == 'abfs'
23
23
  Provides-Extra: adl
@@ -49,7 +49,7 @@ Requires-Dist: distributed; extra == 'full'
49
49
  Requires-Dist: dropbox; extra == 'full'
50
50
  Requires-Dist: dropboxdrivefs; extra == 'full'
51
51
  Requires-Dist: fusepy; extra == 'full'
52
- Requires-Dist: gcsfs; extra == 'full'
52
+ Requires-Dist: gcsfs>2024.2.0; extra == 'full'
53
53
  Requires-Dist: libarchive-c; extra == 'full'
54
54
  Requires-Dist: ocifs; extra == 'full'
55
55
  Requires-Dist: panel; extra == 'full'
@@ -57,7 +57,7 @@ Requires-Dist: paramiko; extra == 'full'
57
57
  Requires-Dist: pyarrow>=1; extra == 'full'
58
58
  Requires-Dist: pygit2; extra == 'full'
59
59
  Requires-Dist: requests; extra == 'full'
60
- Requires-Dist: s3fs; extra == 'full'
60
+ Requires-Dist: s3fs>2024.2.0; extra == 'full'
61
61
  Requires-Dist: smbprotocol; extra == 'full'
62
62
  Requires-Dist: tqdm; extra == 'full'
63
63
  Provides-Extra: fuse
@@ -108,6 +108,7 @@ Requires-Dist: xarray; extra == 'test-downstream'
108
108
  Provides-Extra: test-full
109
109
  Requires-Dist: adlfs; extra == 'test-full'
110
110
  Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test-full'
111
+ Requires-Dist: backports-zstd; (python_version < '3.14') and extra == 'test-full'
111
112
  Requires-Dist: cloudpickle; extra == 'test-full'
112
113
  Requires-Dist: dask; extra == 'test-full'
113
114
  Requires-Dist: distributed; extra == 'test-full'
@@ -143,7 +144,6 @@ Requires-Dist: smbprotocol; extra == 'test-full'
143
144
  Requires-Dist: tqdm; extra == 'test-full'
144
145
  Requires-Dist: urllib3; extra == 'test-full'
145
146
  Requires-Dist: zarr; extra == 'test-full'
146
- Requires-Dist: zstandard; (python_version < '3.14') and extra == 'test-full'
147
147
  Provides-Extra: tqdm
148
148
  Requires-Dist: tqdm; extra == 'tqdm'
149
149
  Description-Content-Type: text/markdown
@@ -197,7 +197,7 @@ CI runtime. For local use, pick a version suitable for you.
197
197
 
198
198
  ```bash
199
199
  # For a new environment (mamba / conda).
200
- mamba create -n fsspec -c conda-forge python=3.9 -y
200
+ mamba create -n fsspec -c conda-forge python=3.10 -y
201
201
  conda activate fsspec
202
202
 
203
203
  # Standard dev install with docs and tests.
@@ -47,7 +47,7 @@ CI runtime. For local use, pick a version suitable for you.
47
47
 
48
48
  ```bash
49
49
  # For a new environment (mamba / conda).
50
- mamba create -n fsspec -c conda-forge python=3.9 -y
50
+ mamba create -n fsspec -c conda-forge python=3.10 -y
51
51
  conda activate fsspec
52
52
 
53
53
  # Standard dev install with docs and tests.
@@ -18,6 +18,7 @@ dependencies:
18
18
  - google-auth-oauthlib
19
19
  - flake8
20
20
  - black
21
+ - psutil
21
22
  - google-cloud-core
22
23
  - google-cloud-storage
23
24
  - google-api-core
@@ -2,4 +2,4 @@ name: fsspec
2
2
  channels:
3
3
  - defaults
4
4
  dependencies:
5
- - python=3.9
5
+ - python=3.10
@@ -235,23 +235,26 @@ documentation carefully before using any particular package.
235
235
  - `irods`_ for access to iRODS servers, with protocol "irods://"
236
236
  - `lakefs`_ for lakeFS data lakes, with protocol "lakefs://"
237
237
  - `morefs`_ for `OverlayFileSystem`, `DictFileSystem`, and others
238
+ - `msgraphfs`_ for Microsoft storage (ie Sharepoint) using the drive API through Microsoft Graph, with protocol "msgd://"
238
239
  - `obstore`_: zero-dependency access to Amazon S3, Google Cloud Storage, and Azure Blob Storage using the underlying Rust `object_store`_ library, with protocols "s3://", "gs://", and "abfs://".
239
240
  - `ocifs`_ for access to Oracle Cloud Object Storage, with protocol "oci://"
240
241
  - `ocilake`_ for OCI Data Lake storage
241
242
  - `ossfs`_ for Alibaba Cloud (Aliyun) Object Storage System (OSS)
242
243
  - `p9fs`_ for 9P (Plan 9 Filesystem Protocol) servers
243
244
  - `PyAthena`_ for S3 access to Amazon Athena, with protocol "s3://" or "s3a://"
245
+ - `fsspec-pydantic`_: Pydantic models to represent file and directory types
244
246
  - `PyDrive2`_ for Google Drive access
247
+ - `fsspec-python`_: A chained filesystem that connects to Python's import system, to allow for importing from an fsspec backend
245
248
  - `fsspec-proxy`_ for "pyscript:" URLs via a proxy server
246
249
  - `s3fs`_ for Amazon S3 and other compatible stores, with protocol "s3://"
247
250
  - `sshfs`_ for access to SSH servers, with protocol "ssh://" or "sftp://"
248
251
  - `swiftspec`_ for OpenStack SWIFT, with protocol "swift://"
249
252
  - `tosfs`_ for ByteDance volcano engine Tinder Object Storage (TOS)
253
+ - `fsspec-union`_: A chained filesystem that unions multiple fsspec backends as a read-through cache
250
254
  - `wandbfs`_ to access Wandb run data (experimental)
251
255
  - `wandbfsspec`_ to access Weights & Biases (experimental)
252
256
  - `webdav4`_ for WebDAV, with protocol "webdav://" or "dav://"
253
257
  - `xrootd`_ for xrootd, with protocol "root://"
254
- - `msgraphfs`_ for Microsoft storage (ie Sharepoint) using the drive API through Microsoft Graph, with protocol "msgd://"
255
258
 
256
259
  .. _abfs: https://github.com/dask/adlfs
257
260
  .. _adl: https://github.com/dask/adlfs
@@ -272,6 +275,7 @@ documentation carefully before using any particular package.
272
275
  .. _irods: https://github.com/xwcl/irods_fsspec
273
276
  .. _lakefs: https://github.com/aai-institute/lakefs-spec
274
277
  .. _morefs: https://github.com/iterative/morefs
278
+ .. _msgraphfs: https://github.com/acsone/msgraphfs
275
279
  .. _object_store: https://docs.rs/object_store/latest/object_store/
276
280
  .. _obstore: https://developmentseed.org/obstore/latest/
277
281
  .. _ocifs: https://ocifs.readthedocs.io/en/latest/
@@ -279,16 +283,18 @@ documentation carefully before using any particular package.
279
283
  .. _ossfs: https://github.com/fsspec/ossfs
280
284
  .. _p9fs: https://github.com/pbchekin/p9fs-py
281
285
  .. _PyAthena: https://github.com/laughingman7743/PyAthena
286
+ .. _fsspec-pydantic: https://github.com/1kbgz/fsspec-pydantic
282
287
  .. _PyDrive2: https://github.com/iterative/PyDrive2
288
+ .. _fsspec-python: https://github.com/1kbgz/fsspec-python
283
289
  .. _s3fs: https://s3fs.readthedocs.io/en/latest/
284
290
  .. _sshfs: https://github.com/fsspec/sshfs
285
291
  .. _swiftspec: https://github.com/fsspec/swiftspec
286
292
  .. _tosfs: https://tosfs.readthedocs.io/en/latest/
293
+ .. _fsspec-union: https://github.com/1kbgz/fsspec-union
287
294
  .. _wandbfs: https://github.com/jkulhanek/wandbfs
288
295
  .. _wandbfsspec: https://github.com/alvarobartt/wandbfsspec
289
296
  .. _webdav4: https://github.com/skshetry/webdav4
290
297
  .. _xrootd: https://github.com/CoffeaTeam/fsspec-xrootd
291
- .. _msgraphfs: https://github.com/acsone/msgraphfs
292
298
 
293
299
  .. _readbuffering:
294
300
 
@@ -1,6 +1,46 @@
1
1
  Changelog
2
2
  =========
3
3
 
4
+ 2026.1.0
5
+ --------
6
+
7
+ Put limits on install extras for s3fs/gcsfs to prevent installing
8
+ ancient versions via pip.
9
+
10
+ Enhancements
11
+
12
+ - configure TLS protocol version for FTP (#1958)
13
+
14
+ Fixes
15
+
16
+ - Respect auto_mkdir in mv for local (#1957)
17
+
18
+ Other
19
+
20
+ - link to pydantic, code and union implementations ($1956)
21
+ - use backport.zstd or builtin (#1962)
22
+ - docs typo (#1963)
23
+
24
+ 2025.12.0
25
+ ---------
26
+
27
+ Enhancements
28
+
29
+ - fsspec.parquet to support filters and multiple files (#1945)
30
+
31
+ Fixes
32
+
33
+ - passing withdirs in aync _glob() (#1953)
34
+ - fix _rm_file/_rm redirection in async (#1951)
35
+ - allow arrowFile to be seekable (#1950)
36
+ - add size attribute to arrowFile (#1944)
37
+
38
+
39
+ Other
40
+
41
+ - support py3.14 and drop 3.9 (#1946)
42
+ - avoid ruff warning (#1942)
43
+
4
44
  2025.10.0
5
45
  ---------
6
46
 
@@ -161,7 +161,7 @@ be possible to reach all paths on the remote.
161
161
  Instance Caching
162
162
  ----------------
163
163
 
164
- In a file-system implementation class is marked as *cachable* (attribute ``.cachable``),
164
+ If a file-system implementation class is marked as *cachable* (attribute ``.cachable``),
165
165
  then its instances will
166
166
  get stored in a class attribute, to enable quick look-up instead of needing to regenerate
167
167
  potentially expensive connections and sessions. They key in the cache is a tokenisation of
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '2025.10.0'
32
- __version_tuple__ = version_tuple = (2025, 10, 0)
31
+ __version__ = version = '2026.1.0'
32
+ __version_tuple__ = version_tuple = (2026, 1, 0)
33
33
 
34
34
  __commit_id__ = commit_id = None
@@ -328,6 +328,11 @@ class AsyncFileSystem(AbstractFileSystem):
328
328
  return self._loop
329
329
 
330
330
  async def _rm_file(self, path, **kwargs):
331
+ if (
332
+ inspect.iscoroutinefunction(self._rm)
333
+ and type(self)._rm is not AsyncFileSystem._rm
334
+ ):
335
+ return await self._rm(path, recursive=False, batch_size=1, **kwargs)
331
336
  raise NotImplementedError
332
337
 
333
338
  async def _rm(self, path, recursive=False, batch_size=None, **kwargs):
@@ -776,6 +781,7 @@ class AsyncFileSystem(AbstractFileSystem):
776
781
  min_idx = min(idx_star, idx_qmark, idx_brace)
777
782
 
778
783
  detail = kwargs.pop("detail", False)
784
+ withdirs = kwargs.pop("withdirs", True)
779
785
 
780
786
  if not has_magic(path):
781
787
  if await self._exists(path, **kwargs):
@@ -805,7 +811,7 @@ class AsyncFileSystem(AbstractFileSystem):
805
811
  depth = None
806
812
 
807
813
  allpaths = await self._find(
808
- root, maxdepth=depth, withdirs=True, detail=True, **kwargs
814
+ root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs
809
815
  )
810
816
 
811
817
  pattern = glob_translate(path + ("/" if ends_with_sep else ""))
@@ -6,20 +6,12 @@ import logging
6
6
  import math
7
7
  import os
8
8
  import threading
9
- import warnings
10
9
  from collections import OrderedDict
10
+ from collections.abc import Callable
11
11
  from concurrent.futures import Future, ThreadPoolExecutor
12
12
  from itertools import groupby
13
13
  from operator import itemgetter
14
- from typing import (
15
- TYPE_CHECKING,
16
- Any,
17
- Callable,
18
- ClassVar,
19
- Generic,
20
- NamedTuple,
21
- TypeVar,
22
- )
14
+ from typing import TYPE_CHECKING, Any, ClassVar, Generic, NamedTuple, TypeVar
23
15
 
24
16
  if TYPE_CHECKING:
25
17
  import mmap
@@ -629,7 +621,7 @@ class KnownPartsOfAFile(BaseCache):
629
621
  fetcher: Fetcher,
630
622
  size: int,
631
623
  data: dict[tuple[int, int], bytes] | None = None,
632
- strict: bool = True,
624
+ strict: bool = False,
633
625
  **_: Any,
634
626
  ):
635
627
  super().__init__(blocksize, fetcher, size)
@@ -653,50 +645,65 @@ class KnownPartsOfAFile(BaseCache):
653
645
  else:
654
646
  self.data = {}
655
647
 
648
+ @property
649
+ def size(self):
650
+ return sum(_[1] - _[0] for _ in self.data)
651
+
652
+ @size.setter
653
+ def size(self, value):
654
+ pass
655
+
656
+ @property
657
+ def nblocks(self):
658
+ return len(self.data)
659
+
660
+ @nblocks.setter
661
+ def nblocks(self, value):
662
+ pass
663
+
656
664
  def _fetch(self, start: int | None, stop: int | None) -> bytes:
657
665
  if start is None:
658
666
  start = 0
659
667
  if stop is None:
660
668
  stop = self.size
669
+ self.total_requested_bytes += stop - start
661
670
 
662
671
  out = b""
663
- for (loc0, loc1), data in self.data.items():
664
- # If self.strict=False, use zero-padded data
665
- # for reads beyond the end of a "known" buffer
672
+ started = False
673
+ loc_old = 0
674
+ for loc0, loc1 in sorted(self.data):
675
+ if (loc0 <= start < loc1) and (loc0 <= stop <= loc1):
676
+ # entirely within the block
677
+ off = start - loc0
678
+ self.hit_count += 1
679
+ return self.data[(loc0, loc1)][off : off + stop - start]
680
+ if stop <= loc0:
681
+ break
682
+ if started and loc0 > loc_old:
683
+ # a gap where we need data
684
+ self.miss_count += 1
685
+ if self.strict:
686
+ raise ValueError
687
+ out += b"\x00" * (loc0 - loc_old)
666
688
  if loc0 <= start < loc1:
689
+ # found the start
690
+ self.hit_count += 1
667
691
  off = start - loc0
668
- out = data[off : off + stop - start]
669
- if not self.strict or loc0 <= stop <= loc1:
670
- # The request is within a known range, or
671
- # it begins within a known range, and we
672
- # are allowed to pad reads beyond the
673
- # buffer with zero
674
- out += b"\x00" * (stop - start - len(out))
675
- self.hit_count += 1
676
- return out
677
- else:
678
- # The request ends outside a known range,
679
- # and we are being "strict" about reads
680
- # beyond the buffer
681
- start = loc1
682
- break
683
-
684
- # We only get here if there is a request outside the
685
- # known parts of the file. In an ideal world, this
686
- # should never happen
687
- if self.fetcher is None:
688
- # We cannot fetch the data, so raise an error
689
- raise ValueError(f"Read is outside the known file parts: {(start, stop)}. ")
690
- # We can fetch the data, but should warn the user
691
- # that this may be slow
692
- warnings.warn(
693
- f"Read is outside the known file parts: {(start, stop)}. "
694
- f"IO/caching performance may be poor!"
695
- )
696
- logger.debug(f"KnownPartsOfAFile cache fetching {start}-{stop}")
697
- self.total_requested_bytes += stop - start
692
+ out = self.data[(loc0, loc1)][off : off + stop - start]
693
+ started = True
694
+ elif start < loc0 and stop > loc1:
695
+ # the whole block
696
+ self.hit_count += 1
697
+ out += self.data[(loc0, loc1)]
698
+ elif loc0 <= stop <= loc1:
699
+ # end block
700
+ self.hit_count += 1
701
+ return out + self.data[(loc0, loc1)][: stop - loc0]
702
+ loc_old = loc1
698
703
  self.miss_count += 1
699
- return out + super()._fetch(start, stop)
704
+ if started and not self.strict:
705
+ return out + b"\x00" * (stop - loc_old)
706
+ raise ValueError
700
707
 
701
708
 
702
709
  class UpdatableLRU(Generic[P, T]):
@@ -1,5 +1,6 @@
1
1
  """Helper functions for a standard streaming compression API"""
2
2
 
3
+ import sys
3
4
  from zipfile import ZipFile
4
5
 
5
6
  import fsspec.utils
@@ -155,26 +156,14 @@ except ImportError:
155
156
  pass
156
157
 
157
158
  try:
158
- # zstd in the standard library for python >= 3.14
159
- from compression.zstd import ZstdFile
160
-
161
- register_compression("zstd", ZstdFile, "zst")
159
+ if sys.version_info >= (3, 14):
160
+ from compression import zstd
161
+ else:
162
+ from backports import zstd
162
163
 
164
+ register_compression("zstd", zstd.ZstdFile, "zst")
163
165
  except ImportError:
164
- try:
165
- import zstandard as zstd
166
-
167
- def zstandard_file(infile, mode="rb"):
168
- if "r" in mode:
169
- cctx = zstd.ZstdDecompressor()
170
- return cctx.stream_reader(infile)
171
- else:
172
- cctx = zstd.ZstdCompressor(level=10)
173
- return cctx.stream_writer(infile)
174
-
175
- register_compression("zstd", zstandard_file, "zst")
176
- except ImportError:
177
- pass
166
+ pass
178
167
 
179
168
 
180
169
  def available_compressions():
@@ -18,7 +18,7 @@ from fsspec.caching import ( # noqa: F401
18
18
  )
19
19
  from fsspec.compression import compr
20
20
  from fsspec.config import conf
21
- from fsspec.registry import filesystem, get_filesystem_class
21
+ from fsspec.registry import available_protocols, filesystem, get_filesystem_class
22
22
  from fsspec.utils import (
23
23
  _unstrip_protocol,
24
24
  build_name_function,
@@ -334,34 +334,51 @@ def _un_chain(path, kwargs):
334
334
 
335
335
  if "::" in path:
336
336
  x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word
337
+ known_protocols = set(available_protocols())
337
338
  bits = []
339
+
340
+ # split on '::', then ensure each bit has a protocol
338
341
  for p in path.split("::"):
339
- if "://" in p or x.match(p):
342
+ if p in known_protocols:
343
+ bits.append(p + "://")
344
+ elif "://" in p or x.match(p):
340
345
  bits.append(p)
341
346
  else:
342
347
  bits.append(p + "://")
343
348
  else:
344
349
  bits = [path]
350
+
345
351
  # [[url, protocol, kwargs], ...]
346
352
  out = []
347
353
  previous_bit = None
348
354
  kwargs = kwargs.copy()
355
+
349
356
  for bit in reversed(bits):
350
357
  protocol = kwargs.pop("protocol", None) or split_protocol(bit)[0] or "file"
351
358
  cls = get_filesystem_class(protocol)
352
359
  extra_kwargs = cls._get_kwargs_from_urls(bit)
353
360
  kws = kwargs.pop(protocol, {})
361
+
354
362
  if bit is bits[0]:
355
363
  kws.update(kwargs)
364
+
356
365
  kw = dict(
357
366
  **{k: v for k, v in extra_kwargs.items() if k not in kws or v != kws[k]},
358
367
  **kws,
359
368
  )
360
369
  bit = cls._strip_protocol(bit)
361
- if "target_protocol" not in kw and issubclass(cls, ChainedFileSystem):
370
+
371
+ if (
372
+ "target_protocol" not in kw
373
+ and issubclass(cls, ChainedFileSystem)
374
+ and not bit
375
+ ):
376
+ # replace bit if we are chaining and no path given
362
377
  bit = previous_bit
378
+
363
379
  out.append((bit, protocol, kw))
364
380
  previous_bit = bit
381
+
365
382
  out.reverse()
366
383
  return out
367
384
 
@@ -205,11 +205,11 @@ class ArrowFSWrapper(AbstractFileSystem):
205
205
  return self.fs.get_file_info(path).mtime
206
206
 
207
207
  def cat_file(self, path, start=None, end=None, **kwargs):
208
- kwargs["seekable"] = start not in [None, 0]
208
+ kwargs.setdefault("seekable", start not in [None, 0])
209
209
  return super().cat_file(path, start=None, end=None, **kwargs)
210
210
 
211
211
  def get_file(self, rpath, lpath, **kwargs):
212
- kwargs["seekable"] = False
212
+ kwargs.setdefault("seekable", False)
213
213
  super().get_file(rpath, lpath, **kwargs)
214
214
 
215
215
 
@@ -223,7 +223,6 @@ class ArrowFSWrapper(AbstractFileSystem):
223
223
  "readable",
224
224
  "writable",
225
225
  "close",
226
- "size",
227
226
  "seekable",
228
227
  ],
229
228
  )
@@ -241,6 +240,10 @@ class ArrowFile(io.IOBase):
241
240
  def __enter__(self):
242
241
  return self
243
242
 
243
+ @property
244
+ def size(self):
245
+ return self.stream.size()
246
+
244
247
  def __exit__(self, *args):
245
248
  return self.close()
246
249
 
@@ -5,6 +5,8 @@ import inspect
5
5
  import fsspec
6
6
  from fsspec.asyn import AsyncFileSystem, running_async
7
7
 
8
+ from .chained import ChainedFileSystem
9
+
8
10
 
9
11
  def async_wrapper(func, obj=None, semaphore=None):
10
12
  """
@@ -35,7 +37,7 @@ def async_wrapper(func, obj=None, semaphore=None):
35
37
  return wrapper
36
38
 
37
39
 
38
- class AsyncFileSystemWrapper(AsyncFileSystem):
40
+ class AsyncFileSystemWrapper(AsyncFileSystem, ChainedFileSystem):
39
41
  """
40
42
  A wrapper class to convert a synchronous filesystem into an asynchronous one.
41
43
 
@@ -15,9 +15,7 @@ except ImportError:
15
15
 
16
16
  if TYPE_CHECKING:
17
17
  from collections.abc import Iterator
18
- from typing import Any, Literal
19
-
20
- from typing_extensions import TypeAlias
18
+ from typing import Any, Literal, TypeAlias
21
19
 
22
20
  from .cached import CachingFileSystem
23
21
 
@@ -6,8 +6,9 @@ import os
6
6
  import tempfile
7
7
  import time
8
8
  import weakref
9
+ from collections.abc import Callable
9
10
  from shutil import rmtree
10
- from typing import TYPE_CHECKING, Any, Callable, ClassVar
11
+ from typing import TYPE_CHECKING, Any, ClassVar
11
12
 
12
13
  from fsspec import filesystem
13
14
  from fsspec.callbacks import DEFAULT_CALLBACK
@@ -1,6 +1,5 @@
1
1
  import base64
2
2
  import io
3
- from typing import Optional
4
3
  from urllib.parse import unquote
5
4
 
6
5
  from fsspec import AbstractFileSystem
@@ -50,7 +49,7 @@ class DataFileSystem(AbstractFileSystem):
50
49
  return io.BytesIO(self.cat_file(path))
51
50
 
52
51
  @staticmethod
53
- def encode(data: bytes, mime: Optional[str] = None):
52
+ def encode(data: bytes, mime: str | None = None):
54
53
  """Format the given data into data-URL syntax
55
54
 
56
55
  This version always base64 encodes, even when the data is ascii/url-safe.
@@ -1,8 +1,9 @@
1
1
  from .. import filesystem
2
2
  from ..asyn import AsyncFileSystem
3
+ from .chained import ChainedFileSystem
3
4
 
4
5
 
5
- class DirFileSystem(AsyncFileSystem):
6
+ class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
6
7
  """Directory prefix filesystem
7
8
 
8
9
  The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with