s3fs 2025.10.0__tar.gz → 2026.1.0__tar.gz

{s3fs-2025.10.0/s3fs.egg-info → s3fs-2026.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: s3fs
-Version: 2025.10.0
+Version: 2026.1.0
 Summary: Convenient Filesystem interface over S3
 Home-page: http://github.com/fsspec/s3fs/
 Maintainer: Martin Durant
@@ -11,21 +11,17 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >= 3.9
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >= 3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: aiobotocore<3.0.0,>=2.5.4
-Requires-Dist: fsspec==2025.10.0
+Requires-Dist: aiobotocore<4.0.0,>=2.5.4
+Requires-Dist: fsspec==2026.1.0
 Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1
-Provides-Extra: awscli
-Requires-Dist: aiobotocore[awscli]<3.0.0,>=2.5.4; extra == "awscli"
-Provides-Extra: boto3
-Requires-Dist: aiobotocore[boto3]<3.0.0,>=2.5.4; extra == "boto3"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
@@ -35,7 +31,6 @@ Dynamic: license
 Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
-Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{s3fs-2025.10.0 → s3fs-2026.1.0}/docs/source/changelog.rst

@@ -1,6 +1,18 @@
 Changelog
 =========
 
+2026.1.0
+--------
+
+- allow aiobotocore 3 (#998)
+
+2025.12.0
+---------
+
+- remove optional dependencies (#995)
+- add support for py3.14 and remove 3.9 (#993)
+- add link docs->repo (#992)
+
 2025.10.0
 ---------
 

{s3fs-2025.10.0 → s3fs-2026.1.0}/docs/source/index.rst

@@ -1,7 +1,11 @@
 S3Fs
 ====
 
-S3Fs is a Pythonic file interface to S3.  It builds on top of botocore_.
+S3Fs is a Pythonic file interface to S3.  It builds on top of botocore_. The project is hosted on `GitHub <https://github.com/fsspec/s3fs>`_ |github_stars|
+
+.. |github_stars| image:: https://img.shields.io/github/stars/fsspec/s3fs?style=social
+   :target: https://github.com/fsspec/s3fs
+   :alt: GitHub Repository
 
 The top-level class :py:class:`.S3FileSystem` holds connection information and allows
 typical file-system style operations like ``cp``, ``mv``, ``ls``, ``du``,

s3fs-2026.1.0/requirements.txt

@@ -0,0 +1,3 @@
+aiobotocore>=2.5.4,<4.0.0
+fsspec==2026.1.0
+aiohttp!=4.0.0a0, !=4.0.0a1

{s3fs-2025.10.0 → s3fs-2026.1.0}/s3fs/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-30T11:05:18-0400",
+ "date": "2026-01-09T10:29:07-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "2750bf8dacee65710da91b4292e8ad9653edf782",
- "version": "2025.10.0"
+ "full-revisionid": "a34eac971b397f0874c7843f3251ec7e54c0e810",
+ "version": "2026.1.0"
 }
 '''  # END VERSION_JSON
 

{s3fs-2025.10.0 → s3fs-2026.1.0}/s3fs/core.py

@@ -1,12 +1,11 @@
-# -*- coding: utf-8 -*-
 import asyncio
 import errno
 import io
 import logging
+import math
 import mimetypes
 import os
 import socket
-from typing import Tuple, Optional
 import weakref
 import re
 
@@ -69,6 +68,8 @@ S3_RETRYABLE_ERRORS = (
     ResponseParserError,
 )
 
+MAX_UPLOAD_PARTS = 10_000  # maximum number of parts for S3 multipart upload
+
 if ClientPayloadError is not None:
     S3_RETRYABLE_ERRORS += (ClientPayloadError,)
 
@@ -166,7 +167,7 @@ def _coalesce_version_id(*args):
     if len(version_ids) > 1:
         raise ValueError(
             "Cannot coalesce version_ids where more than one are defined,"
-            " {}".format(version_ids)
+            f" {version_ids}"
         )
     elif len(version_ids) == 0:
         return None
@@ -174,6 +175,18 @@ def _coalesce_version_id(*args):
         return version_ids.pop()
 
 
+def calculate_chunksize(filesize, chunksize=None, max_parts=MAX_UPLOAD_PARTS) -> int:
+    if chunksize is None:
+        chunksize = 50 * 2**20  # default chunksize set to 50 MiB
+        required_chunks = math.ceil(filesize / chunksize)
+        # increase chunksize to fit within the max_parts limit
+        if required_chunks > max_parts:
+            # S3 supports uploading objects up to 5 TiB in size,
+            # so each chunk can be up to ~524 MiB.
+            chunksize = math.ceil(filesize / max_parts)
+    return chunksize
+
+
 class S3FileSystem(AsyncFileSystem):
     """
     Access S3 as if it were a file system.
@@ -440,7 +453,7 @@ class S3FileSystem(AsyncFileSystem):
             s3_key = s3_components[1]
         return bucket, s3_key
 
-    def split_path(self, path) -> Tuple[str, str, Optional[str]]:
+    def split_path(self, path) -> tuple[str, str, str | None]:
         """
         Normalise S3 path string into bucket and key.
 
@@ -764,6 +777,7 @@ class S3FileSystem(AsyncFileSystem):
                     else:
                         files.append(c)
                 files += dirs
+                files.sort(key=lambda f: f["name"])
             except ClientError as e:
                 raise translate_boto_error(e)
 
@@ -887,38 +901,49 @@ class S3FileSystem(AsyncFileSystem):
         sdirs = set()
         thisdircache = {}
         for o in out:
-            par = self._parent(o["name"])
-            if par not in sdirs:
-                sdirs.add(par)
-                d = False
-                if len(path) <= len(par):
-                    d = {
-                        "Key": self.split_path(par)[1],
-                        "Size": 0,
-                        "name": par,
-                        "StorageClass": "DIRECTORY",
-                        "type": "directory",
-                        "size": 0,
-                    }
-                    dirs.append(d)
-                thisdircache[par] = []
-                ppar = self._parent(par)
-                if ppar in thisdircache:
-                    if d and d not in thisdircache[ppar]:
-                        thisdircache[ppar].append(d)
-            if par in sdirs:
-                thisdircache[par].append(o)
+            # not self._parent, because that strips "/" from placeholders
+            par = o["name"].rsplit("/", maxsplit=1)[0]
+            o["Key"] = o["name"]
+            name = o["name"]
+            while "/" in par:
+                if par not in sdirs:
+                    sdirs.add(par)
+                    d = False
+                    if len(path) <= len(par):
+                        d = {
+                            "Key": par,
+                            "Size": 0,
+                            "name": par,
+                            "StorageClass": "DIRECTORY",
+                            "type": "directory",
+                            "size": 0,
+                        }
+                        dirs.append(d)
+                    thisdircache[par] = []
+                    ppar = self._parent(par)
+                    if ppar in thisdircache:
+                        if d and d not in thisdircache[ppar]:
+                            thisdircache[ppar].append(d)
+                if par in sdirs and not name.endswith("/"):
+                    # exclude placeholdees, they do not belong in the directory listing
+                    thisdircache[par].append(o)
+                par, name, o = par.rsplit("/", maxsplit=1)[0], par, d
+                if par in thisdircache or par in self.dircache:
+                    break
 
         # Explicitly add directories to their parents in the dircache
         for d in dirs:
             par = self._parent(d["name"])
-            if par in thisdircache:
+            # extra condition here (in any()) to deal with directory-marking files
+            if par in thisdircache and not any(
+                _["name"] == d["name"] for _ in thisdircache[par]
+            ):
                 thisdircache[par].append(d)
 
         if not prefix:
             for k, v in thisdircache.items():
                 if k not in self.dircache and len(k) >= len(path):
-                    self.dircache[k] = v
+                    self.dircache[k] = sorted(v, key=lambda x: x["name"])
         if withdirs:
             out = sorted(out + dirs, key=lambda x: x["name"])
         if detail:
@@ -1043,7 +1068,7 @@ class S3FileSystem(AsyncFileSystem):
                     files = await self._lsdir(
                         self._parent(path), refresh=refresh, versions=versions
                     )
-                except IOError:
+                except OSError:
                     pass
                 files = [
                     o
@@ -1230,7 +1255,7 @@ class S3FileSystem(AsyncFileSystem):
         lpath,
         rpath,
         callback=_DEFAULT_CALLBACK,
-        chunksize=50 * 2**20,
+        chunksize=None,
         max_concurrency=None,
         mode="overwrite",
         **kwargs,
@@ -1258,6 +1283,7 @@ class S3FileSystem(AsyncFileSystem):
             if content_type is not None:
                 kwargs["ContentType"] = content_type
 
+        chunksize = calculate_chunksize(size, chunksize=chunksize)
         with open(lpath, "rb") as f0:
             if size < min(5 * 2**30, 2 * chunksize):
                 chunk = f0.read()
@@ -1276,8 +1302,8 @@ class S3FileSystem(AsyncFileSystem):
                         key,
                         mpu,
                         f0,
+                        chunksize,
                         callback=callback,
-                        chunksize=chunksize,
                         max_concurrency=max_concurrency,
                     )
                     parts = [
@@ -1305,8 +1331,8 @@ class S3FileSystem(AsyncFileSystem):
         key,
         mpu,
         f0,
+        chunksize,
         callback=_DEFAULT_CALLBACK,
-        chunksize=50 * 2**20,
         max_concurrency=None,
     ):
         max_concurrency = max_concurrency or self.max_concurrency
@@ -2140,7 +2166,7 @@ class S3FileSystem(AsyncFileSystem):
                 path = self._parent(path)
 
     async def _walk(self, path, maxdepth=None, **kwargs):
-        if path in ["", "*"] + ["{}://".format(p) for p in self.protocol]:
+        if path in ["", "*"] + [f"{p}://" for p in self.protocol]:
             raise ValueError("Cannot crawl all of S3")
         async for _ in super()._walk(path, maxdepth=maxdepth, **kwargs):
             yield _

{s3fs-2025.10.0 → s3fs-2026.1.0}/s3fs/errors.py

@@ -155,7 +155,7 @@ def translate_boto_error(error, message=None, set_cause=True, *args, **kwargs):
         custom_exc = constructor(message, *args, **kwargs)
     else:
         # No match found, wrap this in an IOError with the appropriate message.
-        custom_exc = IOError(errno.EIO, message or str(error), *args)
+        custom_exc = OSError(errno.EIO, message or str(error), *args)
 
     if set_cause:
         custom_exc.__cause__ = error

{s3fs-2025.10.0 → s3fs-2026.1.0}/s3fs/tests/derived/s3fs_fixtures.py

@@ -38,9 +38,7 @@ class S3fsFixtures(AbstractFixtures):
                         "Effect": "Deny",
                         "Principal": "*",
                         "Action": "s3:PutObject",
-                        "Resource": "arn:aws:s3:::{bucket_name}/*".format(
-                            bucket_name=secure_bucket_name
-                        ),
+                        "Resource": f"arn:aws:s3:::{secure_bucket_name}/*",
                         "Condition": {
                             "StringNotEquals": {
                                 "s3:x-amz-server-side-encryption": "aws:kms"

{s3fs-2025.10.0 → s3fs-2026.1.0}/s3fs/tests/test_s3fs.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import asyncio
 import errno
 import datetime
@@ -20,7 +19,7 @@ from dateutil.tz import tzutc
 
 import botocore
 import s3fs.core
-from s3fs.core import S3FileSystem
+from s3fs.core import MAX_UPLOAD_PARTS, S3FileSystem, calculate_chunksize
 from s3fs.utils import ignoring, SSEParams
 from botocore.exceptions import NoCredentialsError
 from fsspec.asyn import sync
@@ -128,9 +127,7 @@ def s3(s3_base):
                     "Effect": "Deny",
                     "Principal": "*",
                     "Action": "s3:PutObject",
-                    "Resource": "arn:aws:s3:::{bucket_name}/*".format(
-                        bucket_name=secure_bucket_name
-                    ),
+                    "Resource": f"arn:aws:s3:::{secure_bucket_name}/*",
                     "Condition": {
                         "StringNotEquals": {
                             "s3:x-amz-server-side-encryption": "aws:kms"
@@ -1672,7 +1669,7 @@ def test_fsspec_versions_multiple(s3):
             fo.write(contents)
         version_lookup[fo.version_id] = contents
     urls = [
-        "s3://{}?versionId={}".format(versioned_file, version)
+        f"s3://{versioned_file}?versionId={version}"
         for version in version_lookup.keys()
     ]
     fs, token, paths = fsspec.core.get_fs_token_paths(
@@ -1702,7 +1699,7 @@ def test_versioned_file_fullpath(s3):
     with s3.open(versioned_file, "wb") as fo:
         fo.write(b"2")
 
-    file_with_version = "{}?versionId={}".format(versioned_file, version_id)
+    file_with_version = f"{versioned_file}?versionId={version_id}"
 
     with s3.open(file_with_version, "rb") as fo:
         assert fo.version_id == version_id
@@ -2365,7 +2362,7 @@ def test_get_file_info_with_selector(s3):
             pass
 
         infos = fs.find(base_dir, maxdepth=None, withdirs=True, detail=True)
-        assert len(infos) == 5  # includes base_dir directory
+        assert len(infos) == 4  # includes base_dir directory
 
         for info in infos.values():
             if info["name"].endswith(file_a):
@@ -2993,3 +2990,81 @@ def test_bucket_info(s3):
     assert "VersionId" in info
     assert info["type"] == "directory"
     assert info["name"] == test_bucket_name
+
+
+MB = 2**20
+GB = 2**30
+TB = 2**40
+
+
+@pytest.mark.parametrize(
+    ["filesize", "chunksize", "expected"],
+    [
+        # small file, use default chunksize
+        (1000, None, 50 * MB),
+        # exact boundary, use default chunksize
+        (50 * MB * MAX_UPLOAD_PARTS, None, 50 * MB),
+        # file requiring increased chunksize
+        (50 * MB * (MAX_UPLOAD_PARTS + 1), None, 52_434_043),
+        # very large files, expect increased chunksize
+        (1 * TB, None, 109_951_163),
+        (5 * TB, None, 549_755_814),
+        # respect explicit chunksize
+        (5 * GB, 10 * MB, 10 * MB),
+    ],
+)
+def test_calculate_chunksize(filesize, chunksize, expected):
+    assert calculate_chunksize(filesize, chunksize) == expected
+
+
+def test_find_ls_fail(s3):
+    # beacuse of https://github.com/fsspec/s3fs/pull/989
+    client = get_boto3_client()
+    files = {
+        f"{test_bucket_name}/find/a/a": b"data",
+        f"{test_bucket_name}/find/a/b": b"data",
+        f"{test_bucket_name}/find/a": b"",  # duplicate of dir, without "/"
+        f"{test_bucket_name}/find/b": b"",  # empty file without "/" and no children
+        f"{test_bucket_name}/find/c/c": b"data",  # directory with no placeholder
+        f"{test_bucket_name}/find/d/d": b"data",  # dir will acquire placeholder with "/"
+    }
+    client.put_object(Bucket=test_bucket_name, Key="find/d/", Body=b"")
+    client.put_object(
+        Bucket=test_bucket_name, Key="find/e/", Body=b""
+    )  # placeholder only
+    s3.pipe(files)
+
+    out0 = s3.ls(f"{test_bucket_name}/find", detail=True)
+    s3.find(test_bucket_name, detail=False)
+    out = s3.ls(f"{test_bucket_name}/find", detail=True)
+    assert out == out0
+
+    s3.invalidate_cache()
+    s3.find(f"{test_bucket_name}/find", detail=False)
+    out = s3.ls(f"{test_bucket_name}/find", detail=True)
+    assert out == out0
+
+
+def test_find_missing_ls(s3):
+    # https://github.com/fsspec/s3fs/issues/988#issuecomment-3436727753
+    BUCKET = test_bucket_name
+    BASE_PREFIX = "disappearing-folders/"
+    BASE = f"s3://{BUCKET}/{BASE_PREFIX}"
+
+    s3_with_cache = S3FileSystem(
+        anon=False,
+        use_listings_cache=True,
+        client_kwargs={"endpoint_url": endpoint_uri},
+    )
+    s3_no_cache = S3FileSystem(
+        anon=False,
+        use_listings_cache=False,
+        client_kwargs={"endpoint_url": endpoint_uri},
+    )
+
+    s3_with_cache.pipe({f"{BASE}folder/foo/1.txt": b"", f"{BASE}bar.txt": b""})
+    s3_with_cache.find(BASE)
+    listed_cached = s3_with_cache.ls(BASE, detail=False)
+    listed_no_cache = s3_no_cache.ls(BASE, detail=False)
+
+    assert set(listed_cached) == set(listed_no_cache)

{s3fs-2025.10.0 → s3fs-2026.1.0}/s3fs/utils.py

@@ -118,7 +118,7 @@ def title_case(string):
     return "".join(x.capitalize() for x in string.split("_"))
 
 
-class ParamKwargsHelper(object):
+class ParamKwargsHelper:
     """
     Utility class to help extract the subset of keys that an s3 method is
     actually using
@@ -152,7 +152,7 @@ class ParamKwargsHelper(object):
         return {k: v for k, v in d.items() if k in valid_keys}
 
 
-class SSEParams(object):
+class SSEParams:
     def __init__(
         self,
         server_side_encryption=None,

{s3fs-2025.10.0 → s3fs-2026.1.0/s3fs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: s3fs
-Version: 2025.10.0
+Version: 2026.1.0
 Summary: Convenient Filesystem interface over S3
 Home-page: http://github.com/fsspec/s3fs/
 Maintainer: Martin Durant
@@ -11,21 +11,17 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >= 3.9
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >= 3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: aiobotocore<3.0.0,>=2.5.4
-Requires-Dist: fsspec==2025.10.0
+Requires-Dist: aiobotocore<4.0.0,>=2.5.4
+Requires-Dist: fsspec==2026.1.0
 Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1
-Provides-Extra: awscli
-Requires-Dist: aiobotocore[awscli]<3.0.0,>=2.5.4; extra == "awscli"
-Provides-Extra: boto3
-Requires-Dist: aiobotocore[boto3]<3.0.0,>=2.5.4; extra == "boto3"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
@@ -35,7 +31,6 @@ Dynamic: license
 Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
-Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

s3fs-2026.1.0/s3fs.egg-info/requires.txt

@@ -0,0 +1,3 @@
+aiobotocore<4.0.0,>=2.5.4
+fsspec==2026.1.0
+aiohttp!=4.0.0a0,!=4.0.0a1

{s3fs-2025.10.0 → s3fs-2026.1.0}/setup.py

@@ -3,14 +3,6 @@
 from setuptools import setup
 import versioneer
 
-with open("requirements.txt") as file:
-    aiobotocore_version_suffix = ""
-    for line in file:
-        parts = line.rstrip().split("aiobotocore")
-        if len(parts) == 2:
-            aiobotocore_version_suffix = parts[1]
-            break
-
 setup(
     name="s3fs",
     version=versioneer.get_version(),
@@ -20,11 +12,11 @@ setup(
         "Intended Audience :: Developers",
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
     ],
     description="Convenient Filesystem interface over S3",
     url="http://github.com/fsspec/s3fs/",
@@ -33,12 +25,8 @@ setup(
     license="BSD",
     keywords="s3, boto",
     packages=["s3fs"],
-    python_requires=">= 3.9",
+    python_requires=">= 3.10",
     install_requires=[open("requirements.txt").read().strip().split("\n")],
-    extras_require={
-        "awscli": [f"aiobotocore[awscli]{aiobotocore_version_suffix}"],
-        "boto3": [f"aiobotocore[boto3]{aiobotocore_version_suffix}"],
-    },
     long_description="README.md",
     long_description_content_type="text/markdown",
     zip_safe=False,

s3fs-2025.10.0/requirements.txt

@@ -1,3 +0,0 @@
-aiobotocore>=2.5.4,<3.0.0
-fsspec==2025.10.0
-aiohttp!=4.0.0a0, !=4.0.0a1

s3fs-2025.10.0/s3fs.egg-info/requires.txt

@@ -1,9 +0,0 @@
-aiobotocore<3.0.0,>=2.5.4
-fsspec==2025.10.0
-aiohttp!=4.0.0a0,!=4.0.0a1
-
-[awscli]
-aiobotocore[awscli]<3.0.0,>=2.5.4
-
-[boto3]
-aiobotocore[boto3]<3.0.0,>=2.5.4