s3fs 2025.9.0__py3-none-any.whl → 2025.12.0__py3-none-any.whl

s3fs/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-02T15:17:31-0400",
+ "date": "2025-12-03T10:32:02-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "380b2c6ad868dd90cf8967c862bac1b3154b357b",
- "version": "2025.9.0"
+ "full-revisionid": "65f394575b9667f33b59473dc28a8f1cf6708745",
+ "version": "2025.12.0"
 }
 '''  # END VERSION_JSON
 

s3fs/core.py

@@ -1,12 +1,11 @@
-# -*- coding: utf-8 -*-
 import asyncio
 import errno
 import io
 import logging
+import math
 import mimetypes
 import os
 import socket
-from typing import Tuple, Optional
 import weakref
 import re
 
@@ -69,6 +68,8 @@ S3_RETRYABLE_ERRORS = (
     ResponseParserError,
 )
 
+MAX_UPLOAD_PARTS = 10_000  # maximum number of parts for S3 multipart upload
+
 if ClientPayloadError is not None:
     S3_RETRYABLE_ERRORS += (ClientPayloadError,)
 
@@ -166,7 +167,7 @@ def _coalesce_version_id(*args):
     if len(version_ids) > 1:
         raise ValueError(
             "Cannot coalesce version_ids where more than one are defined,"
-            " {}".format(version_ids)
+            f" {version_ids}"
         )
     elif len(version_ids) == 0:
         return None
@@ -174,6 +175,18 @@ def _coalesce_version_id(*args):
         return version_ids.pop()
 
 
+def calculate_chunksize(filesize, chunksize=None, max_parts=MAX_UPLOAD_PARTS) -> int:
+    if chunksize is None:
+        chunksize = 50 * 2**20  # default chunksize set to 50 MiB
+        required_chunks = math.ceil(filesize / chunksize)
+        # increase chunksize to fit within the max_parts limit
+        if required_chunks > max_parts:
+            # S3 supports uploading objects up to 5 TiB in size,
+            # so each chunk can be up to ~524 MiB.
+            chunksize = math.ceil(filesize / max_parts)
+    return chunksize
+
+
 class S3FileSystem(AsyncFileSystem):
     """
     Access S3 as if it were a file system.
@@ -440,7 +453,7 @@ class S3FileSystem(AsyncFileSystem):
             s3_key = s3_components[1]
         return bucket, s3_key
 
-    def split_path(self, path) -> Tuple[str, str, Optional[str]]:
+    def split_path(self, path) -> tuple[str, str, str | None]:
         """
         Normalise S3 path string into bucket and key.
 
@@ -764,6 +777,7 @@ class S3FileSystem(AsyncFileSystem):
                     else:
                         files.append(c)
                 files += dirs
+                files.sort(key=lambda f: f["name"])
             except ClientError as e:
                 raise translate_boto_error(e)
 
@@ -887,38 +901,49 @@ class S3FileSystem(AsyncFileSystem):
         sdirs = set()
         thisdircache = {}
         for o in out:
-            par = self._parent(o["name"])
-            if par not in sdirs:
-                sdirs.add(par)
-                d = False
-                if len(path) <= len(par):
-                    d = {
-                        "Key": self.split_path(par)[1],
-                        "Size": 0,
-                        "name": par,
-                        "StorageClass": "DIRECTORY",
-                        "type": "directory",
-                        "size": 0,
-                    }
-                    dirs.append(d)
-                thisdircache[par] = []
-                ppar = self._parent(par)
-                if ppar in thisdircache:
-                    if d and d not in thisdircache[ppar]:
-                        thisdircache[ppar].append(d)
-            if par in sdirs:
-                thisdircache[par].append(o)
+            # not self._parent, because that strips "/" from placeholders
+            par = o["name"].rsplit("/", maxsplit=1)[0]
+            o["Key"] = o["name"]
+            name = o["name"]
+            while "/" in par:
+                if par not in sdirs:
+                    sdirs.add(par)
+                    d = False
+                    if len(path) <= len(par):
+                        d = {
+                            "Key": par,
+                            "Size": 0,
+                            "name": par,
+                            "StorageClass": "DIRECTORY",
+                            "type": "directory",
+                            "size": 0,
+                        }
+                        dirs.append(d)
+                    thisdircache[par] = []
+                    ppar = self._parent(par)
+                    if ppar in thisdircache:
+                        if d and d not in thisdircache[ppar]:
+                            thisdircache[ppar].append(d)
+                if par in sdirs and not name.endswith("/"):
+                    # exclude placeholdees, they do not belong in the directory listing
+                    thisdircache[par].append(o)
+                par, name, o = par.rsplit("/", maxsplit=1)[0], par, d
+                if par in thisdircache or par in self.dircache:
+                    break
 
         # Explicitly add directories to their parents in the dircache
         for d in dirs:
             par = self._parent(d["name"])
-            if par in thisdircache:
+            # extra condition here (in any()) to deal with directory-marking files
+            if par in thisdircache and not any(
+                _["name"] == d["name"] for _ in thisdircache[par]
+            ):
                 thisdircache[par].append(d)
 
         if not prefix:
             for k, v in thisdircache.items():
                 if k not in self.dircache and len(k) >= len(path):
-                    self.dircache[k] = v
+                    self.dircache[k] = sorted(v, key=lambda x: x["name"])
         if withdirs:
             out = sorted(out + dirs, key=lambda x: x["name"])
         if detail:
@@ -1043,7 +1068,7 @@ class S3FileSystem(AsyncFileSystem):
                     files = await self._lsdir(
                         self._parent(path), refresh=refresh, versions=versions
                     )
-                except IOError:
+                except OSError:
                     pass
                 files = [
                     o
@@ -1230,7 +1255,7 @@ class S3FileSystem(AsyncFileSystem):
         lpath,
         rpath,
         callback=_DEFAULT_CALLBACK,
-        chunksize=50 * 2**20,
+        chunksize=None,
         max_concurrency=None,
         mode="overwrite",
         **kwargs,
@@ -1258,6 +1283,7 @@ class S3FileSystem(AsyncFileSystem):
             if content_type is not None:
                 kwargs["ContentType"] = content_type
 
+        chunksize = calculate_chunksize(size, chunksize=chunksize)
         with open(lpath, "rb") as f0:
             if size < min(5 * 2**30, 2 * chunksize):
                 chunk = f0.read()
@@ -1276,8 +1302,8 @@ class S3FileSystem(AsyncFileSystem):
                         key,
                         mpu,
                         f0,
+                        chunksize,
                         callback=callback,
-                        chunksize=chunksize,
                         max_concurrency=max_concurrency,
                     )
                     parts = [
@@ -1305,8 +1331,8 @@ class S3FileSystem(AsyncFileSystem):
         key,
         mpu,
         f0,
+        chunksize,
         callback=_DEFAULT_CALLBACK,
-        chunksize=50 * 2**20,
         max_concurrency=None,
     ):
         max_concurrency = max_concurrency or self.max_concurrency
@@ -1464,6 +1490,18 @@ class S3FileSystem(AsyncFileSystem):
                 pass
             except ClientError as e:
                 raise translate_boto_error(e, set_cause=False)
+        else:
+            try:
+                out = await self._call_s3("head_bucket", Bucket=bucket, **self.req_kw)
+                return {
+                    "name": bucket,
+                    "type": "directory",
+                    "size": 0,
+                    "StorageClass": "DIRECTORY",
+                    "VersionId": out.get("VersionId"),
+                }
+            except ClientError as e:
+                raise translate_boto_error(e, set_cause=False)
 
         try:
             # We check to see if the path is a directory by attempting to list its
@@ -2128,7 +2166,7 @@ class S3FileSystem(AsyncFileSystem):
                 path = self._parent(path)
 
     async def _walk(self, path, maxdepth=None, **kwargs):
-        if path in ["", "*"] + ["{}://".format(p) for p in self.protocol]:
+        if path in ["", "*"] + [f"{p}://" for p in self.protocol]:
             raise ValueError("Cannot crawl all of S3")
         async for _ in super()._walk(path, maxdepth=maxdepth, **kwargs):
             yield _

s3fs/errors.py

@@ -155,7 +155,7 @@ def translate_boto_error(error, message=None, set_cause=True, *args, **kwargs):
         custom_exc = constructor(message, *args, **kwargs)
     else:
         # No match found, wrap this in an IOError with the appropriate message.
-        custom_exc = IOError(errno.EIO, message or str(error), *args)
+        custom_exc = OSError(errno.EIO, message or str(error), *args)
 
     if set_cause:
         custom_exc.__cause__ = error

s3fs/utils.py

@@ -118,7 +118,7 @@ def title_case(string):
     return "".join(x.capitalize() for x in string.split("_"))
 
 
-class ParamKwargsHelper(object):
+class ParamKwargsHelper:
     """
     Utility class to help extract the subset of keys that an s3 method is
     actually using
@@ -152,7 +152,7 @@ class ParamKwargsHelper(object):
         return {k: v for k, v in d.items() if k in valid_keys}
 
 
-class SSEParams(object):
+class SSEParams:
     def __init__(
         self,
         server_side_encryption=None,

{s3fs-2025.9.0.dist-info → s3fs-2025.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: s3fs
-Version: 2025.9.0
+Version: 2025.12.0
 Summary: Convenient Filesystem interface over S3
 Home-page: http://github.com/fsspec/s3fs/
 Maintainer: Martin Durant
@@ -11,21 +11,17 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >= 3.9
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >= 3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: aiobotocore<3.0.0,>=2.5.4
-Requires-Dist: fsspec==2025.9.0
+Requires-Dist: fsspec==2025.12.0
 Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1
-Provides-Extra: awscli
-Requires-Dist: aiobotocore[awscli]<3.0.0,>=2.5.4; extra == "awscli"
-Provides-Extra: boto3
-Requires-Dist: aiobotocore[boto3]<3.0.0,>=2.5.4; extra == "boto3"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
@@ -35,7 +31,6 @@ Dynamic: license
 Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
-Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

s3fs-2025.12.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+s3fs/__init__.py,sha256=_6_Vs_vblhJSJw-62JVIBIM8kTKLhuwPFAIvt3hanls,160
+s3fs/_version.py,sha256=XmBtxzsWJ-pBXPcK4HgA6ctXPj2yuE1Fa8aq9prxaT8,501
+s3fs/core.py,sha256=XuAkb1H4ACeNNG_hPdL1Gu3oiQZZzeP0wRnE8BihaOY,93419
+s3fs/errors.py,sha256=dhdN1nTI786q6Gy1hqqdfidcFQ5qeYaYwYSUjxYcDxg,7961
+s3fs/mapping.py,sha256=FoqEdMne7LXUL4HgPN4j6WsMsrwxpb53GynDhXs9VRI,237
+s3fs/utils.py,sha256=VvGSdzmgAsqZIOLU7MKtv0oEClm3YZQ78f4PMhBAX9E,5209
+s3fs-2025.12.0.dist-info/licenses/LICENSE.txt,sha256=3DWZ-ma8G8_6I2g4vi-04V-EzkgBJE7sk9kOUVz8WNE,1505
+s3fs-2025.12.0.dist-info/METADATA,sha256=hNXXt7WgQ8Fuk94tIsE2kHH1Zkis8-RvzrsrEmplLJk,1159
+s3fs-2025.12.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+s3fs-2025.12.0.dist-info/top_level.txt,sha256=Lf6EI3TdjlPu7TN-92IIY6c-GdrHnkrKGb1n0iwKooI,5
+s3fs-2025.12.0.dist-info/RECORD,,

s3fs-2025.9.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3fs/__init__.py,sha256=_6_Vs_vblhJSJw-62JVIBIM8kTKLhuwPFAIvt3hanls,160
-s3fs/_version.py,sha256=gNVM-2xObjX4jpfgf_pafCudQXOz_08hacWkq2q4qTE,500
-s3fs/core.py,sha256=qNTVMJfDF4sbaJmXZoTSB4yiLcmwDeVLPvn3ynsCjyI,91585
-s3fs/errors.py,sha256=GepxwhJMNrCVKMWVE7WzEfuLMU62pibcD9xlr7RegSg,7961
-s3fs/mapping.py,sha256=FoqEdMne7LXUL4HgPN4j6WsMsrwxpb53GynDhXs9VRI,237
-s3fs/utils.py,sha256=33lK0sBH7uXTFnbO9gHjONn3RgF555koyVleK7EITlQ,5225
-s3fs-2025.9.0.dist-info/licenses/LICENSE.txt,sha256=3DWZ-ma8G8_6I2g4vi-04V-EzkgBJE7sk9kOUVz8WNE,1505
-s3fs-2025.9.0.dist-info/METADATA,sha256=LfrM7e2PCGpw7EkOyQ298FRRwXmf385pIKL1lLQCwXg,1358
-s3fs-2025.9.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-s3fs-2025.9.0.dist-info/top_level.txt,sha256=Lf6EI3TdjlPu7TN-92IIY6c-GdrHnkrKGb1n0iwKooI,5
-s3fs-2025.9.0.dist-info/RECORD,,