PyAthena 3.29.1__tar.gz → 3.29.3__tar.gz

{pyathena-3.29.1 → pyathena-3.29.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyAthena
-Version: 3.29.1
+Version: 3.29.3
 Summary: Python DB API 2.0 (PEP 249) client for Amazon Athena
 Project-URL: homepage, https://github.com/pyathena-dev/PyAthena/
 Project-URL: repository, https://github.com/pyathena-dev/PyAthena/
@@ -115,10 +115,10 @@ Native asyncio is also supported:
 
 ```python
 import asyncio
-from pyathena import aconnect
+from pyathena import aio_connect
 
 async def main():
-    async with await aconnect(s3_staging_dir="s3://YOUR_S3_BUCKET/path/to/",
+    async with await aio_connect(s3_staging_dir="s3://YOUR_S3_BUCKET/path/to/",
                               region_name="us-west-2") as conn:
         cursor = conn.cursor()
         await cursor.execute("SELECT 1")

{pyathena-3.29.1 → pyathena-3.29.3}/README.md

@@ -56,10 +56,10 @@ Native asyncio is also supported:
 
 ```python
 import asyncio
-from pyathena import aconnect
+from pyathena import aio_connect
 
 async def main():
-    async with await aconnect(s3_staging_dir="s3://YOUR_S3_BUCKET/path/to/",
+    async with await aio_connect(s3_staging_dir="s3://YOUR_S3_BUCKET/path/to/",
                               region_name="us-west-2") as conn:
         cursor = conn.cursor()
         await cursor.execute("SELECT 1")

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/__init__.py

@@ -131,7 +131,7 @@ def connect(*args, **kwargs) -> "Connection[Any]":
     return Connection(*args, **kwargs)
 
 
-async def aconnect(*args, **kwargs) -> "AioConnection":
+async def aio_connect(*args, **kwargs) -> "AioConnection":
     """Create a new async database connection to Amazon Athena.
 
     This is the async counterpart of :func:`connect`. It returns an
@@ -147,7 +147,7 @@ async def aconnect(*args, **kwargs) -> "AioConnection":
 
     Example:
         >>> import pyathena
-        >>> conn = await pyathena.aconnect(
+        >>> conn = await pyathena.aio_connect(
         ...     s3_staging_dir='s3://my-bucket/staging/',
         ...     region_name='us-east-1',
         ... )

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.29.1'
-__version_tuple__ = version_tuple = (3, 29, 1)
+__version__ = version = '3.29.3'
+__version_tuple__ = version_tuple = (3, 29, 3)
 
 __commit_id__ = commit_id = None

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/aio/arrow/cursor.py

@@ -29,7 +29,7 @@ class AioArrowCursor(WithAsyncFetch):
     operations, keeping the event loop free.
 
     Example:
-        >>> async with await pyathena.aconnect(...) as conn:
+        >>> async with await pyathena.aio_connect(...) as conn:
         ...     cursor = conn.cursor(AioArrowCursor)
         ...     await cursor.execute("SELECT * FROM my_table")
         ...     table = cursor.as_arrow()

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/aio/pandas/cursor.py

@@ -40,7 +40,7 @@ class AioPandasCursor(WithAsyncFetch):
     when ``chunksize`` is set, as fetch calls trigger lazy S3 reads.
 
     Example:
-        >>> async with await pyathena.aconnect(...) as conn:
+        >>> async with await pyathena.aio_connect(...) as conn:
         ...     cursor = conn.cursor(AioPandasCursor)
         ...     await cursor.execute("SELECT * FROM my_table")
         ...     df = cursor.as_pandas()

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/aio/polars/cursor.py

@@ -31,7 +31,7 @@ class AioPolarsCursor(WithAsyncFetch):
     when ``chunksize`` is set, as fetch calls trigger lazy S3 reads.
 
     Example:
-        >>> async with await pyathena.aconnect(...) as conn:
+        >>> async with await pyathena.aio_connect(...) as conn:
         ...     cursor = conn.cursor(AioPolarsCursor)
         ...     await cursor.execute("SELECT * FROM my_table")
         ...     df = cursor.as_polars()

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/aio/s3fs/cursor.py

@@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union, cast
 from pyathena.aio.common import WithAsyncFetch
 from pyathena.common import CursorIterator
 from pyathena.error import OperationalError, ProgrammingError
+from pyathena.filesystem.s3_async import AioS3FileSystem
 from pyathena.model import AthenaQueryExecution
 from pyathena.s3fs.converter import DefaultS3FSTypeConverter
 from pyathena.s3fs.result_set import AthenaS3FSResultSet, CSVReaderType
@@ -16,14 +17,15 @@ _logger = logging.getLogger(__name__)  # type: ignore
 
 
 class AioS3FSCursor(WithAsyncFetch):
-    """Native asyncio cursor that reads CSV results via S3FileSystem.
+    """Native asyncio cursor that reads CSV results via AioS3FileSystem.
 
-    Uses ``asyncio.to_thread()`` for result set creation and fetch operations
-    because ``AthenaS3FSResultSet`` lazily streams rows from S3 via a CSV
-    reader, making fetch calls blocking I/O.
+    Uses ``AioS3FileSystem`` for S3 operations, which replaces
+    ``ThreadPoolExecutor`` parallelism with ``asyncio.gather`` +
+    ``asyncio.to_thread``. Fetch operations are wrapped in
+    ``asyncio.to_thread()`` because CSV reading is blocking I/O.
 
     Example:
-        >>> async with await pyathena.aconnect(...) as conn:
+        >>> async with await pyathena.aio_connect(...) as conn:
         ...     cursor = conn.cursor(AioS3FSCursor)
         ...     await cursor.execute("SELECT * FROM my_table")
         ...     row = await cursor.fetchone()
@@ -127,6 +129,7 @@ class AioS3FSCursor(WithAsyncFetch):
                 arraysize=self.arraysize,
                 retry_config=self._retry_config,
                 csv_reader=self._csv_reader,
+                filesystem_class=AioS3FileSystem,
                 **kwargs,
             )
         else:

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/aio/spark/cursor.py

@@ -33,7 +33,7 @@ class AioSparkCursor(SparkBaseCursor, WithCalculationExecution):
 
     Example:
         >>> import asyncio
-        >>> async with await pyathena.aconnect(
+        >>> async with await pyathena.aio_connect(
         ...     work_group="spark-workgroup",
         ...     cursor_class=AioSparkCursor,
         ... ) as conn:

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/filesystem/s3.py

@@ -1,13 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import annotations
 
-import itertools
 import logging
 import mimetypes
 import os.path
 import re
 from concurrent.futures import Future, as_completed
-from concurrent.futures.thread import ThreadPoolExecutor
 from copy import deepcopy
 from datetime import datetime
 from multiprocessing import cpu_count
@@ -23,6 +21,7 @@ from fsspec.spec import AbstractBufferedFile
 from fsspec.utils import tokenize
 
 import pyathena
+from pyathena.filesystem.s3_executor import S3Executor, S3ThreadPoolExecutor
 from pyathena.filesystem.s3_object import (
     S3CompleteMultipartUpload,
     S3MultipartUpload,
@@ -686,6 +685,20 @@ class S3FileSystem(AbstractFileSystem):
             **request,
         )
 
+    def _create_executor(self, max_workers: int) -> S3Executor:
+        """Create an executor strategy for parallel operations.
+
+        Subclasses can override to provide alternative execution strategies
+        (e.g., asyncio-based execution).
+
+        Args:
+            max_workers: Maximum number of parallel workers.
+
+        Returns:
+            An S3Executor instance.
+        """
+        return S3ThreadPoolExecutor(max_workers=max_workers)
+
     def _delete_objects(
         self, bucket: str, paths: List[str], max_workers: Optional[int] = None, **kwargs
     ) -> None:
@@ -703,7 +716,7 @@ class S3FileSystem(AbstractFileSystem):
                     object_.update({"VersionId": version_id})
                 delete_objects.append(object_)
 
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with self._create_executor(max_workers=max_workers) as executor:
             fs = []
             for delete in [
                 delete_objects[i : i + self.DELETE_OBJECTS_MAX_KEYS]
@@ -861,7 +874,7 @@ class S3FileSystem(AbstractFileSystem):
             **kwargs,
         )
         parts = []
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with self._create_executor(max_workers=max_workers) as executor:
             fs = [
                 executor.submit(
                     self._upload_part_copy,
@@ -1106,6 +1119,7 @@ class S3FileSystem(AbstractFileSystem):
             mode,
             version_id=None,
             max_workers=max_workers,
+            executor=self._create_executor(max_workers=max_workers),
             block_size=block_size,
             cache_type=cache_type,
             autocommit=autocommit,
@@ -1256,6 +1270,7 @@ class S3File(AbstractBufferedFile):
         mode: str = "rb",
         version_id: Optional[str] = None,
         max_workers: int = (cpu_count() or 1) * 5,
+        executor: Optional[S3Executor] = None,
         block_size: int = S3FileSystem.DEFAULT_BLOCK_SIZE,
         cache_type: str = "bytes",
         autocommit: bool = True,
@@ -1265,7 +1280,7 @@ class S3File(AbstractBufferedFile):
         **kwargs,
     ) -> None:
         self.max_workers = max_workers
-        self._executor = ThreadPoolExecutor(max_workers=max_workers)
+        self._executor: S3Executor = executor or S3ThreadPoolExecutor(max_workers=max_workers)
         self.s3_additional_kwargs = s3_additional_kwargs if s3_additional_kwargs else {}
 
         super().__init__(
@@ -1481,24 +1496,18 @@ class S3File(AbstractBufferedFile):
             start, end, max_workers=self.max_workers, worker_block_size=self.blocksize
         )
         if len(ranges) > 1:
-            object_ = self._merge_objects(
-                list(
-                    self._executor.map(
-                        lambda bucket, key, ranges, version_id, kwargs: self.fs._get_object(
-                            bucket=bucket,
-                            key=key,
-                            ranges=ranges,
-                            version_id=version_id,
-                            **kwargs,
-                        ),
-                        itertools.repeat(self.bucket),
-                        itertools.repeat(self.key),
-                        ranges,
-                        itertools.repeat(self.version_id),
-                        itertools.repeat(self.s3_additional_kwargs),
-                    )
+            futures = [
+                self._executor.submit(
+                    self.fs._get_object,
+                    bucket=self.bucket,
+                    key=self.key,
+                    ranges=r,
+                    version_id=self.version_id,
+                    **self.s3_additional_kwargs,
                 )
-            )
+                for r in ranges
+            ]
+            object_ = self._merge_objects([f.result() for f in as_completed(futures)])
         else:
             object_ = self.fs._get_object(
                 self.bucket,

pyathena-3.29.3/pyathena/filesystem/s3_async.py

@@ -0,0 +1,357 @@
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+import asyncio
+import logging
+from multiprocessing import cpu_count
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
+
+from fsspec.asyn import AsyncFileSystem
+from fsspec.callbacks import _DEFAULT_CALLBACK
+
+from pyathena.filesystem.s3 import S3File, S3FileSystem
+from pyathena.filesystem.s3_executor import S3AioExecutor
+from pyathena.filesystem.s3_object import S3Object
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from pyathena.connection import Connection
+
+_logger = logging.getLogger(__name__)
+
+
+class AioS3FileSystem(AsyncFileSystem):
+    """An async filesystem interface for Amazon S3 using fsspec's AsyncFileSystem.
+
+    This class wraps ``S3FileSystem`` to provide native asyncio support. Instead of
+    using ``ThreadPoolExecutor`` for parallel operations, it uses ``asyncio.gather``
+    with ``asyncio.to_thread`` for natural integration with the asyncio event loop.
+
+    The implementation uses composition: an internal ``S3FileSystem`` instance handles
+    all boto3 calls, while this class delegates to it via ``asyncio.to_thread()``.
+    This avoids diamond inheritance issues and keeps all boto3 logic in one place.
+
+    File handles created by ``_open`` use ``S3AioExecutor`` so that parallel
+    operations (range reads, multipart uploads) are dispatched via the event loop
+    instead of spawning additional threads.
+
+    Attributes:
+        _sync_fs: The internal synchronous S3FileSystem instance.
+
+    Example:
+        >>> from pyathena.filesystem.s3_async import AioS3FileSystem
+        >>> fs = AioS3FileSystem(asynchronous=True)
+        >>>
+        >>> # Use in async context
+        >>> files = await fs._ls('s3://my-bucket/data/')
+        >>>
+        >>> # Sync wrappers also available (auto-generated by fsspec)
+        >>> files = fs.ls('s3://my-bucket/data/')
+    """
+
+    # https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
+    DELETE_OBJECTS_MAX_KEYS: int = 1000
+
+    protocol = ("s3", "s3a")
+    mirror_sync_methods = True
+    async_impl = True
+    _extra_tokenize_attributes = ("default_block_size",)
+
+    def __init__(
+        self,
+        connection: Optional["Connection[Any]"] = None,
+        default_block_size: Optional[int] = None,
+        default_cache_type: Optional[str] = None,
+        max_workers: int = (cpu_count() or 1) * 5,
+        s3_additional_kwargs: Optional[Dict[str, Any]] = None,
+        asynchronous: bool = False,
+        loop: Optional[Any] = None,
+        batch_size: Optional[int] = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            asynchronous=asynchronous,
+            loop=loop,
+            batch_size=batch_size,
+            **kwargs,
+        )
+        self._sync_fs = S3FileSystem(
+            connection=connection,
+            default_block_size=default_block_size,
+            default_cache_type=default_cache_type,
+            max_workers=max_workers,
+            s3_additional_kwargs=s3_additional_kwargs,
+            **kwargs,
+        )
+        # Share dircache for cache coherence between async and sync instances
+        self.dircache = self._sync_fs.dircache
+
+    @staticmethod
+    def parse_path(path: str) -> Tuple[str, Optional[str], Optional[str]]:
+        return S3FileSystem.parse_path(path)
+
+    async def _info(self, path: str, **kwargs) -> S3Object:
+        return await asyncio.to_thread(self._sync_fs.info, path, **kwargs)
+
+    async def _ls(
+        self, path: str, detail: bool = False, **kwargs
+    ) -> Union[List[S3Object], List[str]]:
+        return await asyncio.to_thread(self._sync_fs.ls, path, detail=detail, **kwargs)
+
+    async def _cat_file(
+        self, path: str, start: Optional[int] = None, end: Optional[int] = None, **kwargs
+    ) -> bytes:
+        return await asyncio.to_thread(self._sync_fs.cat_file, path, start=start, end=end, **kwargs)
+
+    async def _exists(self, path: str, **kwargs) -> bool:
+        return await asyncio.to_thread(self._sync_fs.exists, path, **kwargs)
+
+    async def _rm_file(self, path: str, **kwargs) -> None:
+        await asyncio.to_thread(self._sync_fs.rm_file, path, **kwargs)
+
+    async def _pipe_file(self, path: str, value: bytes, **kwargs) -> None:
+        await asyncio.to_thread(self._sync_fs.pipe_file, path, value, **kwargs)
+
+    async def _put_file(self, lpath: str, rpath: str, callback=_DEFAULT_CALLBACK, **kwargs) -> None:
+        await asyncio.to_thread(self._sync_fs.put_file, lpath, rpath, callback=callback, **kwargs)
+
+    async def _get_file(self, rpath: str, lpath: str, callback=_DEFAULT_CALLBACK, **kwargs) -> None:
+        await asyncio.to_thread(self._sync_fs.get_file, rpath, lpath, callback=callback, **kwargs)
+
+    async def _mkdir(self, path: str, create_parents: bool = True, **kwargs) -> None:
+        await asyncio.to_thread(self._sync_fs.mkdir, path, create_parents=create_parents, **kwargs)
+
+    async def _makedirs(self, path: str, exist_ok: bool = False) -> None:
+        await asyncio.to_thread(self._sync_fs.makedirs, path, exist_ok=exist_ok)
+
+    async def _rm(self, path: Union[str, List[str]], recursive: bool = False, **kwargs) -> None:
+        """Remove files or directories using async parallel batch deletion.
+
+        For multiple paths, chunks into batches of 1000 (S3 API limit) and uses
+        ``asyncio.gather`` with ``asyncio.to_thread`` instead of ThreadPoolExecutor.
+        """
+        if isinstance(path, str):
+            path = [path]
+
+        bucket, _, _ = self.parse_path(path[0])
+
+        expand_paths: List[str] = []
+        for p in path:
+            expanded = await asyncio.to_thread(self._sync_fs.expand_path, p, recursive=recursive)
+            expand_paths.extend(expanded)
+
+        if not expand_paths:
+            return
+
+        quiet = kwargs.pop("Quiet", True)
+        delete_objects: List[Dict[str, Any]] = []
+        for p in expand_paths:
+            _, key, version_id = self.parse_path(p)
+            if key:
+                object_: Dict[str, Any] = {"Key": key}
+                if version_id:
+                    object_["VersionId"] = version_id
+                delete_objects.append(object_)
+
+        if not delete_objects:
+            return
+
+        chunks = [
+            delete_objects[i : i + self.DELETE_OBJECTS_MAX_KEYS]
+            for i in range(0, len(delete_objects), self.DELETE_OBJECTS_MAX_KEYS)
+        ]
+
+        async def _delete_chunk(chunk: List[Dict[str, Any]]) -> None:
+            request = {
+                "Bucket": bucket,
+                "Delete": {
+                    "Objects": chunk,
+                    "Quiet": quiet,
+                },
+            }
+            await asyncio.to_thread(
+                self._sync_fs._call, self._sync_fs._client.delete_objects, **request
+            )
+
+        await asyncio.gather(*[_delete_chunk(chunk) for chunk in chunks])
+
+        for p in expand_paths:
+            self._sync_fs.invalidate_cache(p)
+
+    async def _cp_file(self, path1: str, path2: str, **kwargs) -> None:
+        """Copy an S3 object, using async parallel multipart upload for large files."""
+        kwargs.pop("onerror", None)
+        bucket1, key1, version_id1 = self.parse_path(path1)
+        bucket2, key2, version_id2 = self.parse_path(path2)
+        if version_id2:
+            raise ValueError("Cannot copy to a versioned file.")
+        if not key1 or not key2:
+            raise ValueError("Cannot copy buckets.")
+
+        info1 = await self._info(path1)
+        size1 = info1.get("size", 0)
+        if size1 <= S3FileSystem.MULTIPART_UPLOAD_MAX_PART_SIZE:
+            await asyncio.to_thread(
+                self._sync_fs._copy_object,
+                bucket1=bucket1,
+                key1=key1,
+                version_id1=version_id1,
+                bucket2=bucket2,
+                key2=key2,
+                **kwargs,
+            )
+        else:
+            await self._copy_object_with_multipart_upload(
+                bucket1=bucket1,
+                key1=key1,
+                version_id1=version_id1,
+                size1=size1,
+                bucket2=bucket2,
+                key2=key2,
+                **kwargs,
+            )
+        self._sync_fs.invalidate_cache(path2)
+
+    async def _copy_object_with_multipart_upload(
+        self,
+        bucket1: str,
+        key1: str,
+        size1: int,
+        bucket2: str,
+        key2: str,
+        block_size: Optional[int] = None,
+        version_id1: Optional[str] = None,
+        **kwargs,
+    ) -> None:
+        block_size = block_size if block_size else S3FileSystem.MULTIPART_UPLOAD_MAX_PART_SIZE
+        if (
+            block_size < S3FileSystem.MULTIPART_UPLOAD_MIN_PART_SIZE
+            or block_size > S3FileSystem.MULTIPART_UPLOAD_MAX_PART_SIZE
+        ):
+            raise ValueError("Block size must be greater than 5MiB and less than 5GiB.")
+
+        copy_source: Dict[str, Any] = {
+            "Bucket": bucket1,
+            "Key": key1,
+        }
+        if version_id1:
+            copy_source["VersionId"] = version_id1
+
+        ranges = S3File._get_ranges(
+            0,
+            size1,
+            self._sync_fs.max_workers,
+            block_size,
+        )
+        multipart_upload = await asyncio.to_thread(
+            self._sync_fs._create_multipart_upload,
+            bucket=bucket2,
+            key=key2,
+            **kwargs,
+        )
+
+        async def _upload_part(i: int, range_: Tuple[int, int]) -> Dict[str, Any]:
+            result = await asyncio.to_thread(
+                self._sync_fs._upload_part_copy,
+                bucket=bucket2,
+                key=key2,
+                copy_source=copy_source,
+                upload_id=cast(str, multipart_upload.upload_id),
+                part_number=i + 1,
+                copy_source_ranges=range_,
+            )
+            return {
+                "ETag": result.etag,
+                "PartNumber": result.part_number,
+            }
+
+        parts = await asyncio.gather(*[_upload_part(i, r) for i, r in enumerate(ranges)])
+        parts_list = sorted(parts, key=lambda x: x["PartNumber"])
+
+        await asyncio.to_thread(
+            self._sync_fs._complete_multipart_upload,
+            bucket=bucket2,
+            key=key2,
+            upload_id=cast(str, multipart_upload.upload_id),
+            parts=parts_list,
+        )
+
+    async def _find(
+        self,
+        path: str,
+        maxdepth: Optional[int] = None,
+        withdirs: bool = False,
+        **kwargs,
+    ) -> Union[Dict[str, S3Object], List[str]]:
+        detail = kwargs.pop("detail", False)
+        files = await asyncio.to_thread(
+            self._sync_fs._find, path, maxdepth=maxdepth, withdirs=withdirs, **kwargs
+        )
+        if detail:
+            return {f.name: f for f in files}
+        return [f.name for f in files]
+
+    def _open(
+        self,
+        path: str,
+        mode: str = "rb",
+        block_size: Optional[int] = None,
+        cache_type: Optional[str] = None,
+        autocommit: bool = True,
+        cache_options: Optional[Dict[Any, Any]] = None,
+        **kwargs,
+    ) -> "AioS3File":
+        if block_size is None:
+            block_size = self._sync_fs.default_block_size
+        if cache_type is None:
+            cache_type = self._sync_fs.default_cache_type
+        max_workers = kwargs.pop("max_worker", self._sync_fs.max_workers)
+        s3_additional_kwargs = kwargs.pop("s3_additional_kwargs", {})
+        s3_additional_kwargs.update(self._sync_fs.s3_additional_kwargs)
+
+        return AioS3File(
+            self._sync_fs,
+            path,
+            mode,
+            version_id=None,
+            max_workers=max_workers,
+            executor=S3AioExecutor(loop=self._loop),
+            block_size=block_size,
+            cache_type=cache_type,
+            autocommit=autocommit,
+            cache_options=cache_options,
+            s3_additional_kwargs=s3_additional_kwargs,
+            **kwargs,
+        )
+
+    def sign(self, path: str, expiration: int = 3600, **kwargs) -> str:
+        return cast(str, self._sync_fs.sign(path, expiration=expiration, **kwargs))
+
+    def checksum(self, path: str, **kwargs) -> int:
+        return cast(int, self._sync_fs.checksum(path, **kwargs))
+
+    def created(self, path: str) -> "datetime":
+        return self._sync_fs.created(path)
+
+    def modified(self, path: str) -> "datetime":
+        return self._sync_fs.modified(path)
+
+    def invalidate_cache(self, path: Optional[str] = None) -> None:
+        self._sync_fs.invalidate_cache(path)
+
+    async def _touch(self, path: str, truncate: bool = True, **kwargs) -> None:
+        await asyncio.to_thread(self._sync_fs.touch, path, truncate=truncate, **kwargs)
+
+
+class AioS3File(S3File):
+    """Async-aware S3 file handle using ``S3AioExecutor``.
+
+    Functionally identical to ``S3File``; exists as a distinct type for
+    ``isinstance`` checks and to document the async execution model.
+    All parallel operations (range reads, multipart uploads) are dispatched
+    through the ``S3Executor`` interface — the ``S3AioExecutor``
+    provided by ``AioS3FileSystem`` uses the event loop instead of threads.
+    """
+
+    pass

pyathena-3.29.3/pyathena/filesystem/s3_executor.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+import asyncio
+from abc import ABCMeta, abstractmethod
+from concurrent.futures import Future
+from concurrent.futures.thread import ThreadPoolExecutor
+from typing import Any, Callable, Optional, TypeVar
+
+T = TypeVar("T")
+
+
+class S3Executor(metaclass=ABCMeta):
+    """Abstract executor for parallel S3 operations.
+
+    Defines the interface used by ``S3File`` and ``S3FileSystem`` for submitting
+    work to run in parallel and for shutting down the executor when done.
+    Both ``submit`` and ``shutdown`` mirror the ``concurrent.futures.Executor``
+    interface so that ``as_completed()`` and ``Future.cancel()`` work unchanged.
+    """
+
+    @abstractmethod
+    def submit(self, fn: Callable[..., T], *args: Any, **kwargs: Any) -> Future[T]:
+        """Submit a callable for execution and return a Future."""
+        ...
+
+    @abstractmethod
+    def shutdown(self, wait: bool = True) -> None:
+        """Shut down the executor, freeing any resources."""
+        ...
+
+    def __enter__(self) -> "S3Executor":
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.shutdown(wait=True)
+
+
+class S3ThreadPoolExecutor(S3Executor):
+    """Executor that delegates to a ``ThreadPoolExecutor``.
+
+    This is the default executor used by ``S3File`` and ``S3FileSystem``
+    for synchronous parallel operations.
+    """
+
+    def __init__(self, max_workers: int) -> None:
+        self._executor = ThreadPoolExecutor(max_workers=max_workers)
+
+    def submit(self, fn: Callable[..., T], *args: Any, **kwargs: Any) -> Future[T]:
+        return self._executor.submit(fn, *args, **kwargs)
+
+    def shutdown(self, wait: bool = True) -> None:
+        self._executor.shutdown(wait=wait)
+
+
+class S3AioExecutor(S3Executor):
+    """Executor that schedules work on an asyncio event loop.
+
+    Uses ``asyncio.run_coroutine_threadsafe(asyncio.to_thread(fn), loop)`` to
+    dispatch blocking functions onto the event loop's thread pool, returning
+    ``concurrent.futures.Future`` objects that are compatible with
+    ``as_completed()`` and ``Future.cancel()``.
+
+    This avoids thread-in-thread nesting when ``S3File`` is used from within
+    ``asyncio.to_thread()`` calls (the pattern used by ``AioS3FileSystem``).
+
+    Args:
+        loop: A running asyncio event loop.
+
+    Raises:
+        RuntimeError: If the event loop is not running when ``submit`` is called.
+    """
+
+    def __init__(self, loop: Optional[asyncio.AbstractEventLoop] = None) -> None:
+        self._loop = loop
+
+    def submit(self, fn: Callable[..., T], *args: Any, **kwargs: Any) -> Future[T]:
+        if self._loop is not None and self._loop.is_running():
+            return asyncio.run_coroutine_threadsafe(
+                asyncio.to_thread(fn, *args, **kwargs), self._loop
+            )
+        raise RuntimeError(
+            "S3AioExecutor requires a running event loop. "
+            "Use S3ThreadPoolExecutor for synchronous usage."
+        )
+
+    def shutdown(self, wait: bool = True) -> None:
+        # No resources to release — work is dispatched to the event loop.
+        pass

{pyathena-3.29.1 → pyathena-3.29.3}/pyathena/s3fs/result_set.py

@@ -5,6 +5,8 @@ import logging
 from io import TextIOWrapper
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
 
+from fsspec import AbstractFileSystem
+
 from pyathena.converter import Converter
 from pyathena.error import OperationalError, ProgrammingError
 from pyathena.filesystem.s3 import S3FileSystem
@@ -62,6 +64,7 @@ class AthenaS3FSResultSet(AthenaResultSet):
         retry_config: RetryConfig,
         block_size: Optional[int] = None,
         csv_reader: Optional[CSVReaderType] = None,
+        filesystem_class: Optional[Type[AbstractFileSystem]] = None,
         **kwargs,
     ) -> None:
         super().__init__(
@@ -77,6 +80,7 @@ class AthenaS3FSResultSet(AthenaResultSet):
         self._arraysize = arraysize
         self._block_size = block_size if block_size else self.DEFAULT_BLOCK_SIZE
         self._csv_reader_class: CSVReaderType = csv_reader or AthenaCSVReader
+        self._filesystem_class: Type[AbstractFileSystem] = filesystem_class or S3FileSystem
         self._fs = self._create_s3_file_system()
         self._csv_reader: Optional[Any] = None
 
@@ -92,9 +96,9 @@ class AthenaS3FSResultSet(AthenaResultSet):
         if not self._csv_reader and not self._rows and pre_fetched_rows:
             self._rows.extend(pre_fetched_rows)
 
-    def _create_s3_file_system(self) -> S3FileSystem:
+    def _create_s3_file_system(self) -> AbstractFileSystem:
         """Create S3FileSystem using connection settings."""
-        return S3FileSystem(
+        return self._filesystem_class(
             connection=self.connection,
             default_block_size=self._block_size,
         )

{pyathena-3.29.1 → pyathena-3.29.3}/pyproject.toml

@@ -178,7 +178,7 @@ exclude = [
 legacy_tox_ini = """
 [tox]
 isolated_build = true
-envlist = py{310,311,312,313,314}
+envlist = py{310,311,312,313,314}-{pyathena,sqla,sqla_async}
 
 [gh-actions]
 python =
@@ -188,6 +188,12 @@ python =
     3.13: py313
     3.14: py314
 
+[gh-actions:env]
+TEST_TYPE =
+    pyathena: pyathena
+    sqla: sqla
+    sqla_async: sqla_async
+
 [testenv]
 allowlist_externals =
     uv
@@ -195,9 +201,9 @@ allowlist_externals =
     make
 commands =
     uv sync --group dev
-    make test
-    make test-sqla
-    make test-sqla-async
+    pyathena: make test
+    sqla: make test-sqla
+    sqla_async: make test-sqla-async
 passenv =
     TOXENV
     AWS_*