tracktolib 0.65.1__tar.gz → 0.66.2__tar.gz

{tracktolib-0.65.1 → tracktolib-0.66.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracktolib
-Version: 0.65.1
+Version: 0.66.2
 Summary: Utility library for python
 Keywords: utility
 Author-email: julien.brayere@tracktor.fr
@@ -14,13 +14,15 @@ Requires-Dist: fastapi>=0.103.2 ; extra == 'api'
 Requires-Dist: pydantic>=2 ; extra == 'api'
 Requires-Dist: httpx>=0.25.0 ; extra == 'http'
 Requires-Dist: python-json-logger>=3.2.1 ; extra == 'logs'
-Requires-Dist: niquests>=3.15.2 ; extra == 'notion'
+Requires-Dist: niquests>=3.17.0 ; extra == 'notion'
 Requires-Dist: asyncpg>=0.27.0 ; extra == 'pg'
 Requires-Dist: rich>=13.6.0 ; extra == 'pg'
 Requires-Dist: psycopg>=3.1.12 ; extra == 'pg-sync'
 Requires-Dist: aiobotocore>=2.9.0 ; extra == 's3'
 Requires-Dist: minio>=7.2.0 ; extra == 's3-minio'
 Requires-Dist: pycryptodome>=3.20.0 ; extra == 's3-minio'
+Requires-Dist: botocore>=1.35.36 ; extra == 's3-niquests'
+Requires-Dist: niquests>=3.17.0 ; extra == 's3-niquests'
 Requires-Dist: deepdiff>=8.1.0 ; extra == 'tests'
 Requires-Python: >=3.12, <4.0
 Provides-Extra: api
@@ -31,6 +33,7 @@ Provides-Extra: pg
 Provides-Extra: pg-sync
 Provides-Extra: s3
 Provides-Extra: s3-minio
+Provides-Extra: s3-niquests
 Provides-Extra: tests
 Description-Content-Type: text/markdown
 
@@ -38,9 +41,9 @@ Description-Content-Type: text/markdown
 
 [![Python versions](https://img.shields.io/pypi/pyversions/tracktolib)](https://pypi.python.org/pypi/tracktolib)
 [![Latest PyPI version](https://img.shields.io/pypi/v/tracktolib?logo=pypi)](https://pypi.python.org/pypi/tracktolib)
-[![CircleCI](https://circleci.com/gh/Tracktor/tracktolib/tree/master.svg?style=shield)](https://app.circleci.com/pipelines/github/Tracktor/tracktolib?branch=master)
+[![CI](https://github.com/Tracktor/tracktolib/actions/workflows/ci.yml/badge.svg)](https://github.com/Tracktor/tracktolib/actions/workflows/ci.yml)
 
-Utility library for Python 3.12+
+Tracktor Swiss-knife Utility library.
 
 ## Installation
 
@@ -121,7 +124,15 @@ S3 helpers using [minio](https://min.io/docs/minio/linux/developers/python/API.h
 uv add tracktolib[s3-minio]
 ```
 
-### http
+### s3-niquests
+
+Async S3 helpers using [niquests](https://github.com/jawah/niquests) and [botocore](https://github.com/boto/botocore).
+
+```bash
+uv add tracktolib[s3-niquests]
+```
+
+### http (deprecated)
 
 HTTP client helpers using [httpx](https://www.python-httpx.org/).
 
@@ -151,4 +162,4 @@ Testing utilities using [deepdiff](https://github.com/seperman/deepdiff).
 
 ```bash
 uv add tracktolib[tests]
-```
+```

{tracktolib-0.65.1 → tracktolib-0.66.2}/README.md

@@ -2,9 +2,9 @@
 
 [![Python versions](https://img.shields.io/pypi/pyversions/tracktolib)](https://pypi.python.org/pypi/tracktolib)
 [![Latest PyPI version](https://img.shields.io/pypi/v/tracktolib?logo=pypi)](https://pypi.python.org/pypi/tracktolib)
-[![CircleCI](https://circleci.com/gh/Tracktor/tracktolib/tree/master.svg?style=shield)](https://app.circleci.com/pipelines/github/Tracktor/tracktolib?branch=master)
+[![CI](https://github.com/Tracktor/tracktolib/actions/workflows/ci.yml/badge.svg)](https://github.com/Tracktor/tracktolib/actions/workflows/ci.yml)
 
-Utility library for Python 3.12+
+Tracktor Swiss-knife Utility library.
 
 ## Installation
 
@@ -85,7 +85,15 @@ S3 helpers using [minio](https://min.io/docs/minio/linux/developers/python/API.h
 uv add tracktolib[s3-minio]
 ```
 
-### http
+### s3-niquests
+
+Async S3 helpers using [niquests](https://github.com/jawah/niquests) and [botocore](https://github.com/boto/botocore).
+
+```bash
+uv add tracktolib[s3-niquests]
+```
+
+### http (deprecated)
 
 HTTP client helpers using [httpx](https://www.python-httpx.org/).
 
@@ -115,4 +123,4 @@ Testing utilities using [deepdiff](https://github.com/seperman/deepdiff).
 
 ```bash
 uv add tracktolib[tests]
-```
+```

{tracktolib-0.65.1 → tracktolib-0.66.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tracktolib"
-version = "0.65.1"
+version = "0.66.2"
 authors = [
     { email = "julien.brayere@tracktor.fr" }
 ]
@@ -21,6 +21,7 @@ include = [
     "LICENSE"
 ]
 requires-python = ">=3.12,<4.0"
+dependencies = []
 
 [project.optional-dependencies]
 logs = ["python-json-logger>=3.2.1"]
@@ -31,6 +32,10 @@ s3-minio = [
     "pycryptodome>=3.20.0",
 ]
 s3 = ["aiobotocore>=2.9.0"]
+s3-niquests = [
+    "botocore>=1.35.36",
+    "niquests>=3.17.0",
+]
 tests = ["deepdiff>=8.1.0"]
 http = ["httpx>=0.25.0"]
 api = [
@@ -42,7 +47,7 @@ pg = [
     "rich>=13.6.0",
 ]
 notion = [
-    "niquests>=3.15.2"
+    "niquests>=3.17.0"
 ]
 
 [dependency-groups]
@@ -60,14 +65,21 @@ dev = [
     "minio>=7.2.7",
     "pycryptodome>=3.23.0",
     "aiobotocore==2.15.2",
+    # niquests for s3-niquests tests (botocore comes from aiobotocore)
+    "niquests>=3.17.0",
     "python-json-logger>=3.2.0",
-    "pre-commit>=4.4.0",
+    "prek>=0.2.30",
 ]
 
 bump = [
     "commitizen>=4.10.0"
 ]
 
+docs = [
+    "mkdocs>=1.6.0",
+    "mkdocs-material>=9.5.0",
+]
+
 [build-system]
 requires = ["uv_build>=0.9.4,<0.10.0"]
 build-backend = "uv_build"
@@ -96,7 +108,7 @@ pythonPlatform = "Linux"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.65.1"
+version = "0.66.2"
 tag_format = "$version"
 version_files = [
     "pyproject.toml:version"

{tracktolib-0.65.1 → tracktolib-0.66.2}/tracktolib/api.py

@@ -4,7 +4,6 @@ from collections.abc import Mapping
 from dataclasses import field, dataclass
 from inspect import getdoc
 from typing import (
-    TypeVar,
     Callable,
     Any,
     Literal,
@@ -31,11 +30,9 @@ try:
 except ImportError:
     raise ImportError('Please install fastapi, pydantic or tracktolib with "api" to use this module')
 
-D = TypeVar("D")
-
 
 # noqa: N802
-def Depends(
+def Depends[D](
     dependency: Callable[
         ...,
         Coroutine[Any, Any, D] | Coroutine[Any, Any, D | None] | AsyncIterator[D] | D,
@@ -48,20 +45,19 @@ def Depends(
     return params.Depends(dependency, use_cache=use_cache)  # pyright: ignore [reportReturnType]
 
 
-B = TypeVar("B", bound=BaseModel | None | Sequence[BaseModel])
-
-Response = Mapping | Sequence[Mapping] | B
+type _BaseModelBound = BaseModel | None | Sequence[BaseModel]
+type Response[B: _BaseModelBound] = Mapping | Sequence[Mapping] | B
 
 Method = Literal["GET", "POST", "DELETE", "PATCH", "PUT"]
 
-EnpointFn = Callable[..., Response]
+type EnpointFn[B: _BaseModelBound] = Callable[..., Response[B]]
 
 Dependencies: TypeAlias = Sequence[params.Depends] | None
 StatusCode: TypeAlias = int | None
 
 
 class MethodMeta(TypedDict):
-    fn: EnpointFn
+    fn: EnpointFn[Any]
     status_code: StatusCode
     dependencies: Dependencies
     path: str | None
@@ -81,7 +77,7 @@ class Endpoint:
     def methods(self):
         return self._methods
 
-    def get(
+    def get[B: _BaseModelBound](
         self,
         status_code: StatusCode = None,
         dependencies: Dependencies = None,
@@ -107,7 +103,7 @@ class Endpoint:
             deprecated=deprecated,
         )
 
-    def post(
+    def post[B: _BaseModelBound](
         self,
         *,
         status_code: StatusCode = None,
@@ -134,7 +130,7 @@ class Endpoint:
             deprecated=deprecated,
         )
 
-    def put(
+    def put[B: _BaseModelBound](
         self,
         status_code: StatusCode = None,
         dependencies: Dependencies = None,
@@ -160,7 +156,7 @@ class Endpoint:
             deprecated=deprecated,
         )
 
-    def delete(
+    def delete[B: _BaseModelBound](
         self,
         status_code: StatusCode = None,
         dependencies: Dependencies = None,
@@ -186,7 +182,7 @@ class Endpoint:
             deprecated=deprecated,
         )
 
-    def patch(
+    def patch[B: _BaseModelBound](
         self,
         status_code: StatusCode = None,
         dependencies: Dependencies = None,
@@ -213,7 +209,7 @@ class Endpoint:
         )
 
 
-def _get_method_wrapper(
+def _get_method_wrapper[B: _BaseModelBound](
     cls: Endpoint,
     method: Method,
     *,
@@ -339,7 +335,7 @@ def check_status(resp, status: int = starlette.status.HTTP_200_OK):
         raise AssertionError(json.dumps(resp.json(), indent=4))
 
 
-def generate_list_name_model(model: Type[B], status: int | None = None) -> dict:
+def generate_list_name_model[B: _BaseModelBound](model: Type[B], status: int | None = None) -> dict:
     _status = "200" if status is None else str(status)
     if get_origin(model) and get_origin(model) is list:
         _title = f"Array[{get_args(model)[0].__name__}]"

{tracktolib-0.65.1 → tracktolib-0.66.2}/tracktolib/http_utils.py

@@ -1,7 +1,14 @@
 import typing
+import warnings
 from io import TextIOWrapper, BufferedWriter
 from typing import BinaryIO, Callable, TextIO
 
+warnings.warn(
+    "tracktolib.http_utils is deprecated and will be removed in a future version.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
 try:
     import httpx
     from httpx._types import QueryParamTypes

{tracktolib-0.65.1 → tracktolib-0.66.2}/tracktolib/notion/fetch.py

@@ -33,6 +33,11 @@ def _use_data_source_api(api_version: str) -> bool:
     return api_version >= "2025-09-03"
 
 
+def _get_api_version(session: niquests.Session, api_version: str | None) -> str:
+    """Get API version from parameter or session headers."""
+    return api_version or str(session.headers.get("Notion-Version", DEFAULT_API_VERSION))
+
+
 __all__ = (
     # Auth helpers
     "get_notion_headers",
@@ -223,7 +228,7 @@ async def create_page(
     For older API versions, parent should use {"database_id": "..."}.
     The function will automatically convert between the two formats.
     """
-    _api_version = api_version or session.headers.get("Notion-Version", DEFAULT_API_VERSION)
+    _api_version = _get_api_version(session, api_version)
     converted_parent = _convert_parent_for_api_version(parent, _api_version)
     payload: dict[str, Any] = {
         "parent": converted_parent,
@@ -280,7 +285,7 @@ async def fetch_database(
     For API version 2025-09-03+, uses /v1/data_sources/{id} endpoint.
     For older API versions, uses /v1/databases/{id} endpoint.
     """
-    _api_version = api_version or session.headers.get("Notion-Version", DEFAULT_API_VERSION)
+    _api_version = _get_api_version(session, api_version)
     if _use_data_source_api(_api_version):
         endpoint = f"{NOTION_API_URL}/v1/data_sources/{database_id}"
     else:
@@ -306,7 +311,7 @@ async def query_database(
     For API version 2025-09-03+, uses /v1/data_sources/{id}/query endpoint.
     For older API versions, uses /v1/databases/{id}/query endpoint.
     """
-    _api_version = api_version or session.headers.get("Notion-Version", DEFAULT_API_VERSION)
+    _api_version = _get_api_version(session, api_version)
     payload: dict[str, Any] = {}
     if filter:
         payload["filter"] = filter
@@ -405,7 +410,7 @@ async def fetch_search(
     converted to 'data_source'. For older versions, 'data_source' is
     converted to 'database'.
     """
-    _api_version = api_version or session.headers.get("Notion-Version", DEFAULT_API_VERSION)
+    _api_version = _get_api_version(session, api_version)
     payload: dict[str, Any] = {}
     if query:
         payload["query"] = query

{tracktolib-0.65.1 → tracktolib-0.66.2}/tracktolib/pg/query.py

@@ -1,6 +1,6 @@
 import typing
 from dataclasses import dataclass, field
-from typing import TypeVar, Iterable, Callable, Generic, Iterator, TypeAlias, overload, Any, Literal
+from typing import Iterable, Callable, Iterator, TypeAlias, overload, Any, Literal
 
 from ..pg_utils import get_conflict_query
 
@@ -11,21 +11,18 @@ except ImportError:
 
 from tracktolib.utils import fill_dict
 
-K = TypeVar("K", bound=str)
-V = TypeVar("V")
 
-
-def _get_insert_query(table: str, columns: Iterable[K], values: str) -> str:
+def _get_insert_query[K: str](table: str, columns: Iterable[K], values: str) -> str:
     _columns = ", ".join(columns)
     return f"INSERT INTO {table} AS t ({_columns}) VALUES ({values})"
 
 
-def _get_returning_query(query: str, returning: Iterable[K]) -> str:
+def _get_returning_query[K: str](query: str, returning: Iterable[K]) -> str:
     _returning = ", ".join(returning)
     return f"{query} RETURNING {_returning}"
 
 
-def _get_on_conflict_query(
+def _get_on_conflict_query[K: str](
     query: str,
     columns: Iterable[K],
     update_columns: Iterable[K] | None,
@@ -47,14 +44,14 @@ def _get_on_conflict_query(
     return f"{query} {_on_conflict}"
 
 
-ReturningFn = Callable[[Iterable[K] | None, K | None], None]
-ConflictFn = Callable[[Iterable[K] | None, Iterable[K] | None, str | None], None]
+type ReturningFn[K: str] = Callable[[Iterable[K] | None, K | None], None]
+type ConflictFn[K: str] = Callable[[Iterable[K] | None, Iterable[K] | None, str | None], None]
 
-_Connection = asyncpg.Connection | asyncpg.pool.Pool
+type _Connection = asyncpg.Connection | asyncpg.pool.Pool
 
 
 @dataclass
-class PGReturningQuery(Generic[K]):
+class PGReturningQuery[K: str]:
     returning_ids: Iterable[K] | None = None
     query: str | None = None
 
@@ -69,7 +66,7 @@ class PGReturningQuery(Generic[K]):
 
 
 @dataclass
-class PGConflictQuery(Generic[K]):
+class PGConflictQuery[K: str]:
     keys: Iterable[K] | None = None
     ignore_keys: Iterable[K] | None = None
     query: str | None = None
@@ -86,7 +83,7 @@ class PGConflictQuery(Generic[K]):
 
 
 @dataclass
-class PGQuery(Generic[K, V]):
+class PGQuery[K: str, V]:
     table: str
     items: list[dict[K, V]]
 
@@ -328,7 +325,7 @@ class PGUpdateQuery(PGQuery):
 OnConflict: TypeAlias = PGConflictQuery | str
 
 
-def insert_pg(
+def insert_pg[K: str](
     table: str,
     items: list[dict],
     *,
@@ -344,8 +341,7 @@ def insert_pg(
     )
 
 
-Q = TypeVar("Q", bound=PGInsertQuery | PGUpdateQuery)
-QueryCallback = Callable[[Q], None]
+type QueryCallback[Q: PGInsertQuery | PGUpdateQuery] = Callable[[Q], None]
 
 
 async def insert_one(
@@ -488,7 +484,7 @@ async def fetch_count(conn: _Connection, query: str, *args) -> int:
     return typing.cast(int, c)
 
 
-def Conflict(
+def Conflict[K: str](
     keys: Iterable[K],
     ignore_keys: Iterable[K] | None = None,
 ) -> PGConflictQuery:

{tracktolib-0.65.1 → tracktolib-0.66.2}/tracktolib/pg_sync.py

@@ -47,18 +47,18 @@ def fetch_count(engine: Connection, table: str, *args, where: str | None = None)
 
 
 @overload
-def fetch_one(engine: Connection, query: Query, *args, required: Literal[False]) -> dict | None: ...
+def fetch_one(engine: Connection, query: LiteralString, *args, required: Literal[False]) -> dict | None: ...
 
 
 @overload
-def fetch_one(engine: Connection, query: Query, *args, required: Literal[True]) -> dict: ...
+def fetch_one(engine: Connection, query: LiteralString, *args, required: Literal[True]) -> dict: ...
 
 
 @overload
-def fetch_one(engine: Connection, query: Query, *args) -> dict | None: ...
+def fetch_one(engine: Connection, query: LiteralString, *args) -> dict | None: ...
 
 
-def fetch_one(engine: Connection, query: Query, *args, required: bool = False) -> dict | None:
+def fetch_one(engine: Connection, query: LiteralString, *args, required: bool = False) -> dict | None:
     with engine.cursor(row_factory=dict_row) as cur:
         _data = cur.execute(query, args).fetchone()
     engine.commit()

tracktolib-0.66.2/tracktolib/s3/__init__.py

@@ -0,0 +1 @@
+

tracktolib-0.66.2/tracktolib/s3/niquests.py

@@ -0,0 +1,669 @@
+from __future__ import annotations
+
+from collections import namedtuple
+from pathlib import Path
+
+import http
+import xml.etree.ElementTree as ET
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from typing import AsyncIterator, Callable, Literal, Self, TypedDict, Unpack
+
+try:
+    import botocore.client
+    import botocore.session
+    import jmespath
+    from botocore.config import Config
+except ImportError as e:
+    raise ImportError("botocore is required for S3 operations. Install with tracktolib[s3-niquests]") from e
+
+try:
+    import niquests
+except ImportError as e:
+    raise ImportError("niquests is required for S3 operations. Install with tracktolib[s3-niquests]") from e
+
+from ..utils import get_stream_chunk
+
+__all__ = (
+    "S3Session",
+    "s3_delete_object",
+    "s3_delete_objects",
+    "s3_list_files",
+    "s3_put_object",
+    "s3_get_object",
+    "s3_download_file",
+    "s3_upload_file",
+    "s3_create_multipart_upload",
+    "s3_multipart_upload",
+    "s3_file_upload",
+    "S3MultipartUpload",
+    "S3Object",
+    "S3ObjectParams",
+    "UploadPart",
+    "build_s3_headers",
+    "build_s3_presigned_params",
+)
+
+ACL = Literal[
+    "private",
+    "public-read",
+    "public-read-write",
+    "authenticated-read",
+    "aws-exec-read",
+    "bucket-owner-read",
+    "bucket-owner-full-control",
+]
+
+StorageClass = Literal[
+    "STANDARD",
+    "REDUCED_REDUNDANCY",
+    "STANDARD_IA",
+    "ONEZONE_IA",
+    "INTELLIGENT_TIERING",
+    "GLACIER",
+    "DEEP_ARCHIVE",
+    "OUTPOSTS",
+    "GLACIER_IR",
+    "EXPRESS_ONEZONE",
+]
+
+ServerSideEncryption = Literal["AES256", "aws:kms", "aws:kms:dsse"]
+
+
+class S3ObjectParams(TypedDict, total=False):
+    """
+    Parameters for S3 object uploads (PutObject, CreateMultipartUpload).
+
+    See:
+    - https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
+    - https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html
+    """
+
+    acl: ACL | None
+    content_type: str | None
+    content_disposition: str | None
+    content_encoding: str | None
+    content_language: str | None
+    cache_control: str | None
+    storage_class: StorageClass | None
+    server_side_encryption: ServerSideEncryption | None
+    sse_kms_key_id: str | None
+    tagging: str | None  # URL-encoded key=value pairs
+    metadata: dict[str, str] | None  # User-defined metadata (x-amz-meta-*)
+
+
+def build_s3_headers(params: S3ObjectParams) -> dict[str, str]:
+    """
+    Build S3 request headers from S3ObjectParams.
+
+    Returns a dict of HTTP headers to include in the request.
+    """
+    headers: dict[str, str] = {}
+
+    if (acl := params.get("acl")) is not None:
+        headers["x-amz-acl"] = acl
+    if (content_type := params.get("content_type")) is not None:
+        headers["Content-Type"] = content_type
+    if (content_disposition := params.get("content_disposition")) is not None:
+        headers["Content-Disposition"] = content_disposition
+    if (content_encoding := params.get("content_encoding")) is not None:
+        headers["Content-Encoding"] = content_encoding
+    if (content_language := params.get("content_language")) is not None:
+        headers["Content-Language"] = content_language
+    if (cache_control := params.get("cache_control")) is not None:
+        headers["Cache-Control"] = cache_control
+    if (storage_class := params.get("storage_class")) is not None:
+        headers["x-amz-storage-class"] = storage_class
+    if (sse := params.get("server_side_encryption")) is not None:
+        headers["x-amz-server-side-encryption"] = sse
+    if (sse_kms_key_id := params.get("sse_kms_key_id")) is not None:
+        headers["x-amz-server-side-encryption-aws-kms-key-id"] = sse_kms_key_id
+    if (tagging := params.get("tagging")) is not None:
+        headers["x-amz-tagging"] = tagging
+    if (metadata := params.get("metadata")) is not None:
+        for key, value in metadata.items():
+            headers[f"x-amz-meta-{key}"] = value
+
+    return headers
+
+
+def build_s3_presigned_params(bucket: str, key: str, params: S3ObjectParams) -> dict:
+    """
+    Build parameters dict for botocore generate_presigned_url.
+
+    Maps S3ObjectParams to the Params dict expected by botocore.
+    """
+    presigned_params: dict = {"Bucket": bucket, "Key": key}
+
+    if (acl := params.get("acl")) is not None:
+        presigned_params["ACL"] = acl
+    if (content_type := params.get("content_type")) is not None:
+        presigned_params["ContentType"] = content_type
+    if (content_disposition := params.get("content_disposition")) is not None:
+        presigned_params["ContentDisposition"] = content_disposition
+    if (content_encoding := params.get("content_encoding")) is not None:
+        presigned_params["ContentEncoding"] = content_encoding
+    if (content_language := params.get("content_language")) is not None:
+        presigned_params["ContentLanguage"] = content_language
+    if (cache_control := params.get("cache_control")) is not None:
+        presigned_params["CacheControl"] = cache_control
+    if (storage_class := params.get("storage_class")) is not None:
+        presigned_params["StorageClass"] = storage_class
+    if (sse := params.get("server_side_encryption")) is not None:
+        presigned_params["ServerSideEncryption"] = sse
+    if (sse_kms_key_id := params.get("sse_kms_key_id")) is not None:
+        presigned_params["SSEKMSKeyId"] = sse_kms_key_id
+    if (tagging := params.get("tagging")) is not None:
+        presigned_params["Tagging"] = tagging
+    if (metadata := params.get("metadata")) is not None:
+        presigned_params["Metadata"] = metadata
+
+    return presigned_params
+
+
+@dataclass
+class S3Session:
+    """
+    Utility class that wraps botocore S3 client and niquests async session.
+
+    Usage:
+        async with S3Session(
+            endpoint_url='http://localhost:9000',
+            access_key='foo',
+            secret_key='bar',
+        ) as s3:
+            await s3.put_object('my-bucket', 'path/to/file.txt', b'content')
+            content = await s3.get_object('my-bucket', 'path/to/file.txt')
+
+        # With custom clients:
+        async with S3Session(
+            endpoint_url='...',
+            access_key='...',
+            secret_key='...',
+            s3_client=my_s3_client,
+            http_client=my_http_session,
+        ) as s3:
+            ...
+    """
+
+    endpoint_url: str
+    access_key: str
+    secret_key: str
+    region: str
+    s3_config: Config | None = None
+    s3_client: botocore.client.BaseClient | None = None
+    http_client: niquests.AsyncSession = field(default_factory=niquests.AsyncSession)
+
+    def __post_init__(self):
+        if self.s3_client is None:
+            session = botocore.session.Session()
+            self.s3_client = session.create_client(
+                "s3",
+                endpoint_url=self.endpoint_url,
+                region_name=self.region,
+                aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key,
+                config=self.s3_config,
+            )
+
+    @property
+    def _s3(self) -> botocore.client.BaseClient:
+        if self.s3_client is None:
+            raise ValueError("s3_client not initialized")
+        return self.s3_client
+
+    async def __aenter__(self) -> Self:
+        await self.http_client.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.http_client.__aexit__(exc_type, exc_val, exc_tb)
+        self._s3.close()
+
+    async def delete_object(self, bucket: str, key: str) -> niquests.Response:
+        """Delete an object from S3."""
+        return await s3_delete_object(self._s3, self.http_client, bucket, key)
+
+    async def delete_objects(self, bucket: str, keys: list[str]) -> list[niquests.Response]:
+        """Delete multiple objects from S3."""
+        return await s3_delete_objects(self._s3, self.http_client, bucket, keys)
+
+    def list_files(
+        self,
+        bucket: str,
+        prefix: str,
+        *,
+        search_query: str | None = None,
+        max_items: int | None = None,
+        page_size: int | None = None,
+        starting_token: str | None = None,
+    ) -> AsyncIterator[S3Object]:
+        """List files in an S3 bucket with a given prefix."""
+        return s3_list_files(
+            self._s3,
+            self.http_client,
+            bucket,
+            prefix,
+            search_query=search_query,
+            max_items=max_items,
+            page_size=page_size,
+            starting_token=starting_token,
+        )
+
+    async def put_object(
+        self, bucket: str, key: str, data: bytes, **kwargs: Unpack[S3ObjectParams]
+    ) -> niquests.Response:
+        """Upload an object to S3."""
+        return await s3_put_object(self._s3, self.http_client, bucket, key, data, **kwargs)
+
+    async def upload_file(
+        self, bucket: str, file: Path, path: str, **kwargs: Unpack[S3ObjectParams]
+    ) -> niquests.Response:
+        """Upload a file to S3."""
+        return await s3_upload_file(self._s3, self.http_client, bucket, file, path, **kwargs)
+
+    async def get_object(self, bucket: str, key: str) -> bytes | None:
+        """Download an object from S3."""
+        return await s3_get_object(self._s3, self.http_client, bucket, key)
+
+    async def download_file(
+        self,
+        bucket: str,
+        key: str,
+        on_chunk: Callable[[bytes], None] | None = None,
+        chunk_size: int = 1024 * 1024,
+    ) -> AsyncIterator[bytes]:
+        """Download a file from S3 with streaming support."""
+        async for chunk in s3_download_file(self._s3, self.http_client, bucket, key, chunk_size=chunk_size):
+            if on_chunk:
+                on_chunk(chunk)
+            yield chunk
+
+    def multipart_upload(self, bucket: str, key: str, *, expires_in: int = 3600, **kwargs: Unpack[S3ObjectParams]):
+        """Create a multipart upload context manager."""
+        return s3_multipart_upload(self._s3, self.http_client, bucket, key, expires_in=expires_in, **kwargs)
+
+    async def file_upload(
+        self,
+        bucket: str,
+        key: str,
+        data: AsyncIterator[bytes],
+        *,
+        min_part_size: int = 5 * 1024 * 1024,
+        on_chunk_received: Callable[[bytes], None] | None = None,
+        content_length: int | None = None,
+        **kwargs: Unpack[S3ObjectParams],
+    ) -> None:
+        """Upload a file to S3 using streaming (multipart for large files)."""
+        return await s3_file_upload(
+            self._s3,
+            self.http_client,
+            bucket,
+            key,
+            data,
+            min_part_size=min_part_size,
+            on_chunk_received=on_chunk_received,
+            content_length=content_length,
+            **kwargs,
+        )
+
+
+S3MultipartUpload = namedtuple(
+    "S3MultipartUpload", ["fetch_create", "fetch_complete", "upload_part", "generate_presigned_url", "fetch_abort"]
+)
+
+
+class UploadPart(TypedDict):
+    PartNumber: int
+    ETag: str | None
+
+
+class S3Object(TypedDict, total=False):
+    Key: str
+    LastModified: str
+    ETag: str
+    Size: int
+    StorageClass: str
+
+
+async def s3_delete_object(
+    s3: botocore.client.BaseClient, client: niquests.AsyncSession, bucket: str, key: str
+) -> niquests.Response:
+    """Delete an object from S3 using presigned URL."""
+    url = s3.generate_presigned_url(
+        ClientMethod="delete_object",
+        Params={
+            "Bucket": bucket,
+            "Key": key,
+        },
+    )
+    return (await client.delete(url)).raise_for_status()
+
+
+async def s3_delete_objects(
+    s3: botocore.client.BaseClient, client: niquests.AsyncSession, bucket: str, keys: list[str]
+) -> list[niquests.Response]:
+    """Delete multiple objects from S3 using presigned URLs."""
+    responses = []
+    for key in keys:
+        resp = await s3_delete_object(s3, client, bucket, key)
+        responses.append(resp)
+    return responses
+
+
+async def s3_list_files(
+    s3: botocore.client.BaseClient,
+    client: niquests.AsyncSession,
+    bucket: str,
+    prefix: str,
+    *,
+    search_query: str | None = None,
+    max_items: int | None = None,
+    page_size: int | None = None,
+    starting_token: str | None = None,
+) -> AsyncIterator[S3Object]:
+    """
+    List files in an S3 bucket with a given prefix.
+
+    Yields dicts with 'Key', 'LastModified', 'Size', etc. Use `search_query` for
+    JMESPath filtering (e.g. "Contents[?Size > `100`][]"), `max_items` to limit
+    total results, `page_size` to control items per request, and `starting_token`
+    to resume from a previous continuation token.
+    """
+    api_version = s3.meta.service_model.api_version
+    ns = {"s3": f"http://s3.amazonaws.com/doc/{api_version}/"}
+
+    continuation_token = starting_token
+    items_yielded = 0
+
+    while True:
+        params: dict = {"Bucket": bucket, "Prefix": prefix}
+        if continuation_token:
+            params["ContinuationToken"] = continuation_token
+        if page_size is not None:
+            params["MaxKeys"] = page_size
+
+        url = s3.generate_presigned_url(
+            ClientMethod="list_objects_v2",
+            Params=params,
+        )
+
+        resp = (await client.get(url)).raise_for_status()
+        if resp.content is None:
+            return
+        root = ET.fromstring(resp.content)
+
+        page_items: list[S3Object] = []
+        for contents in root.findall("s3:Contents", ns):
+            item: S3Object = {}
+            for child in contents:
+                tag = child.tag.replace(f"{{{ns['s3']}}}", "")
+                item[tag] = child.text
+            if "Size" in item:
+                item["Size"] = int(item["Size"])
+            page_items.append(item)
+
+        if search_query:
+            page_items = jmespath.search(search_query, {"Contents": page_items}) or []
+
+        for item in page_items:
+            if max_items is not None and items_yielded >= max_items:
+                return
+            yield item
+            items_yielded += 1
+
+        is_truncated = root.find("s3:IsTruncated", ns)
+        if is_truncated is not None and is_truncated.text == "true":
+            next_token = root.find("s3:NextContinuationToken", ns)
+            continuation_token = next_token.text if next_token is not None else None
+        else:
+            break
+
+
+async def s3_put_object(
+    s3: botocore.client.BaseClient,
+    client: niquests.AsyncSession,
+    bucket: str,
+    key: str,
+    data: bytes,
+    **kwargs: Unpack[S3ObjectParams],
+) -> niquests.Response:
+    """
+    Upload an object to S3 using presigned URL.
+
+    See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
+    """
+    obj_params: S3ObjectParams = kwargs
+    presigned_params = build_s3_presigned_params(bucket, key, obj_params)
+    headers = build_s3_headers(obj_params)
+
+    url = s3.generate_presigned_url(
+        ClientMethod="put_object",
+        Params=presigned_params,
+    )
+    resp = (await client.put(url, data=data, headers=headers if headers else None)).raise_for_status()
+    return resp
+
+
+async def s3_upload_file(
+    s3: botocore.client.BaseClient,
+    client: niquests.AsyncSession,
+    bucket: str,
+    file: Path,
+    path: str,
+    **kwargs: Unpack[S3ObjectParams],
+) -> niquests.Response:
+    """
+    Upload a file to S3 using presigned URL.
+    This is a convenience wrapper around s3_put_object that reads the file content.
+    """
+    return await s3_put_object(s3, client, bucket, path, file.read_bytes(), **kwargs)
+
+
+async def s3_get_object(
+    s3: botocore.client.BaseClient, client: niquests.AsyncSession, bucket: str, key: str
+) -> bytes | None:
+    """Download an object from S3 using presigned URL."""
+    url = s3.generate_presigned_url(
+        ClientMethod="get_object",
+        Params={
+            "Bucket": bucket,
+            "Key": key,
+        },
+    )
+    resp = await client.get(url)
+    if resp.status_code == http.HTTPStatus.NOT_FOUND:
+        return None
+    resp.raise_for_status()
+    return resp.content
+
+
+async def s3_download_file(
+    s3: botocore.client.BaseClient,
+    client: niquests.AsyncSession,
+    bucket: str,
+    key: str,
+    *,
+    chunk_size: int = 1024 * 1024,
+) -> AsyncIterator[bytes]:
+    """Download an object from S3 with streaming support."""
+    url = s3.generate_presigned_url(
+        ClientMethod="get_object",
+        Params={"Bucket": bucket, "Key": key},
+    )
+    resp = await client.get(url, stream=True)
+    resp.raise_for_status()
+    async for chunk in await resp.iter_content(chunk_size):
+        yield chunk
+
+
+async def s3_create_multipart_upload(
+    s3: botocore.client.BaseClient,
+    client: niquests.AsyncSession,
+    bucket: str,
+    key: str,
+    *,
+    expires_in: int = 3600,
+    generate_presigned_url: Callable[..., str] | None = None,
+    **kwargs: Unpack[S3ObjectParams],
+) -> str:
+    """
+    Initiate a multipart upload and return the UploadId.
+
+    See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html
+    """
+    obj_params: S3ObjectParams = kwargs
+    headers = build_s3_headers(obj_params)
+
+    if generate_presigned_url is not None:
+        url = generate_presigned_url("create_multipart_upload")
+    else:
+        presigned_params = build_s3_presigned_params(bucket, key, obj_params)
+        url = s3.generate_presigned_url(
+            ClientMethod="create_multipart_upload",
+            Params=presigned_params,
+            ExpiresIn=expires_in,
+        )
+    resp = (await client.post(url, headers=headers if headers else None)).raise_for_status()
+    if resp.content is None:
+        raise ValueError("Empty response from create_multipart_upload")
+    api_version = s3.meta.service_model.api_version
+    ns = {"s3": f"http://s3.amazonaws.com/doc/{api_version}/"}
+    root = ET.fromstring(resp.content)
+    upload_id_elem = root.find("s3:UploadId", ns)
+    if upload_id_elem is None or upload_id_elem.text is None:
+        raise ValueError("UploadId not found in response")
+    return upload_id_elem.text
+
+
+@asynccontextmanager
+async def s3_multipart_upload(
+    s3: botocore.client.BaseClient,
+    client: niquests.AsyncSession,
+    bucket: str,
+    key: str,
+    *,
+    expires_in: int = 3600,
+    **kwargs: Unpack[S3ObjectParams],
+) -> AsyncIterator[S3MultipartUpload]:
+    """Async context manager for S3 multipart upload with automatic cleanup."""
+    obj_params: S3ObjectParams = kwargs
+    upload_id: str | None = None
+    _part_number: int = 1
+    _parts: list[UploadPart] = []
+    _has_been_aborted = False
+
+    async def fetch_complete():
+        if upload_id is None:
+            raise ValueError("Upload ID is not set")
+        complete_url = _generate_presigned_url("complete_multipart_upload", UploadId=upload_id)
+        # Create XML payload for completing multipart upload
+        parts_xml = "".join(
+            f"<Part><PartNumber>{part['PartNumber']}</PartNumber><ETag>{part['ETag']}</ETag></Part>" for part in _parts
+        )
+        xml_payload = f"<CompleteMultipartUpload>{parts_xml}</CompleteMultipartUpload>"
+
+        return (
+            await client.post(complete_url, data=xml_payload, headers={"Content-Type": "application/xml"})
+        ).raise_for_status()
+
+    async def fetch_abort():
+        nonlocal _has_been_aborted
+        if upload_id is None:
+            raise ValueError("Upload ID is not set")
+        abort_url = _generate_presigned_url("abort_multipart_upload", UploadId=upload_id)
+        abort_resp = (await client.delete(abort_url)).raise_for_status()
+        _has_been_aborted = True
+        return abort_resp
+
+    async def upload_part(data: bytes) -> UploadPart:
+        nonlocal _part_number, _parts
+        if upload_id is None:
+            raise ValueError("Upload ID is not set")
+        presigned_url = _generate_presigned_url("upload_part", UploadId=upload_id, PartNumber=_part_number)
+        upload_resp = (await client.put(presigned_url, data=data)).raise_for_status()
+        _etag = upload_resp.headers.get("ETag")
+        etag: str | None = _etag.decode() if isinstance(_etag, bytes) else _etag
+        _part: UploadPart = {"PartNumber": _part_number, "ETag": etag}
+        _parts.append(_part)
+        _part_number += 1
+        return _part
+
+    def _generate_presigned_url(method: str, **params):
+        if method == "create_multipart_upload":
+            _params = {**build_s3_presigned_params(bucket, key, obj_params), **params}
+        else:
+            _params = {"Bucket": bucket, "Key": key, **params}
+        return s3.generate_presigned_url(ClientMethod=method, Params=_params, ExpiresIn=expires_in)
+
+    async def fetch_create() -> str:
+        nonlocal upload_id
+        upload_id = await s3_create_multipart_upload(
+            s3, client, bucket, key, expires_in=expires_in, generate_presigned_url=_generate_presigned_url, **kwargs
+        )
+        return upload_id
+
+    try:
+        yield S3MultipartUpload(
+            fetch_create=fetch_create,
+            fetch_complete=fetch_complete,
+            upload_part=upload_part,
+            fetch_abort=fetch_abort,
+            generate_presigned_url=_generate_presigned_url,
+        )
+    except Exception as e:
+        if not _has_been_aborted and upload_id is not None:
+            await fetch_abort()
+        raise e
+    else:
+        if not _has_been_aborted and upload_id is not None:
+            await fetch_complete()
+
+
+async def s3_file_upload(
+    s3: botocore.client.BaseClient,
+    client: niquests.AsyncSession,
+    bucket: str,
+    key: str,
+    data: AsyncIterator[bytes],
+    *,
+    # 5MB minimum for S3 parts
+    min_part_size: int = 5 * 1024 * 1024,
+    on_chunk_received: Callable[[bytes], None] | None = None,
+    content_length: int | None = None,
+    **kwargs: Unpack[S3ObjectParams],
+) -> None:
+    """
+    Upload a file to S3 from an async byte stream.
+
+    Uses multipart upload for large files. If `content_length` is provided and smaller
+    than `min_part_size`, uses a single PUT instead. Use `on_chunk_received` callback
+    to track upload progress.
+    """
+    if content_length is not None and content_length < min_part_size:
+        # Small file - use single PUT operation
+        _data = b""
+        async for chunk in data:
+            _data += chunk
+            if on_chunk_received:
+                on_chunk_received(chunk)
+        await s3_put_object(s3, client, bucket=bucket, key=key, data=_data, **kwargs)
+        return
+
+    async with s3_multipart_upload(s3, client, bucket=bucket, key=key, **kwargs) as mpart:
+        await mpart.fetch_create()
+        has_uploaded_parts = False
+        async for chunk in get_stream_chunk(data, min_size=min_part_size):
+            if on_chunk_received:
+                on_chunk_received(chunk)
+            if len(chunk) < min_part_size:
+                if not has_uploaded_parts:
+                    # No parts uploaded yet, abort multipart and use single PUT
+                    await mpart.fetch_abort()
+                    await s3_put_object(s3, client, bucket=bucket, key=key, data=chunk, **kwargs)
+                else:
+                    # Parts already uploaded, upload final chunk as last part (S3 allows last part to be smaller)
+                    await mpart.upload_part(chunk)
+                return
+            await mpart.upload_part(chunk)
+            has_uploaded_parts = True

{tracktolib-0.65.1 → tracktolib-0.66.2}/tracktolib/s3/s3.py

@@ -1,8 +1,15 @@
 import datetime as dt
+import warnings
 from io import BytesIO
 from pathlib import Path
 from typing import TypedDict, Literal, Callable
 
+warnings.warn(
+    "tracktolib.s3.s3 is deprecated, use tracktolib.s3.niquests instead",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
 try:
     from aiobotocore.client import AioBaseClient
 except ImportError:
@@ -82,37 +89,12 @@ async def download_file(
 
 
 async def delete_file(client: AioBaseClient, bucket: str, path: str) -> dict:
-    """
-    Delete a file from an S3 bucket.
-
-    Args:
-        client (AioBaseClient): The client to interact with the S3 service.
-        bucket (str): The name of the S3 bucket.
-        path (str): The path to the file within the S3 bucket.
-
-    Return:
-        dict: The response from the S3 service after attempting to delete the file.
-              This typically includes metadata about the operation, such as HTTP status code,
-              any errors encountered, and information about the deleted object.
-    """
+    """Delete a file from an S3 bucket."""
     return await client.delete_object(Bucket=bucket, Key=path)  # type:ignore
 
 
 async def delete_files(client: AioBaseClient, bucket: str, paths: list[str], quiet: bool = True) -> dict:
-    """
-    Delete multiple files from an S3 bucket.
-
-    Args:
-        client (AioBaseClient): The client to interact with the S3 service.
-        bucket (str): The name of the S3 bucket.
-        paths (str): The paths to the files to delete within the S3 bucket.
-        quiet (bool): Whether to suppress printing messages to stdout (default: True).
-
-    Return:
-        dict: The response from the S3 service after attempting to delete the files.
-              This typically includes metadata about the operation, such as HTTP status code,
-              any errors encountered, and information about the deleted object.
-    """
+    """Delete multiple files from an S3 bucket. Set `quiet=False` to print deletion messages."""
     delete_request = {"Objects": [{"Key": path} for path in paths], "Quiet": quiet}
     return await client.delete_objects(Bucket=bucket, Delete=delete_request)  # type:ignore
 

{tracktolib-0.65.1 → tracktolib-0.66.2}/tracktolib/utils.py

@@ -1,5 +1,5 @@
-import sys
-from types import ModuleType
+from pathlib import Path
+
 import asyncio
 import datetime as dt
 import importlib.util
@@ -7,13 +7,11 @@ import itertools
 import mmap
 import os
 import subprocess
+import sys
 from decimal import Decimal
 from ipaddress import IPv4Address, IPv6Address
-from pathlib import Path
-from typing import Iterable, TypeVar, Iterator, Literal, overload, Any, Callable
-
-
-T = TypeVar("T")
+from types import ModuleType
+from typing import AsyncIterable, AsyncIterator, Iterable, Iterator, Literal, overload, Any, Callable
 
 type OnCmdUpdate = Callable[[str], None]
 type OnCmdDone = Callable[[str, str, int], None]
@@ -87,24 +85,46 @@ def import_module(path: Path):
 
 
 @overload
-def get_chunks(it: Iterable[T], size: int, *, as_list: Literal[False]) -> Iterator[Iterable[T]]: ...
+def get_chunks[T](it: Iterable[T], size: int, *, as_list: Literal[False]) -> Iterator[Iterable[T]]: ...
 
 
 @overload
-def get_chunks(it: Iterable[T], size: int, *, as_list: Literal[True]) -> Iterator[list[T]]: ...
+def get_chunks[T](it: Iterable[T], size: int, *, as_list: Literal[True]) -> Iterator[list[T]]: ...
 
 
 @overload
-def get_chunks(it: Iterable[T], size: int) -> Iterator[list[T]]: ...
+def get_chunks[T](it: Iterable[T], size: int) -> Iterator[list[T]]: ...
 
 
-def get_chunks(it: Iterable[T], size: int, *, as_list: bool = True) -> Iterator[Iterable[T]]:
+def get_chunks[T](it: Iterable[T], size: int, *, as_list: bool = True) -> Iterator[Iterable[T]]:
     iterator = iter(it)
     for first in iterator:
         d = itertools.chain([first], itertools.islice(iterator, size - 1))
         yield d if not as_list else list(d)
 
 
+async def get_stream_chunk[S: (bytes, str)](data_stream: AsyncIterable[S], min_size: int) -> AsyncIterator[S]:
+    """Buffers an async stream and yields chunks of at least `min_size`."""
+    buffer: S | None = None
+    buffer_size = 0
+
+    async for chunk in data_stream:
+        if not chunk:
+            continue
+        buffer = chunk if buffer is None else buffer + chunk  # type: ignore[operator]
+        buffer_size += len(chunk)
+
+        # Yield chunks of min_size while we have enough data for at least 2 chunks
+        while buffer_size >= min_size * 2:
+            yield buffer[:min_size]
+            buffer = buffer[min_size:]
+            buffer_size -= min_size
+
+    # Handle the final chunk(s)
+    if buffer is not None and buffer_size > 0:
+        yield buffer
+
+
 def json_serial(obj):
     """JSON serializer for objects not serializable by default json code"""
     if isinstance(obj, (dt.datetime, dt.date)):