cloudbridge 4.3.0__tar.gz → 4.4.0__tar.gz

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudbridge
-Version: 4.3.0
+Version: 4.4.0
 Summary: A simple layer of abstraction over multiple cloud providers.
 Author-email: Galaxy and GVL Projects <help@genome.edu.au>
 License-Expression: MIT

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/__init__.py

@@ -3,7 +3,7 @@ import logging
 from typing import Any
 
 # Current version of the library
-__version__ = '4.3.0'
+__version__ = '4.4.0'
 
 
 def get_version() -> str:

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/base/resources.py

@@ -8,7 +8,7 @@ import logging
 import os
 import queue
 import re
-import shutil
+import threading
 import time
 import uuid
 from concurrent.futures import FIRST_COMPLETED
@@ -70,8 +70,6 @@ from cloudbridge.interfaces.resources import VolumeState
 from . import helpers as cb_helpers
 
 if TYPE_CHECKING:
-    from _typeshed import SupportsRead
-
     from cloudbridge.base.provider import BaseCloudProvider
     from cloudbridge.base.services import BaseStorageService
     from cloudbridge.interfaces.services import BucketObjectService
@@ -839,6 +837,15 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
     # Number of parts uploaded in parallel by the transparent multipart path.
     CB_MULTIPART_MAX_CONCURRENCY = int(os.environ.get(
         'CB_MULTIPART_MAX_CONCURRENCY', 5))
+    # Size of each chunk yielded by the single-stream read path
+    # (``iter_content``/``save_content``). Sits at the knee of the
+    # throughput curve: reading an HTTP body at 1 MiB is ~12x cheaper per
+    # byte than at 4 KiB, while 4 MiB and above measure the same as 1 MiB
+    # and cost proportionally more memory per concurrent stream. Unrelated
+    # to CB_MULTIPART_PART_SIZE, which sizes the parts of a *parallel*
+    # transfer rather than the chunks of a sequential read.
+    CB_ITER_CHUNK_SIZE = int(os.environ.get(
+        'CB_ITER_CHUNK_SIZE', 1024 * 1024))              # 1 MiB
 
     def __init__(self, provider: CloudProvider) -> None:
         super(BaseBucketObject, self).__init__(provider)
@@ -877,23 +884,54 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
                 "in: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMeta"
                 "data.html#object-key-guidelines" % name)
 
-    def save_content(self, target_stream: IO[bytes]) -> None:
-        # iter_content() is declared Iterable[bytes] on the interface, but the
-        # concrete objects returned by providers also support .read(); cast so
-        # copyfileobj accepts it without changing behavior.
-        shutil.copyfileobj(
-            cast("SupportsRead[bytes]", self.iter_content()), target_stream)
+    def save_content(self, target_stream: IO[bytes],
+                     chunk_size: int | None = None) -> None:
+        # Written in terms of iter_content so that the interface's promise -
+        # an Iterable[bytes] - is all a provider has to deliver. Copying via
+        # shutil.copyfileobj would additionally require a .read(), which not
+        # every provider's return value has.
+        for chunk in self.iter_content(chunk_size=chunk_size):
+            target_stream.write(chunk)
 
     def download_to_file(self, path: str,
                          config: TransferConfig | None = None) -> None:
+        # Assemble the object in a private file alongside the destination and
+        # rename it into place once complete, so ``path`` only ever holds a
+        # whole object. Callers commonly download every copy of an object to
+        # one well-known path (a cache entry, say), so writing in place would
+        # let concurrent downloads truncate each other's file - or rename it
+        # away mid-transfer - and would destroy a previously downloaded copy
+        # when a transfer fails.
+        part_path = f"{path}.{uuid.uuid4().hex}.cbpart"
+        try:
+            self._download_to_path(part_path, config)
+            os.replace(part_path, path)
+        except BaseException:
+            try:
+                os.remove(part_path)
+            except OSError:
+                pass
+            raise
+
+    def _download_to_path(self, path: str,
+                          config: TransferConfig | None = None) -> None:
+        """
+        Write this object's content to ``path``, which the caller owns.
+
+        Providers with an efficient, thread-safe native downloader (e.g. AWS
+        via boto3's ``download_file``, Azure via ``download_blob``) override
+        this to use it; the default implementation streams small objects and
+        fetches larger ones as parallel ranged reads.
+        """
         size = self.size
         if size <= self._multipart_threshold(config):
             with open(path, 'wb') as f:
                 self.save_content(f)
             return
-        self._download_ranged(path, size, config)
+        with open(path, 'w+b') as f:
+            self._download_ranged(f, size, config)
 
-    def _download_ranged(self, path: str, size: int,
+    def _download_ranged(self, target: IO[bytes], size: int,
                          config: TransferConfig | None = None) -> None:
         """
         Fetch the object as ranged reads across a bounded thread pool,
@@ -902,40 +940,26 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
         To stay safe even on providers whose SDK client/connection is not
         thread-safe, each worker reads through its own cloned provider (see
         :meth:`.CloudProvider.clone`), so no provider state is shared between
-        threads. Memory is bounded to ~concurrency * part_size. On any
-        failure the partial file is removed and the error re-raised.
-
-        Providers with an efficient, thread-safe native downloader (e.g. AWS
-        via boto3's ``download_file``, Azure via ``download_blob``) override
-        ``download_to_file`` to use it directly.
+        threads. Memory is bounded to ~concurrency * part_size.
         """
         part_size = self._multipart_part_size(config)
         if part_size < 1:
             raise InvalidValueException('part_size', part_size)
         concurrency = max(1, self._multipart_max_concurrency(config))
+        target.truncate(size)
         ranges = [(offset, min(part_size, size - offset))
                   for offset in range(0, size, part_size)]
-        try:
-            with open(path, 'wb') as f:
-                f.truncate(size)
-            if concurrency == 1:
-                bucket_objects = self._bucket_objects
-                with open(path, 'r+b') as f:
-                    for offset, length in ranges:
-                        f.seek(offset)
-                        f.write(bucket_objects.download_range(
-                            self.bucket, self.name, offset, length))
-            else:
-                self._download_ranges_concurrently(path, ranges, concurrency)
-        except Exception:
-            try:
-                os.remove(path)
-            except OSError:
-                pass
-            raise
+        if concurrency == 1:
+            bucket_objects = self._bucket_objects
+            for offset, length in ranges:
+                target.seek(offset)
+                target.write(bucket_objects.download_range(
+                    self.bucket, self.name, offset, length))
+        else:
+            self._download_ranges_concurrently(target, ranges, concurrency)
 
     def _download_ranges_concurrently(
-            self, path: str, ranges: list[tuple[int, int]],
+            self, target: IO[bytes], ranges: list[tuple[int, int]],
             concurrency: int) -> None:
         # A pool of cloned bucket-object services, one per worker, so each
         # thread touches an isolated provider/connection.
@@ -947,6 +971,7 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
 
         bucket = self.bucket
         name = self.name
+        write_lock = threading.Lock()
 
         def fetch_one(offset: int, length: int) -> None:
             service = clones.get()
@@ -954,13 +979,14 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
                 data = service.download_range(bucket, name, offset, length)
             finally:
                 clones.put(service)
-            # Each worker writes through its own handle at its own offset;
-            # ranges never overlap, so no locking is needed. Data is released
-            # as soon as it is written, bounding memory to
-            # ~concurrency * part_size.
-            with open(path, 'r+b') as f:
-                f.seek(offset)
-                f.write(data)
+            # Ranges are fetched in parallel but written through the one
+            # handle the caller opened, so a range can never be written to a
+            # file that has since been replaced. Serializing the writes costs
+            # little next to the fetches, and the data is released as soon as
+            # it is written, bounding memory to ~concurrency * part_size.
+            with write_lock:
+                target.seek(offset)
+                target.write(data)
 
         with ThreadPoolExecutor(max_workers=concurrency) as executor:
             futures = [executor.submit(fetch_one, offset, length)
@@ -990,6 +1016,22 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
         return int(self._provider._get_config_value(
             'multipart_max_concurrency', self.CB_MULTIPART_MAX_CONCURRENCY))
 
+    def _iter_chunk_size(self, chunk_size: int | None = None) -> int:
+        """
+        Resolve the chunk size for a single-stream read: an explicit
+        ``chunk_size``, else the provider/global config, else the class
+        default. Providers call this at the top of ``iter_content`` so the
+        value is validated before any request is issued.
+        """
+        if chunk_size is None:
+            chunk_size = int(self._provider._get_config_value(
+                'iter_chunk_size', self.CB_ITER_CHUNK_SIZE))
+        else:
+            chunk_size = int(chunk_size)
+        if chunk_size <= 0:
+            raise InvalidValueException('iter_chunk_size', chunk_size)
+        return chunk_size
+
     @staticmethod
     def _data_size(data: str | bytes | IO[bytes]) -> int | None:
         """

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/base/services.py

@@ -320,7 +320,8 @@ class BaseNetworkService(
             log.info("Creating a CloudBridge-default network labeled %s",
                      BaseNetwork.CB_DEFAULT_NETWORK_LABEL)
             return self.provider.networking.networks.create(
-                BaseNetwork.CB_DEFAULT_NETWORK_LABEL, '10.0.0.0/16')
+                BaseNetwork.CB_DEFAULT_NETWORK_LABEL,
+                BaseNetwork.CB_DEFAULT_IPV4RANGE)
 
     @dispatch(event="provider.networking.networks.find",
               priority=BaseCloudService.STANDARD_EVENT_PRIORITY)

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/interfaces/resources.py

@@ -2357,20 +2357,52 @@ class BucketObject(CloudResource):
         pass
 
     @abstractmethod
-    def iter_content(self) -> Iterable[bytes]:
+    def iter_content(self, chunk_size: int | None = None) -> Iterable[bytes]:
         """
-        Returns this object's content as an iterable.
+        Returns this object's content as an iterable of byte chunks.
+
+        The object is streamed rather than held in memory, so this is safe
+        for objects of any size. Chunks are sized by ``chunk_size``, never by
+        the content itself - binary data is never split on newlines - and
+        only the final chunk may be shorter.
+
+        ``chunk_size`` trades per-chunk overhead against memory and latency.
+        Each chunk costs a read from the provider plus whatever the caller
+        does per chunk, so small values are expensive over a large object;
+        conversely a chunk is buffered in full before it is yielded, so large
+        values cost memory per concurrent stream and delay the first chunk.
+        The default suits most callers.
+
+        :type chunk_size: ``int``
+        :param chunk_size: Maximum size in bytes of each chunk yielded. If
+            ``None``, falls back to the provider/global configuration
+            (``iter_chunk_size`` / the ``CB_ITER_CHUNK_SIZE`` setting,
+            1 MiB by default). Must be positive.
 
         :rtype: Iterable
         :return: An iterable of the file contents
 
+        :raise: ``InvalidValueException`` if ``chunk_size`` is not positive.
         """
         pass
 
     @abstractmethod
-    def save_content(self, target_stream: IO[bytes]) -> None:
+    def save_content(self, target_stream: IO[bytes],
+                     chunk_size: int | None = None) -> None:
         """
         Save this object and write its contents to the ``target_stream``.
+
+        The object is streamed through ``iter_content``, so ``chunk_size``
+        means what it does there.
+
+        :type target_stream: ``IO[bytes]``
+        :param target_stream: A writable binary stream to write to.
+
+        :type chunk_size: ``int``
+        :param chunk_size: Maximum size in bytes of each chunk read from the
+            provider. See :meth:`.iter_content`.
+
+        :raise: ``InvalidValueException`` if ``chunk_size`` is not positive.
         """
         pass
 
@@ -2388,8 +2420,12 @@ class BucketObject(CloudResource):
         remain single-stream alternatives for arbitrary target streams.
 
         :type path: ``str``
-        :param path: Local path to write the object's content to. An existing
-            file is overwritten; on failure no partial file is left behind.
+        :param path: Local path to write the object's content to. The object
+            is assembled out of the way and moved into place once complete,
+            so ``path`` never holds a partial object: an existing file is
+            replaced atomically, and a failed transfer leaves it untouched.
+            Concurrent downloads to one path are therefore safe, with the
+            last to complete winning.
 
         :type config: :class:`.TransferConfig`
         :param config: Optional per-call transfer tuning (threshold, part

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/providers/aws/helpers.py

@@ -25,6 +25,18 @@ log = logging.getLogger(__name__)
 
 T = TypeVar("T")
 
+# How many records to request per call while satisfying a list method,
+# overridable per provider with the ``aws_page_size`` config value. This is a
+# transport setting and is deliberately not ``default_result_limit``, which
+# bounds how many results the caller receives - see
+# BotoGenericService._page_size for why conflating them is expensive.
+#
+# AWS-specific because it only means anything where the provider walks pages
+# itself. GCP, Azure and OpenStack each return a single page plus a
+# continuation token and let the caller drive, so there is nothing for a page
+# size to amplify; boto3's paginator is the odd one out.
+DEFAULT_PAGE_SIZE = 500
+
 
 def trim_empty_params(params_dict: dict[str, Any]) -> dict[str, Any]:
     """
@@ -153,6 +165,46 @@ class BotoGenericService(object):
         else:
             return None
 
+    def _page_size(self, client: Any, list_op: str) -> int:
+        """
+        Transport page size for a paginated call.
+
+        ``PageSize`` is how many items the service returns per request;
+        ``MaxItems`` is how many the caller asked for. Using the caller's
+        limit for both makes a small limit walk a large scan in tiny
+        increments: a filtered ``describe_images`` that takes 10.0s at a page
+        size of 1000 takes 977.6s at 5, for the same single result. Ask for a
+        full page regardless of the limit and let ``MaxItems`` do the
+        bounding - at worst one page more data is fetched than was wanted,
+        and a limit larger than a page is served over several requests.
+
+        The size comes from the ``aws_page_size`` config value so it can be
+        tuned per provider, then is clamped to whatever the service model
+        declares for this operation. EC2's bounds differ per call -
+        ``DescribeRouteTables`` permits 100 where most permit 1000, and
+        several require at least 5 - and falling outside them is a hard
+        ``InvalidParameterValue`` rather than a clamp, so neither the default
+        nor a configured value is passed through unchecked.
+        """
+        page_size = int(self.provider._get_config_value(
+            'aws_page_size', DEFAULT_PAGE_SIZE))
+        api_name = client.meta.method_to_api_mapping.get(list_op)
+        if not api_name:
+            return page_size
+        input_shape = client.meta.service_model.operation_model(
+            api_name).input_shape
+        max_results = input_shape.members.get('MaxResults') if input_shape \
+            else None
+        # Not every operation declares bounds, even where the documentation
+        # gives them; the default is within all of the undeclared ones.
+        bounds = max_results.metadata if max_results else {}
+        ceiling, floor = bounds.get('max'), bounds.get('min')
+        if ceiling:
+            page_size = min(page_size, ceiling)
+        if floor:
+            page_size = max(page_size, floor)
+        return page_size
+
     def _get_list_operation(self) -> str:
         """
         This function discovers the list operation for a particular resource
@@ -195,7 +247,9 @@ class BotoGenericService(object):
         paginator = client.get_paginator(list_op)
         PaginationConfig: dict[str, Any] = {}
         if limit:
-            PaginationConfig = {'MaxItems': limit, 'PageSize': limit}
+            PaginationConfig = {
+                'MaxItems': limit,
+                'PageSize': self._page_size(client, list_op)}
 
         if marker:
             PaginationConfig.update({'StartingToken': marker})

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/providers/aws/resources.py

@@ -861,14 +861,21 @@ class AWSVMFirewallRule(BaseVMFirewallRule):
 
 class AWSBucketObject(BaseBucketObject):
     class BucketObjIterator():
-        CHUNK_SIZE = 4096
+        """
+        Chunked reader over a boto3 ``StreamingBody``.
+
+        Also exposes ``read``/``close`` so the value handed back by
+        ``iter_content`` stays usable as a file-like object, as it has been
+        since this replaced boto2's ``Key``.
+        """
 
-        def __init__(self, body: Any) -> None:
+        def __init__(self, body: Any, chunk_size: int) -> None:
             self.body = body
+            self.chunk_size = chunk_size
 
         def __iter__(self) -> Iterator[bytes]:
             while True:
-                data = self.read(self.CHUNK_SIZE)
+                data = self.read(self.chunk_size)
                 if data:
                     yield data
                 else:
@@ -910,8 +917,12 @@ class AWSBucketObject(BaseBucketObject):
             cast("AWSCloudProvider", self._provider).s3_conn
             .Bucket(self._obj.bucket_name))
 
-    def iter_content(self) -> Iterable[bytes]:
-        return self.BucketObjIterator(self._obj.get().get('Body'))
+    def iter_content(self, chunk_size: int | None = None) -> Iterable[bytes]:
+        # Resolve (and validate) the chunk size before the GET, so a bad
+        # value does not leave an unread response body behind.
+        chunk_size = self._iter_chunk_size(chunk_size)
+        return self.BucketObjIterator(
+            self._obj.get().get('Body'), chunk_size)
 
     def _upload_single_shot(self,
                             data: str | bytes | IO[bytes]) -> BucketObject:
@@ -944,8 +955,8 @@ class AWSBucketObject(BaseBucketObject):
         self._obj.upload_file(path, Config=transfer_config)
         return self
 
-    def download_to_file(self, path: str,
-                         config: TransferConfig | None = None) -> None:
+    def _download_to_path(self, path: str,
+                          config: TransferConfig | None = None) -> None:
         # boto3's TransferManager downloads large objects as parallel ranged
         # GETs with a thread-safe client, so the transparent ranged path
         # delegates to it rather than CloudBridge's generic clone-pool driver.

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/providers/aws/services.py

@@ -806,8 +806,22 @@ class AWSImageService(BaseImageService):
             log.debug("Searching for AWS Image Service %s", label)
             obj_list.extend(
                 self.svc.find(filters={'name': label}, **extra_args))
+            # A tag filter can only ever match images in the calling account.
+            # AMI tags are not visible across accounts, so an image owned by
+            # anyone else can never satisfy tag:Name however it is tagged -
+            # asking for every image carrying any visible tag, unscoped across
+            # a whole region, returns only this account's. Leaving Owners off
+            # therefore does not widen what the search can find; it only makes
+            # EC2 evaluate the filter against every public image in the
+            # region. Measured in ap-southeast-1, identical single-image
+            # results in 10.0s unscoped against 0.1s scoped.
+            #
+            # An explicit owners argument still wins, so a caller can ask for
+            # someone else's images and get the same (empty) answer as before.
+            tag_args = dict(extra_args)
+            tag_args.setdefault('Owners', ['self'])
             obj_list.extend(
-                self.svc.find(filters={'tag:Name': label}, **extra_args))
+                self.svc.find(filters={'tag:Name': label}, **tag_args))
         return ClientPagedResultList(self.provider, obj_list)
 
     # Intentionally extends the base list() with a leading filter_by_owner
@@ -1023,6 +1037,9 @@ class AWSVMTypeService(BaseVMTypeService):
 
     def __init__(self, provider: CloudProvider) -> None:
         super(AWSVMTypeService, self).__init__(provider)
+        # Raw instance type dicts, keyed by availability zone. See
+        # _get_catalogue for why this is memoised.
+        self._catalogue: dict[str | None, list[dict[str, Any]]] = {}
 
     @dispatch(event="provider.compute.vm_types.get",
               priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
@@ -1039,15 +1056,12 @@ class AWSVMTypeService(BaseVMTypeService):
             else:
                 raise e
 
-    @dispatch(event="provider.compute.vm_types.list",
-              priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
-    def list(self, limit: int | None = None,
-             marker: str | None = None) -> ResultList[VMType]:
+    def _fetch_catalogue(self, zone: str | None) -> list[dict[str, Any]]:
         client = cast("AWSCloudProvider", self.provider).ec2_conn.meta.client
         vmt_list_resp = client.describe_instance_type_offerings(
             LocationType='availability-zone',
             Filters=[{'Name': 'location',
-                      'Values': [self.provider.zone_name]}],
+                      'Values': [zone]}],
             # MaxResults is set to max value (1000)
             # and client-side pagination is used
             **trim_empty_params({'MaxResults': 1000, 'NextToken': None}))
@@ -1056,7 +1070,7 @@ class AWSVMTypeService(BaseVMTypeService):
             vmt_list_resp = client.describe_instance_type_offerings(
                 LocationType='availability-zone',
                 Filters=[{'Name': 'location',
-                          'Values': [self.provider.zone_name]}],
+                          'Values': [zone]}],
                 **trim_empty_params(
                     {'MaxResults': 1000,
                      'NextToken': vmt_list_resp.get("NextToken")}))
@@ -1067,12 +1081,41 @@ class AWSVMTypeService(BaseVMTypeService):
         # describe_instance_types call can get at most 100 types at once
         chunks = [vmt_list_names[x:x + 100]
                   for x in range(0, len(vmt_list_names), 100)]
-        raw_types = []
+        raw_types: list[dict[str, Any]] = []
         for chunk in chunks:
             raw_chunk = client.describe_instance_types(
                 InstanceTypes=chunk).get('InstanceTypes')
             raw_types.extend(raw_chunk)
-        cb_types = [AWSVMType(cast("AWSCloudProvider", self.provider), t) for t in raw_types]
+        return raw_types
+
+    def _get_catalogue(self) -> list[dict[str, Any]]:
+        """
+        Return the raw instance type catalogue for the provider's zone,
+        fetching it at most once per zone.
+
+        EC2 has no server-side paging for instance types, so ``list()`` must
+        materialise the whole catalogue and page it client-side. Fetching it
+        costs one ``DescribeInstanceTypeOfferings`` walk plus one
+        ``DescribeInstanceTypes`` call per 100 types — around 14 calls for a
+        real region. Without memoisation every page of a ``list(marker=...)``
+        walk repeats all of that, making a full walk quadratic in API calls:
+        1343 types at a result limit of 5 costs ~4300 calls instead of ~14.
+
+        The catalogue is static for the lifetime of a provider, so it is held
+        per zone (a provider may be cloned to another zone, which genuinely
+        offers a different set of types).
+        """
+        zone = self.provider.zone_name
+        if zone not in self._catalogue:
+            self._catalogue[zone] = self._fetch_catalogue(zone)
+        return self._catalogue[zone]
+
+    @dispatch(event="provider.compute.vm_types.list",
+              priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
+    def list(self, limit: int | None = None,
+             marker: str | None = None) -> ResultList[VMType]:
+        cb_types = [AWSVMType(cast("AWSCloudProvider", self.provider), t)
+                    for t in self._get_catalogue()]
         return ClientPagedResultList(self.provider, cb_types,
                                      limit=limit, marker=marker)
 
@@ -1674,6 +1717,14 @@ class AWSDnsZoneService(BaseDnsZoneService):
             client.delete_hosted_zone(Id=dns_zone.aws_id)
 
 
+# Route53 reports record changes INSYNC within seconds, but boto3's
+# resource_record_sets_changed waiter polls every 30s by default, so every
+# record change costs a full 30s of sleep no matter how fast it propagated.
+# Poll often enough that the granularity stops dominating, while keeping the
+# same ~30 minute ceiling for changes that genuinely are slow.
+DNS_CHANGE_WAITER_CONFIG = {'Delay': 5, 'MaxAttempts': 360}
+
+
 class AWSDnsRecordService(BaseDnsRecordService):
 
     def __init__(self, provider: CloudProvider) -> None:
@@ -1765,7 +1816,8 @@ class AWSDnsRecordService(BaseDnsRecordService):
         # waiting, this is skipped for mock tests.
         if not cast("AWSCloudProvider", self.provider).PROVIDER_ID == 'mock':
             waiter = client.get_waiter('resource_record_sets_changed')
-            waiter.wait(Id=response.get('ChangeInfo').get('Id'))
+            waiter.wait(Id=response.get('ChangeInfo').get('Id'),
+                        WaiterConfig=DNS_CHANGE_WAITER_CONFIG)
         return cast(DnsRecord, self.get(dns_zone, name + ":" + type))
 
     def delete(self, dns_zone: DnsZone | str,
@@ -1793,4 +1845,5 @@ class AWSDnsRecordService(BaseDnsRecordService):
         # waiting, this is skipped for mock tests.
         if not cast("AWSCloudProvider", self.provider).PROVIDER_ID == 'mock':
             waiter = client.get_waiter('resource_record_sets_changed')
-            waiter.wait(Id=response.get('ChangeInfo').get('Id'))
+            waiter.wait(Id=response.get('ChangeInfo').get('Id'),
+                        WaiterConfig=DNS_CHANGE_WAITER_CONFIG)

{cloudbridge-4.3.0 → cloudbridge-4.4.0}/cloudbridge/providers/azure/resources.py

@@ -4,7 +4,6 @@ DataTypes used by this provider
 from __future__ import annotations
 
 import collections
-import io
 import logging
 from datetime import datetime
 from typing import Any
@@ -245,37 +244,27 @@ class AzureBucketObject(BaseBucketObject):
         """
         return self._blob_properties.last_modified.strftime("%Y-%m-%dT%H:%M:%S.%f")
 
-    def iter_content(self) -> Iterable[bytes]:
+    def iter_content(self, chunk_size: int | None = None) -> Iterable[bytes]:
         """
-        Returns this object's content as an
-        iterable stream.
+        Returns this object's content as an iterable of byte chunks.
         """
-
-        def iterable_to_stream(iterable: Iterator[bytes]) -> io.RawIOBase:
-            class IterStream(io.RawIOBase):
-                def __init__(self) -> None:
-                    self.leftover: bytes | None = None
-
-                def readable(self) -> bool:
-                    return True
-
-                def readinto(self, b: Any) -> int:
-                    try:
-                        buffer_length = len(b)  # We're supposed to return at most this much
-                        chunk = self.leftover or next(iterable)
-                        output, self.leftover = chunk[:buffer_length], chunk[buffer_length:]
-                        b[:len(output)] = output
-                        return len(output)
-                    except StopIteration:
-                        return 0  # indicate EOF
-
-            return IterStream()
+        chunk_size = self._iter_chunk_size(chunk_size)
+        # The downloader buffers the service's own chunks (max_chunk_get_size,
+        # 4 MiB by default) and read() serves chunk_size slices out of them
+        # over a single connection, so the caller's chunk size is independent
+        # of the service's. Previously this returned a RawIOBase wrapper,
+        # which iterates by *line* rather than by size - splitting binary
+        # content on b"\n", and buffering a newline-free blob whole.
+        downloader = self._blob_client.download_blob()
 
         def blob_iterator() -> Iterator[bytes]:
-            for chunk in self._blob_client.download_blob().chunks():
-                yield chunk
+            while True:
+                data = downloader.read(chunk_size)
+                if not data:
+                    break
+                yield data
 
-        return iterable_to_stream(blob_iterator())
+        return blob_iterator()
 
     @property
     def bucket(self) -> AzureBucket:
@@ -302,8 +291,8 @@ class AzureBucketObject(BaseBucketObject):
             max_concurrency=self._multipart_max_concurrency(config))
         return self
 
-    def download_to_file(self, path: str,
-                         config: TransferConfig | None = None) -> None:
+    def _download_to_path(self, path: str,
+                          config: TransferConfig | None = None) -> None:
         # azure-storage-blob's downloader fetches block ranges concurrently
         # with a thread-safe client, so delegate to it rather than
         # CloudBridge's generic clone-pool driver.