cloudbridge 4.3.0__tar.gz → 4.3.1__tar.gz

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudbridge
-Version: 4.3.0
+Version: 4.3.1
 Summary: A simple layer of abstraction over multiple cloud providers.
 Author-email: Galaxy and GVL Projects <help@genome.edu.au>
 License-Expression: MIT

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge/__init__.py

@@ -3,7 +3,7 @@ import logging
 from typing import Any
 
 # Current version of the library
-__version__ = '4.3.0'
+__version__ = '4.3.1'
 
 
 def get_version() -> str:

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge/base/resources.py

@@ -9,6 +9,7 @@ import os
 import queue
 import re
 import shutil
+import threading
 import time
 import uuid
 from concurrent.futures import FIRST_COMPLETED
@@ -886,14 +887,43 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
 
     def download_to_file(self, path: str,
                          config: TransferConfig | None = None) -> None:
+        # Assemble the object in a private file alongside the destination and
+        # rename it into place once complete, so ``path`` only ever holds a
+        # whole object. Callers commonly download every copy of an object to
+        # one well-known path (a cache entry, say), so writing in place would
+        # let concurrent downloads truncate each other's file - or rename it
+        # away mid-transfer - and would destroy a previously downloaded copy
+        # when a transfer fails.
+        part_path = f"{path}.{uuid.uuid4().hex}.cbpart"
+        try:
+            self._download_to_path(part_path, config)
+            os.replace(part_path, path)
+        except BaseException:
+            try:
+                os.remove(part_path)
+            except OSError:
+                pass
+            raise
+
+    def _download_to_path(self, path: str,
+                          config: TransferConfig | None = None) -> None:
+        """
+        Write this object's content to ``path``, which the caller owns.
+
+        Providers with an efficient, thread-safe native downloader (e.g. AWS
+        via boto3's ``download_file``, Azure via ``download_blob``) override
+        this to use it; the default implementation streams small objects and
+        fetches larger ones as parallel ranged reads.
+        """
         size = self.size
         if size <= self._multipart_threshold(config):
             with open(path, 'wb') as f:
                 self.save_content(f)
             return
-        self._download_ranged(path, size, config)
+        with open(path, 'w+b') as f:
+            self._download_ranged(f, size, config)
 
-    def _download_ranged(self, path: str, size: int,
+    def _download_ranged(self, target: IO[bytes], size: int,
                          config: TransferConfig | None = None) -> None:
         """
         Fetch the object as ranged reads across a bounded thread pool,
@@ -902,40 +932,26 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
         To stay safe even on providers whose SDK client/connection is not
         thread-safe, each worker reads through its own cloned provider (see
         :meth:`.CloudProvider.clone`), so no provider state is shared between
-        threads. Memory is bounded to ~concurrency * part_size. On any
-        failure the partial file is removed and the error re-raised.
-
-        Providers with an efficient, thread-safe native downloader (e.g. AWS
-        via boto3's ``download_file``, Azure via ``download_blob``) override
-        ``download_to_file`` to use it directly.
+        threads. Memory is bounded to ~concurrency * part_size.
         """
         part_size = self._multipart_part_size(config)
         if part_size < 1:
             raise InvalidValueException('part_size', part_size)
         concurrency = max(1, self._multipart_max_concurrency(config))
+        target.truncate(size)
         ranges = [(offset, min(part_size, size - offset))
                   for offset in range(0, size, part_size)]
-        try:
-            with open(path, 'wb') as f:
-                f.truncate(size)
-            if concurrency == 1:
-                bucket_objects = self._bucket_objects
-                with open(path, 'r+b') as f:
-                    for offset, length in ranges:
-                        f.seek(offset)
-                        f.write(bucket_objects.download_range(
-                            self.bucket, self.name, offset, length))
-            else:
-                self._download_ranges_concurrently(path, ranges, concurrency)
-        except Exception:
-            try:
-                os.remove(path)
-            except OSError:
-                pass
-            raise
+        if concurrency == 1:
+            bucket_objects = self._bucket_objects
+            for offset, length in ranges:
+                target.seek(offset)
+                target.write(bucket_objects.download_range(
+                    self.bucket, self.name, offset, length))
+        else:
+            self._download_ranges_concurrently(target, ranges, concurrency)
 
     def _download_ranges_concurrently(
-            self, path: str, ranges: list[tuple[int, int]],
+            self, target: IO[bytes], ranges: list[tuple[int, int]],
             concurrency: int) -> None:
         # A pool of cloned bucket-object services, one per worker, so each
         # thread touches an isolated provider/connection.
@@ -947,6 +963,7 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
 
         bucket = self.bucket
         name = self.name
+        write_lock = threading.Lock()
 
         def fetch_one(offset: int, length: int) -> None:
             service = clones.get()
@@ -954,13 +971,14 @@ class BaseBucketObject(BaseCloudResource, BucketObject):
                 data = service.download_range(bucket, name, offset, length)
             finally:
                 clones.put(service)
-            # Each worker writes through its own handle at its own offset;
-            # ranges never overlap, so no locking is needed. Data is released
-            # as soon as it is written, bounding memory to
-            # ~concurrency * part_size.
-            with open(path, 'r+b') as f:
-                f.seek(offset)
-                f.write(data)
+            # Ranges are fetched in parallel but written through the one
+            # handle the caller opened, so a range can never be written to a
+            # file that has since been replaced. Serializing the writes costs
+            # little next to the fetches, and the data is released as soon as
+            # it is written, bounding memory to ~concurrency * part_size.
+            with write_lock:
+                target.seek(offset)
+                target.write(data)
 
         with ThreadPoolExecutor(max_workers=concurrency) as executor:
             futures = [executor.submit(fetch_one, offset, length)

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge/interfaces/resources.py

@@ -2388,8 +2388,12 @@ class BucketObject(CloudResource):
         remain single-stream alternatives for arbitrary target streams.
 
         :type path: ``str``
-        :param path: Local path to write the object's content to. An existing
-            file is overwritten; on failure no partial file is left behind.
+        :param path: Local path to write the object's content to. The object
+            is assembled out of the way and moved into place once complete,
+            so ``path`` never holds a partial object: an existing file is
+            replaced atomically, and a failed transfer leaves it untouched.
+            Concurrent downloads to one path are therefore safe, with the
+            last to complete winning.
 
         :type config: :class:`.TransferConfig`
         :param config: Optional per-call transfer tuning (threshold, part

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge/providers/aws/resources.py

@@ -944,8 +944,8 @@ class AWSBucketObject(BaseBucketObject):
         self._obj.upload_file(path, Config=transfer_config)
         return self
 
-    def download_to_file(self, path: str,
-                         config: TransferConfig | None = None) -> None:
+    def _download_to_path(self, path: str,
+                          config: TransferConfig | None = None) -> None:
         # boto3's TransferManager downloads large objects as parallel ranged
         # GETs with a thread-safe client, so the transparent ranged path
         # delegates to it rather than CloudBridge's generic clone-pool driver.

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge/providers/aws/services.py

@@ -1023,6 +1023,9 @@ class AWSVMTypeService(BaseVMTypeService):
 
     def __init__(self, provider: CloudProvider) -> None:
         super(AWSVMTypeService, self).__init__(provider)
+        # Raw instance type dicts, keyed by availability zone. See
+        # _get_catalogue for why this is memoised.
+        self._catalogue: dict[str | None, list[dict[str, Any]]] = {}
 
     @dispatch(event="provider.compute.vm_types.get",
               priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
@@ -1039,15 +1042,12 @@ class AWSVMTypeService(BaseVMTypeService):
             else:
                 raise e
 
-    @dispatch(event="provider.compute.vm_types.list",
-              priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
-    def list(self, limit: int | None = None,
-             marker: str | None = None) -> ResultList[VMType]:
+    def _fetch_catalogue(self, zone: str | None) -> list[dict[str, Any]]:
         client = cast("AWSCloudProvider", self.provider).ec2_conn.meta.client
         vmt_list_resp = client.describe_instance_type_offerings(
             LocationType='availability-zone',
             Filters=[{'Name': 'location',
-                      'Values': [self.provider.zone_name]}],
+                      'Values': [zone]}],
             # MaxResults is set to max value (1000)
             # and client-side pagination is used
             **trim_empty_params({'MaxResults': 1000, 'NextToken': None}))
@@ -1056,7 +1056,7 @@ class AWSVMTypeService(BaseVMTypeService):
             vmt_list_resp = client.describe_instance_type_offerings(
                 LocationType='availability-zone',
                 Filters=[{'Name': 'location',
-                          'Values': [self.provider.zone_name]}],
+                          'Values': [zone]}],
                 **trim_empty_params(
                     {'MaxResults': 1000,
                      'NextToken': vmt_list_resp.get("NextToken")}))
@@ -1067,12 +1067,41 @@ class AWSVMTypeService(BaseVMTypeService):
         # describe_instance_types call can get at most 100 types at once
         chunks = [vmt_list_names[x:x + 100]
                   for x in range(0, len(vmt_list_names), 100)]
-        raw_types = []
+        raw_types: list[dict[str, Any]] = []
         for chunk in chunks:
             raw_chunk = client.describe_instance_types(
                 InstanceTypes=chunk).get('InstanceTypes')
             raw_types.extend(raw_chunk)
-        cb_types = [AWSVMType(cast("AWSCloudProvider", self.provider), t) for t in raw_types]
+        return raw_types
+
+    def _get_catalogue(self) -> list[dict[str, Any]]:
+        """
+        Return the raw instance type catalogue for the provider's zone,
+        fetching it at most once per zone.
+
+        EC2 has no server-side paging for instance types, so ``list()`` must
+        materialise the whole catalogue and page it client-side. Fetching it
+        costs one ``DescribeInstanceTypeOfferings`` walk plus one
+        ``DescribeInstanceTypes`` call per 100 types — around 14 calls for a
+        real region. Without memoisation every page of a ``list(marker=...)``
+        walk repeats all of that, making a full walk quadratic in API calls:
+        1343 types at a result limit of 5 costs ~4300 calls instead of ~14.
+
+        The catalogue is static for the lifetime of a provider, so it is held
+        per zone (a provider may be cloned to another zone, which genuinely
+        offers a different set of types).
+        """
+        zone = self.provider.zone_name
+        if zone not in self._catalogue:
+            self._catalogue[zone] = self._fetch_catalogue(zone)
+        return self._catalogue[zone]
+
+    @dispatch(event="provider.compute.vm_types.list",
+              priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
+    def list(self, limit: int | None = None,
+             marker: str | None = None) -> ResultList[VMType]:
+        cb_types = [AWSVMType(cast("AWSCloudProvider", self.provider), t)
+                    for t in self._get_catalogue()]
         return ClientPagedResultList(self.provider, cb_types,
                                      limit=limit, marker=marker)
 
@@ -1674,6 +1703,14 @@ class AWSDnsZoneService(BaseDnsZoneService):
             client.delete_hosted_zone(Id=dns_zone.aws_id)
 
 
+# Route53 reports record changes INSYNC within seconds, but boto3's
+# resource_record_sets_changed waiter polls every 30s by default, so every
+# record change costs a full 30s of sleep no matter how fast it propagated.
+# Poll often enough that the granularity stops dominating, while keeping the
+# same ~30 minute ceiling for changes that genuinely are slow.
+DNS_CHANGE_WAITER_CONFIG = {'Delay': 5, 'MaxAttempts': 360}
+
+
 class AWSDnsRecordService(BaseDnsRecordService):
 
     def __init__(self, provider: CloudProvider) -> None:
@@ -1765,7 +1802,8 @@ class AWSDnsRecordService(BaseDnsRecordService):
         # waiting, this is skipped for mock tests.
         if not cast("AWSCloudProvider", self.provider).PROVIDER_ID == 'mock':
             waiter = client.get_waiter('resource_record_sets_changed')
-            waiter.wait(Id=response.get('ChangeInfo').get('Id'))
+            waiter.wait(Id=response.get('ChangeInfo').get('Id'),
+                        WaiterConfig=DNS_CHANGE_WAITER_CONFIG)
         return cast(DnsRecord, self.get(dns_zone, name + ":" + type))
 
     def delete(self, dns_zone: DnsZone | str,
@@ -1793,4 +1831,5 @@ class AWSDnsRecordService(BaseDnsRecordService):
         # waiting, this is skipped for mock tests.
         if not cast("AWSCloudProvider", self.provider).PROVIDER_ID == 'mock':
             waiter = client.get_waiter('resource_record_sets_changed')
-            waiter.wait(Id=response.get('ChangeInfo').get('Id'))
+            waiter.wait(Id=response.get('ChangeInfo').get('Id'),
+                        WaiterConfig=DNS_CHANGE_WAITER_CONFIG)

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge/providers/azure/resources.py

@@ -302,8 +302,8 @@ class AzureBucketObject(BaseBucketObject):
             max_concurrency=self._multipart_max_concurrency(config))
         return self
 
-    def download_to_file(self, path: str,
-                         config: TransferConfig | None = None) -> None:
+    def _download_to_path(self, path: str,
+                          config: TransferConfig | None = None) -> None:
         # azure-storage-blob's downloader fetches block ranges concurrently
         # with a thread-safe client, so delegate to it rather than
         # CloudBridge's generic clone-pool driver.

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudbridge
-Version: 4.3.0
+Version: 4.3.1
 Summary: A simple layer of abstraction over multiple cloud providers.
 Author-email: Galaxy and GVL Projects <help@genome.edu.au>
 License-Expression: MIT

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/cloudbridge.egg-info/SOURCES.txt

@@ -51,6 +51,8 @@ cloudbridge/providers/openstack/provider.py
 cloudbridge/providers/openstack/resources.py
 cloudbridge/providers/openstack/services.py
 cloudbridge/providers/openstack/subservices.py
+tests/test_aws_dns_waiters.py
+tests/test_aws_vm_types.py
 tests/test_base_helpers.py
 tests/test_block_store_service.py
 tests/test_cloud_factory.py

cloudbridge-4.3.1/tests/test_aws_dns_waiters.py

@@ -0,0 +1,141 @@
+"""
+Unit tests for how ``AWSDnsRecordService`` waits on Route53 changes.
+
+Creating or deleting a record blocks until Route53 reports the change INSYNC.
+boto3's ``resource_record_sets_changed`` waiter polls every 30 seconds by
+default, so a change that propagates in a few seconds still costs a full 30 --
+and a test that makes four record changes pays 120 seconds of pure sleep.
+Measured against real Route53, INSYNC was reached within the first poll
+interval every time, making the granularity the entire cost.
+
+These tests drive the real botocore waiter against a simulated clock, so they
+assert on how long we *would* sleep without actually sleeping.
+"""
+import unittest
+from unittest import mock
+
+import botocore.client
+import botocore.waiter
+from botocore.exceptions import WaiterError
+
+from cloudbridge.providers.aws import AWSCloudProvider
+from cloudbridge.providers.aws.resources import AWSDnsRecord
+from cloudbridge.providers.aws.resources import AWSDnsZone
+from cloudbridge.providers.aws.services import AWSDnsRecordService
+
+# Simulated seconds before Route53 reports INSYNC. Real-world measurement put
+# this comfortably inside one 30s poll interval.
+INSYNC_AFTER = 6.0
+BOTO_DEFAULT_DELAY = 30.0
+# The waiter's ceiling must stay at roughly 30 minutes however it is polled.
+REQUIRED_CEILING = 1700.0
+
+ZONE = {'Id': '/hostedzone/Z1EXAMPLE', 'Name': 'example.com.'}
+RECORD = {'Name': 'sub.example.com.', 'Type': 'CNAME', 'TTL': 500,
+          'ResourceRecords': [{'Value': 'hello.com.'}]}
+
+
+class _Route53Sim:
+    """Canned Route53 responses driven by a simulated clock."""
+
+    def __init__(self, insync_after=INSYNC_AFTER):
+        self.insync_after = insync_after
+        self.clock = 0.0
+        self.sleeps = []
+        self.get_change_calls = 0
+
+    def api(self, operation_name, params):
+        if operation_name == 'ChangeResourceRecordSets':
+            return {'ChangeInfo': {'Id': '/change/C1', 'Status': 'PENDING'}}
+        if operation_name == 'GetChange':
+            self.get_change_calls += 1
+            status = ('INSYNC' if self.clock >= self.insync_after
+                      else 'PENDING')
+            return {'ChangeInfo': {'Id': '/change/C1', 'Status': status}}
+        if operation_name == 'ListResourceRecordSets':
+            return {'ResourceRecordSets': [RECORD], 'IsTruncated': False}
+        raise AssertionError('unexpected operation: ' + operation_name)
+
+    def sleep(self, secs):
+        self.sleeps.append(secs)
+        self.clock += secs
+
+    @property
+    def total_wait(self):
+        return sum(self.sleeps)
+
+
+def _provider():
+    return AWSCloudProvider({'aws_access_key': 'dummy',
+                             'aws_secret_key': 'dummy',
+                             'aws_zone_name': 'us-east-1a'})
+
+
+def _run(sim, fn):
+    """Run fn with Route53 stubbed and the waiter's clock simulated."""
+    with mock.patch.object(botocore.client.BaseClient, '_make_api_call',
+                           lambda self, op, params: sim.api(op, params)), \
+            mock.patch.object(botocore.waiter.time, 'sleep', sim.sleep):
+        return fn()
+
+
+class AWSDnsWaiterTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.provider = _provider()
+        self.svc = AWSDnsRecordService(self.provider)
+        self.zone = AWSDnsZone(self.provider, ZONE)
+        self.record = AWSDnsRecord(self.provider, self.zone, RECORD)
+
+    def test_create_does_not_burn_a_full_poll_interval_on_a_fast_change(self):
+        sim = _Route53Sim()
+
+        _run(sim, lambda: self.svc.create(
+            self.zone, 'sub.example.com.', 'CNAME', 'hello.com', ttl=500))
+
+        self.assertLess(
+            sim.total_wait, BOTO_DEFAULT_DELAY,
+            "A change that went INSYNC after %ss cost %ss of sleep; the "
+            "waiter is still polling at boto3's %ss default"
+            % (INSYNC_AFTER, sim.total_wait, BOTO_DEFAULT_DELAY))
+
+    def test_delete_does_not_burn_a_full_poll_interval_on_a_fast_change(self):
+        sim = _Route53Sim()
+
+        _run(sim, lambda: self.svc.delete(self.zone, self.record))
+
+        self.assertLess(
+            sim.total_wait, BOTO_DEFAULT_DELAY,
+            "A change that went INSYNC after %ss cost %ss of sleep; the "
+            "waiter is still polling at boto3's %ss default"
+            % (INSYNC_AFTER, sim.total_wait, BOTO_DEFAULT_DELAY))
+
+    def test_waiter_polls_until_the_change_is_actually_insync(self):
+        """Faster polling must not mean giving up early."""
+        sim = _Route53Sim(insync_after=47.0)
+
+        _run(sim, lambda: self.svc.create(
+            self.zone, 'sub.example.com.', 'CNAME', 'hello.com', ttl=500))
+
+        self.assertGreaterEqual(sim.clock, 47.0,
+                                "Returned before the change was INSYNC")
+        self.assertGreater(sim.get_change_calls, 1)
+
+    def test_waiter_ceiling_is_still_about_thirty_minutes(self):
+        """Polling more often must not shrink how long we are willing to
+        wait -- a genuinely slow change should still be given ~30 minutes
+        before the waiter gives up."""
+        sim = _Route53Sim(insync_after=float('inf'))
+
+        with self.assertRaises(WaiterError):
+            _run(sim, lambda: self.svc.create(
+                self.zone, 'sub.example.com.', 'CNAME', 'hello.com', ttl=500))
+
+        self.assertGreaterEqual(
+            sim.total_wait, REQUIRED_CEILING,
+            "Waiter gave up after only %ss of simulated waiting"
+            % sim.total_wait)
+
+
+if __name__ == '__main__':
+    unittest.main()

cloudbridge-4.3.1/tests/test_aws_vm_types.py

@@ -0,0 +1,166 @@
+"""
+Unit tests for ``AWSVMTypeService`` catalogue retrieval.
+
+AWS has no server-side pagination for VM types, so ``list()`` materialises the
+whole catalogue and hands it to ``ClientPagedResultList`` for client-side
+paging. Fetching that catalogue is expensive: one
+``DescribeInstanceTypeOfferings`` walk plus a ``DescribeInstanceTypes`` call
+per 100 types (~14 calls for a real region). Re-fetching it for every page
+makes walking the full list quadratic in API calls, so these tests pin the
+catalogue down to a single fetch per zone.
+
+Exercised against an in-memory fake connection so they run in CI without
+cloud credentials.
+"""
+import unittest
+
+from cloudbridge.providers.aws import AWSCloudProvider
+
+# Enough types to force several pages at the tests' small result limit, and
+# more than one 100-type DescribeInstanceTypes chunk.
+CATALOGUE_SIZE = 250
+CHUNK_SIZE = 100
+PAGE_SIZE = 5
+
+
+class _FakeEC2Client:
+    """Records every catalogue call made against it."""
+
+    def __init__(self, types_by_zone):
+        self._types_by_zone = types_by_zone
+        self.offering_calls = []          # zone per call
+        self.describe_type_calls = []     # list of requested type names
+
+    def describe_instance_type_offerings(self, **kwargs):
+        zone = kwargs['Filters'][0]['Values'][0]
+        self.offering_calls.append(zone)
+        names = self._types_by_zone[zone]
+        return {'InstanceTypeOfferings': [{'InstanceType': n} for n in names]}
+
+    def describe_instance_types(self, **kwargs):
+        requested = kwargs['InstanceTypes']
+        self.describe_type_calls.append(requested)
+        return {'InstanceTypes': [{'InstanceType': n,
+                                   'CurrentGeneration': True,
+                                   'VCpuInfo': {'DefaultVCpus': 2}}
+                                  for n in requested]}
+
+
+class _FakeEC2Conn:
+    def __init__(self, client):
+        self.meta = type('_Meta', (), {'client': client})()
+
+
+def _make_provider(zone, types_by_zone):
+    provider = AWSCloudProvider({
+        'aws_access_key': 'dummy',
+        'aws_secret_key': 'dummy',
+        'aws_zone_name': zone,
+        'default_result_limit': PAGE_SIZE,
+    })
+    client = _FakeEC2Client(types_by_zone)
+    # ec2_conn is a lazily-populated property backed by this attribute.
+    provider._ec2_conn = _FakeEC2Conn(client)
+    return provider, client
+
+
+def _zone_types(prefix, count=CATALOGUE_SIZE):
+    return ['{0}.type{1}'.format(prefix, i) for i in range(count)]
+
+
+class AWSVMTypeCatalogueTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.zone = 'us-east-1a'
+        self.types = {self.zone: _zone_types('a')}
+
+    def _calls_for_one_catalogue_fetch(self):
+        expected_chunks = -(-CATALOGUE_SIZE // CHUNK_SIZE)  # ceil div
+        return 1, expected_chunks
+
+    def test_single_list_fetches_catalogue_once(self):
+        provider, client = _make_provider(self.zone, self.types)
+
+        provider.compute.vm_types.list()
+
+        offerings, chunks = self._calls_for_one_catalogue_fetch()
+        self.assertEqual(len(client.offering_calls), offerings)
+        self.assertEqual(len(client.describe_type_calls), chunks)
+
+    def test_repeated_list_calls_reuse_the_catalogue(self):
+        provider, client = _make_provider(self.zone, self.types)
+
+        provider.compute.vm_types.list()
+        provider.compute.vm_types.list()
+        provider.compute.vm_types.list()
+
+        offerings, chunks = self._calls_for_one_catalogue_fetch()
+        self.assertEqual(
+            len(client.offering_calls), offerings,
+            "Catalogue offerings should be fetched once and reused, but were "
+            "fetched %s times" % len(client.offering_calls))
+        self.assertEqual(
+            len(client.describe_type_calls), chunks,
+            "Instance type details should be fetched once and reused, but "
+            "%s calls were made" % len(client.describe_type_calls))
+
+    def test_paging_through_all_pages_does_not_refetch_catalogue(self):
+        """The cost of walking every page must not scale with the page count.
+
+        This is the access pattern used by the standard-behaviour test helper
+        (``check_list``) and it is what made the AWS suite take ~50 minutes.
+        """
+        provider, client = _make_provider(self.zone, self.types)
+
+        result = provider.compute.vm_types.list()
+        pages = 1
+        while result.is_truncated:
+            result = provider.compute.vm_types.list(marker=result.marker)
+            pages += 1
+
+        self.assertEqual(pages, -(-CATALOGUE_SIZE // PAGE_SIZE),
+                         "Expected to walk the whole catalogue")
+        offerings, chunks = self._calls_for_one_catalogue_fetch()
+        self.assertEqual(
+            len(client.offering_calls), offerings,
+            "Walking %s pages refetched the offerings %s times"
+            % (pages, len(client.offering_calls)))
+        self.assertEqual(
+            len(client.describe_type_calls), chunks,
+            "Walking %s pages made %s DescribeInstanceTypes calls; the "
+            "catalogue should be fetched once"
+            % (pages, len(client.describe_type_calls)))
+
+    def test_list_still_returns_the_full_catalogue_contents(self):
+        """Caching must not change what callers observe."""
+        provider, client = _make_provider(self.zone, self.types)
+
+        seen = []
+        result = provider.compute.vm_types.list()
+        seen.extend(t.id for t in result)
+        while result.is_truncated:
+            result = provider.compute.vm_types.list(marker=result.marker)
+            seen.extend(t.id for t in result)
+
+        self.assertEqual(seen, self.types[self.zone])
+
+    def test_catalogue_is_keyed_by_zone(self):
+        """A provider cloned to another zone must not reuse the first zone's
+        catalogue — zones genuinely offer different instance types."""
+        other_zone = 'us-east-1b'
+        types = {self.zone: _zone_types('a'), other_zone: _zone_types('b')}
+        provider, client = _make_provider(self.zone, types)
+
+        first = [t.id for t in provider.compute.vm_types.list()]
+
+        provider.config['aws_zone_name'] = other_zone
+        provider._zone_name = other_zone
+        second = [t.id for t in provider.compute.vm_types.list()]
+
+        self.assertEqual(first, types[self.zone][:PAGE_SIZE])
+        self.assertEqual(second, types[other_zone][:PAGE_SIZE])
+        self.assertEqual(client.offering_calls, [self.zone, other_zone])
+
+
+if __name__ == '__main__':
+    unittest.main()

{cloudbridge-4.3.0 → cloudbridge-4.3.1}/tests/test_download_driver.py

@@ -9,6 +9,7 @@ driver is exercised here directly against in-memory fakes so it has coverage
 in CI without cloud credentials.
 """
 import os
+import shutil
 import tempfile
 import threading
 import unittest
@@ -32,12 +33,15 @@ class _Recorder:
         self.active = 0
         self.max_active = 0
         self.fail_on_offset = None  # offset that should raise
+        self.on_serve = None        # hook called as each range is served
 
     def serve_range(self, service, offset, length):
         with self._lock:
             self.active += 1
             self.max_active = max(self.max_active, self.active)
         try:
+            if self.on_serve:
+                self.on_serve()
             if self.fail_on_offset == offset:
                 raise RuntimeError("boom at offset %d" % offset)
             # Hold briefly so concurrent fetches genuinely overlap.
@@ -222,6 +226,72 @@ class DownloadDriverTestCase(unittest.TestCase):
             if os.path.exists(path):
                 os.remove(path)
 
+    def test_destination_only_appears_once_complete(self):
+        content = bytes(range(256))
+        recorder = _Recorder(content)
+        driver = self._driver(
+            recorder, threshold=1, part_size=16, concurrency=3)
+        fd, path = tempfile.mkstemp()
+        os.close(fd)
+        os.remove(path)
+        seen_early = []
+        recorder.on_serve = lambda: seen_early.append(os.path.exists(path))
+        try:
+            driver.download_to_file(path)
+            with open(path, 'rb') as f:
+                self.assertEqual(f.read(), content)
+        finally:
+            if os.path.exists(path):
+                os.remove(path)
+        # A partially written object is never visible at the destination.
+        self.assertTrue(seen_early)
+        self.assertNotIn(True, seen_early)
+
+    def test_survives_concurrent_downloader_taking_the_destination(self):
+        # Galaxy gives every download of a dataset the same cache .tmp path,
+        # so a second download of the same dataset can rename the destination
+        # away while this one is still fetching ranges.
+        content = bytes(range(256))
+        recorder = _Recorder(content)
+        driver = self._driver(
+            recorder, threshold=1, part_size=16, concurrency=3)
+        directory = tempfile.mkdtemp()
+        path = os.path.join(directory, 'dataset.dat')
+        taken = os.path.join(directory, 'taken.dat')
+
+        def steal_destination():
+            if os.path.exists(path):
+                os.replace(path, taken)
+
+        recorder.on_serve = steal_destination
+        try:
+            driver.download_to_file(path)
+            with open(path, 'rb') as f:
+                self.assertEqual(f.read(), content)
+        finally:
+            shutil.rmtree(directory)
+
+    def test_failed_download_leaves_an_existing_destination_intact(self):
+        content = bytes(range(64))
+        recorder = _Recorder(content)
+        recorder.fail_on_offset = 16
+        driver = self._driver(
+            recorder, threshold=1, part_size=16, concurrency=2)
+        directory = tempfile.mkdtemp()
+        path = os.path.join(directory, 'dataset.dat')
+        with open(path, 'wb') as f:
+            f.write(b'previously cached')
+        try:
+            with self.assertRaises(Exception):
+                driver.download_to_file(path)
+            # The cached copy survives a failed refetch, and no scratch file
+            # is left behind next to it.
+            with open(path, 'rb') as f:
+                self.assertEqual(f.read(), b'previously cached')
+            self.assertEqual(os.listdir(directory), ['dataset.dat'])
+        finally:
+            shutil.rmtree(directory)
+
     def test_part_size_must_be_positive(self):
         content = bytes(range(16))
         recorder = _Recorder(content)