cloudbridge 4.4.2__tar.gz → 4.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/PKG-INFO +1 -1
  2. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/__init__.py +1 -1
  3. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/base/resources.py +59 -3
  4. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/base/subservices.py +16 -0
  5. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/interfaces/exceptions.py +9 -0
  6. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/interfaces/resources.py +139 -7
  7. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/interfaces/services.py +43 -0
  8. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/interfaces/subservices.py +61 -0
  9. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/aws/services.py +76 -6
  10. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/azure/azure_client.py +32 -14
  11. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/azure/services.py +77 -4
  12. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/gcp/helpers.py +7 -12
  13. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/gcp/services.py +112 -25
  14. cloudbridge-4.5.0/cloudbridge/providers/openstack/helpers.py +92 -0
  15. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/openstack/resources.py +4 -14
  16. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/openstack/services.py +70 -3
  17. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge.egg-info/PKG-INFO +1 -1
  18. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge.egg-info/SOURCES.txt +2 -0
  19. cloudbridge-4.5.0/tests/test_gcp_presigned_uploads.py +265 -0
  20. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_object_store_service.py +199 -0
  21. cloudbridge-4.5.0/tests/test_openstack_temp_url_key.py +126 -0
  22. cloudbridge-4.4.2/cloudbridge/providers/openstack/helpers.py +0 -47
  23. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/LICENSE +0 -0
  24. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/README.rst +0 -0
  25. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/base/__init__.py +0 -0
  26. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/base/helpers.py +0 -0
  27. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/base/middleware.py +0 -0
  28. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/base/provider.py +0 -0
  29. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/base/services.py +0 -0
  30. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/factory.py +0 -0
  31. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/interfaces/__init__.py +0 -0
  32. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/interfaces/provider.py +0 -0
  33. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/__init__.py +0 -0
  34. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/aws/__init__.py +0 -0
  35. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/aws/helpers.py +0 -0
  36. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/aws/provider.py +0 -0
  37. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/aws/resources.py +0 -0
  38. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/aws/subservices.py +0 -0
  39. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/azure/__init__.py +0 -0
  40. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/azure/helpers.py +0 -0
  41. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/azure/provider.py +0 -0
  42. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/azure/resources.py +0 -0
  43. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/azure/subservices.py +0 -0
  44. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/gcp/__init__.py +0 -0
  45. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/gcp/provider.py +0 -0
  46. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/gcp/resources.py +0 -0
  47. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/gcp/subservices.py +0 -0
  48. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/mock/__init__.py +0 -0
  49. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/mock/provider.py +0 -0
  50. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/openstack/__init__.py +0 -0
  51. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/openstack/provider.py +0 -0
  52. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/providers/openstack/subservices.py +0 -0
  53. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge/py.typed +0 -0
  54. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge.egg-info/dependency_links.txt +0 -0
  55. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge.egg-info/requires.txt +0 -0
  56. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/cloudbridge.egg-info/top_level.txt +0 -0
  57. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/pyproject.toml +0 -0
  58. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/setup.cfg +0 -0
  59. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_aws_dns_waiters.py +0 -0
  60. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_aws_pagination.py +0 -0
  61. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_aws_vm_types.py +0 -0
  62. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_base_helpers.py +0 -0
  63. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_block_store_service.py +0 -0
  64. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_cloud_factory.py +0 -0
  65. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_cloud_helpers.py +0 -0
  66. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_compute_service.py +0 -0
  67. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_default_network.py +0 -0
  68. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_dns_service.py +0 -0
  69. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_download_driver.py +0 -0
  70. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_gcp_metadata_save.py +0 -0
  71. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_image_service.py +0 -0
  72. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_interface.py +0 -0
  73. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_iter_content.py +0 -0
  74. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_middleware_system.py +0 -0
  75. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_multipart_driver.py +0 -0
  76. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_network_service.py +0 -0
  77. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_object_life_cycle.py +0 -0
  78. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_openstack_credentials.py +0 -0
  79. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_region_service.py +0 -0
  80. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_security_service.py +0 -0
  81. {cloudbridge-4.4.2 → cloudbridge-4.5.0}/tests/test_vm_types_service.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cloudbridge
3
- Version: 4.4.2
3
+ Version: 4.5.0
4
4
  Summary: A simple layer of abstraction over multiple cloud providers.
5
5
  Author-email: Galaxy and GVL Projects <help@genome.edu.au>
6
6
  License-Expression: MIT
@@ -3,7 +3,7 @@ import logging
3
3
  from typing import Any
4
4
 
5
5
  # Current version of the library
6
- __version__ = '4.4.2'
6
+ __version__ = '4.5.0'
7
7
 
8
8
 
9
9
  def get_version() -> str:
@@ -52,6 +52,7 @@ from cloudbridge.interfaces.resources import NetworkState
52
52
  from cloudbridge.interfaces.resources import ObjectLifeCycleMixin
53
53
  from cloudbridge.interfaces.resources import PageableObjectMixin
54
54
  from cloudbridge.interfaces.resources import PlacementZone
55
+ from cloudbridge.interfaces.resources import PresignedRequest
55
56
  from cloudbridge.interfaces.resources import Region
56
57
  from cloudbridge.interfaces.resources import ResultList
57
58
  from cloudbridge.interfaces.resources import Router
@@ -734,16 +735,54 @@ class BaseRegion(BaseCloudResource, Region):
734
735
  return next(iter(self.zones))
735
736
 
736
737
 
738
+ class BasePresignedRequest(PresignedRequest):
739
+ """
740
+ A presigned request, as concrete providers return it from
741
+ ``generate_upload_url`` and ``generate_part_url``.
742
+ """
743
+
744
+ def __init__(self, method: str, url: str,
745
+ headers: dict[str, str] | None = None,
746
+ enforces_size: bool = False) -> None:
747
+ self._method = method
748
+ self._url = url
749
+ self._headers = dict(headers or {})
750
+ self._enforces_size = enforces_size
751
+
752
+ @property
753
+ def method(self) -> str:
754
+ return self._method
755
+
756
+ @property
757
+ def url(self) -> str:
758
+ return self._url
759
+
760
+ @property
761
+ def headers(self) -> dict[str, str]:
762
+ return self._headers
763
+
764
+ @property
765
+ def enforces_size(self) -> bool:
766
+ return self._enforces_size
767
+
768
+ def __repr__(self) -> str:
769
+ # The query string carries the signature.
770
+ return "<CB-{0}: {1} {2}>".format(
771
+ self.__class__.__name__, self._method, self._url.split('?', 1)[0])
772
+
773
+
737
774
  class BaseUploadPart(UploadPart):
738
775
  """
739
776
  A simple, serializable handle for a single uploaded part. Concrete
740
- providers return these from ``upload_part`` and consume them in
741
- ``complete_multipart_upload``.
777
+ providers return these from ``upload_part``, ``reported_part`` and
778
+ ``list_parts``, and consume them in ``complete_multipart_upload``.
742
779
  """
743
780
 
744
- def __init__(self, part_number: int, etag: object) -> None:
781
+ def __init__(self, part_number: int, etag: object,
782
+ size: int | None = None) -> None:
745
783
  self._part_number = part_number
746
784
  self._etag = etag
785
+ self._size = size
747
786
 
748
787
  @property
749
788
  def part_number(self) -> int:
@@ -753,6 +792,10 @@ class BaseUploadPart(UploadPart):
753
792
  def etag(self) -> object:
754
793
  return self._etag
755
794
 
795
+ @property
796
+ def size(self) -> int | None:
797
+ return self._size
798
+
756
799
  def __repr__(self) -> str:
757
800
  return "<CB-{0}: {1} ({2})>".format(
758
801
  self.__class__.__name__, self._part_number, self._etag)
@@ -797,6 +840,19 @@ class BaseMultipartUpload(BaseCloudResource, MultipartUpload):
797
840
  return self._bucket_objects.upload_part(
798
841
  self._bucket, self, part_number, data)
799
842
 
843
+ def generate_part_url(self, part_number: int, expires_in: int,
844
+ size: int | None = None) -> PresignedRequest:
845
+ return self._bucket_objects.generate_part_url(
846
+ self._bucket, self, part_number, expires_in, size)
847
+
848
+ def reported_part(self, part_number: int,
849
+ etag: str | None) -> UploadPart:
850
+ return self._bucket_objects.reported_part(
851
+ self._bucket, self, part_number, etag)
852
+
853
+ def list_parts(self) -> list[UploadPart]:
854
+ return self._bucket_objects.list_parts(self._bucket, self)
855
+
800
856
  def complete(self, parts: list[UploadPart]) -> BucketObject:
801
857
  # pylint:disable=protected-access
802
858
  return self._bucket_objects.complete_multipart_upload(
@@ -12,7 +12,9 @@ from cloudbridge.interfaces.resources import DnsZone
12
12
  from cloudbridge.interfaces.resources import FloatingIP
13
13
  from cloudbridge.interfaces.resources import Gateway
14
14
  from cloudbridge.interfaces.resources import InternetGateway
15
+ from cloudbridge.interfaces.resources import MultipartUpload
15
16
  from cloudbridge.interfaces.resources import Network
17
+ from cloudbridge.interfaces.resources import PresignedRequest
16
18
  from cloudbridge.interfaces.resources import ResultList
17
19
  from cloudbridge.interfaces.resources import Subnet
18
20
  from cloudbridge.interfaces.resources import TrafficDirection
@@ -26,6 +28,7 @@ from cloudbridge.interfaces.subservices import GatewaySubService
26
28
  from cloudbridge.interfaces.subservices import SubnetSubService
27
29
  from cloudbridge.interfaces.subservices import VMFirewallRuleSubService
28
30
 
31
+ from .resources import BaseMultipartUpload
29
32
  from .resources import BasePageableObjectMixin
30
33
 
31
34
  if TYPE_CHECKING:
@@ -66,6 +69,19 @@ class BaseBucketObjectSubService(BasePageableObjectMixin[BucketObject],
66
69
  def create(self, name: str) -> BucketObject:
67
70
  return self._bucket_objects.create(self.bucket, name)
68
71
 
72
+ def generate_upload_url(self, name: str, expires_in: int,
73
+ size: int | None = None) -> PresignedRequest:
74
+ return self._bucket_objects.generate_upload_url(
75
+ self.bucket, name, expires_in, size)
76
+
77
+ def create_multipart_upload(self, name: str) -> MultipartUpload:
78
+ return self._bucket_objects.create_multipart_upload(self.bucket, name)
79
+
80
+ def get_multipart_upload(self, name: str,
81
+ upload_id: str) -> MultipartUpload:
82
+ return BaseMultipartUpload(self._provider, self.bucket, name,
83
+ upload_id)
84
+
69
85
 
70
86
  class BaseGatewaySubService(GatewaySubService,
71
87
  BasePageableObjectMixin[InternetGateway]):
@@ -102,3 +102,12 @@ class InvalidParamException(InvalidNameException):
102
102
 
103
103
  def __init__(self, msg: str) -> None:
104
104
  super(InvalidParamException, self).__init__(msg)
105
+
106
+
107
+ class InvalidPartException(CloudBridgeBaseException):
108
+ """
109
+ Thrown when completing a multipart upload with parts the store does not
110
+ hold as reported: a part that was never uploaded, or one replaced after its
111
+ uploader received its ETag.
112
+ """
113
+ pass
@@ -2202,14 +2202,70 @@ class TransferConfig(object):
2202
2202
  self.threshold, self.part_size, self.max_concurrency))
2203
2203
 
2204
2204
 
2205
+ class PresignedRequest(object):
2206
+ """
2207
+ A request to the object store that anyone holding it can send, without
2208
+ credentials, until it expires.
2209
+
2210
+ Returned by :meth:`.BucketObjectSubService.generate_upload_url` and
2211
+ :meth:`.MultipartUpload.generate_part_url`. Send the payload as the body of
2212
+ a ``method`` request to ``url``, with ``headers``.
2213
+ """
2214
+ __metaclass__ = ABCMeta
2215
+
2216
+ @abstractproperty
2217
+ def method(self) -> str:
2218
+ """
2219
+ The HTTP method of the request.
2220
+
2221
+ :rtype: ``str``
2222
+ :return: The method, such as ``PUT``.
2223
+ """
2224
+ pass
2225
+
2226
+ @abstractproperty
2227
+ def url(self) -> str:
2228
+ """
2229
+ The URL to send the request to. It carries the signature.
2230
+
2231
+ :rtype: ``str``
2232
+ :return: The presigned URL.
2233
+ """
2234
+ pass
2235
+
2236
+ @abstractproperty
2237
+ def headers(self) -> dict[str, str]:
2238
+ """
2239
+ Headers the store requires, which an HTTP client would not send of
2240
+ its own accord.
2241
+
2242
+ :rtype: ``dict``
2243
+ :return: The names and values of the headers to send.
2244
+ """
2245
+ pass
2246
+
2247
+ @abstractproperty
2248
+ def enforces_size(self) -> bool:
2249
+ """
2250
+ Whether the store refuses a body of another size than the request was
2251
+ signed for. Where it doesn't, check the size after the upload.
2252
+
2253
+ :rtype: ``bool``
2254
+ :return: ``True`` if the store enforces the size.
2255
+ """
2256
+ pass
2257
+
2258
+
2205
2259
  class UploadPart(object):
2206
2260
  """
2207
2261
  A handle for a single part uploaded as part of a multipart upload.
2208
2262
 
2209
- Returned by :meth:`.MultipartUpload.upload_part`. Callers must retain the
2210
- parts they receive and pass them to :meth:`.MultipartUpload.complete`.
2211
- The handle is a simple, serializable value object so that parts may be
2212
- collected across threads or processes when uploading in parallel.
2263
+ Returned by :meth:`.MultipartUpload.upload_part`,
2264
+ :meth:`.MultipartUpload.reported_part` and
2265
+ :meth:`.MultipartUpload.list_parts`. Callers must retain the parts they
2266
+ receive and pass them to :meth:`.MultipartUpload.complete`. The handle is
2267
+ a simple, serializable value object so that parts may be collected across
2268
+ threads or processes when uploading in parallel.
2213
2269
  """
2214
2270
  __metaclass__ = ABCMeta
2215
2271
 
@@ -2229,7 +2285,7 @@ class UploadPart(object):
2229
2285
  Opaque provider handle identifying the stored part.
2230
2286
 
2231
2287
  Its concrete form varies by provider (e.g. an S3 ETag, an Azure block
2232
- id, a GCS temp-object name, or a Swift segment descriptor) and should
2288
+ id, a GCS part's MD5, or a Swift segment descriptor) and should
2233
2289
  not be interpreted by clients.
2234
2290
 
2235
2291
  :rtype: ``object``
@@ -2237,6 +2293,17 @@ class UploadPart(object):
2237
2293
  """
2238
2294
  pass
2239
2295
 
2296
+ @abstractproperty
2297
+ def size(self) -> int | None:
2298
+ """
2299
+ The size of the part in bytes, where known.
2300
+
2301
+ :rtype: ``int`` or ``None``
2302
+ :return: The part's size, as listed by :meth:`.MultipartUpload.list_parts`;
2303
+ ``None`` for a part reported by its uploader.
2304
+ """
2305
+ pass
2306
+
2240
2307
 
2241
2308
  class MultipartUpload(CloudResource):
2242
2309
  """
@@ -2289,17 +2356,82 @@ class MultipartUpload(CloudResource):
2289
2356
  """
2290
2357
  pass
2291
2358
 
2359
+ @abstractmethod
2360
+ def generate_part_url(self, part_number: int, expires_in: int,
2361
+ size: int | None = None) -> PresignedRequest:
2362
+ """
2363
+ Generate a request that uploads one part, for a client without
2364
+ credentials.
2365
+
2366
+ The client sends the part with the request and reports the ``ETag``
2367
+ response header back, to be passed to :meth:`reported_part`.
2368
+
2369
+ :type part_number: ``int``
2370
+ :param part_number: 1-based index of the part, as for
2371
+ :meth:`upload_part`.
2372
+
2373
+ :type expires_in: ``int``
2374
+ :param expires_in: Time to live of the request in seconds. The request
2375
+ can be sent again until it expires, replacing the part.
2376
+
2377
+ :type size: ``int``
2378
+ :param size: The part's size in bytes. Where the store can bind it
2379
+ into the signature (see :attr:`.PresignedRequest.enforces_size`),
2380
+ it refuses a part of another size.
2381
+
2382
+ :rtype: :class:`.PresignedRequest`
2383
+ :return: The request that uploads the part.
2384
+ """
2385
+ pass
2386
+
2387
+ @abstractmethod
2388
+ def reported_part(self, part_number: int,
2389
+ etag: str | None) -> UploadPart:
2390
+ """
2391
+ A handle for a part uploaded through :meth:`generate_part_url`, from
2392
+ what its uploader reported.
2393
+
2394
+ :meth:`complete` refuses the part if the store doesn't hold it, or,
2395
+ where the store keeps a checksum of each part, if the part it holds is
2396
+ not the one that received ``etag``: a part URL can be used again
2397
+ until it expires, to replace the part.
2398
+
2399
+ :type part_number: ``int``
2400
+ :param part_number: 1-based index of the part.
2401
+
2402
+ :type etag: ``str``
2403
+ :param etag: The ``ETag`` response header the part's upload received,
2404
+ as is; ``None`` if it had none.
2405
+
2406
+ :rtype: :class:`.UploadPart`
2407
+ :return: A part handle to pass to :meth:`complete`.
2408
+ """
2409
+ pass
2410
+
2411
+ @abstractmethod
2412
+ def list_parts(self) -> list[UploadPart]:
2413
+ """
2414
+ The parts the store holds for this upload, with their sizes.
2415
+
2416
+ :rtype: ``list`` of :class:`.UploadPart`
2417
+ :return: The uploaded parts, in ascending ``part_number``.
2418
+ """
2419
+ pass
2420
+
2292
2421
  @abstractmethod
2293
2422
  def complete(self, parts: list[UploadPart]) -> BucketObject:
2294
2423
  """
2295
2424
  Finalize the upload, assembling parts in ascending ``part_number``.
2296
2425
 
2297
2426
  :type parts: ``list`` of :class:`.UploadPart`
2298
- :param parts: The handles returned by :meth:`upload_part`. They may be
2299
- supplied in any order.
2427
+ :param parts: The handles returned by :meth:`upload_part` or
2428
+ :meth:`reported_part`. They may be supplied in any order.
2300
2429
 
2301
2430
  :rtype: :class:`.BucketObject`
2302
2431
  :return: The completed object.
2432
+
2433
+ :raises InvalidPartException: if the store doesn't hold a reported
2434
+ part as its uploader reported it.
2303
2435
  """
2304
2436
  pass
2305
2437
 
@@ -22,6 +22,7 @@ from cloudbridge.interfaces.resources import MachineImage
22
22
  from cloudbridge.interfaces.resources import MultipartUpload
23
23
  from cloudbridge.interfaces.resources import Network
24
24
  from cloudbridge.interfaces.resources import PageableObjectMixin
25
+ from cloudbridge.interfaces.resources import PresignedRequest
25
26
  from cloudbridge.interfaces.resources import Region
26
27
  from cloudbridge.interfaces.resources import ResultList
27
28
  from cloudbridge.interfaces.resources import Router
@@ -1348,6 +1349,48 @@ class BucketObjectService(CloudService):
1348
1349
  """
1349
1350
  pass
1350
1351
 
1352
+ @abstractmethod
1353
+ def generate_upload_url(self, bucket: Bucket | str, object_name: str,
1354
+ expires_in: int,
1355
+ size: int | None) -> PresignedRequest:
1356
+ """
1357
+ Generate a request that uploads an object without credentials.
1358
+
1359
+ See :meth:`.BucketObjectSubService.generate_upload_url`.
1360
+ """
1361
+ pass
1362
+
1363
+ @abstractmethod
1364
+ def generate_part_url(self, bucket: Bucket | str, upload: MultipartUpload,
1365
+ part_number: int, expires_in: int,
1366
+ size: int | None) -> PresignedRequest:
1367
+ """
1368
+ Generate a request that uploads one part without credentials.
1369
+
1370
+ See :meth:`.MultipartUpload.generate_part_url`.
1371
+ """
1372
+ pass
1373
+
1374
+ @abstractmethod
1375
+ def reported_part(self, bucket: Bucket | str, upload: MultipartUpload,
1376
+ part_number: int, etag: str | None) -> UploadPart:
1377
+ """
1378
+ A handle for a part uploaded through a part URL.
1379
+
1380
+ See :meth:`.MultipartUpload.reported_part`.
1381
+ """
1382
+ pass
1383
+
1384
+ @abstractmethod
1385
+ def list_parts(self, bucket: Bucket | str,
1386
+ upload: MultipartUpload) -> builtins.list[UploadPart]:
1387
+ """
1388
+ The parts the store holds for an upload, with their sizes.
1389
+
1390
+ See :meth:`.MultipartUpload.list_parts`.
1391
+ """
1392
+ pass
1393
+
1351
1394
  @abstractmethod
1352
1395
  def complete_multipart_upload(self, bucket: Bucket | str,
1353
1396
  upload: MultipartUpload,
@@ -10,7 +10,9 @@ from cloudbridge.interfaces.resources import DnsRecord
10
10
  from cloudbridge.interfaces.resources import FloatingIP
11
11
  from cloudbridge.interfaces.resources import Gateway
12
12
  from cloudbridge.interfaces.resources import InternetGateway
13
+ from cloudbridge.interfaces.resources import MultipartUpload
13
14
  from cloudbridge.interfaces.resources import PageableObjectMixin
15
+ from cloudbridge.interfaces.resources import PresignedRequest
14
16
  from cloudbridge.interfaces.resources import ResultList
15
17
  from cloudbridge.interfaces.resources import Subnet
16
18
  from cloudbridge.interfaces.resources import TrafficDirection
@@ -86,6 +88,65 @@ class BucketObjectSubService(PageableObjectMixin[BucketObject]):
86
88
  """
87
89
  pass
88
90
 
91
+ @abstractmethod
92
+ def generate_upload_url(self, name: str, expires_in: int,
93
+ size: int | None = None) -> PresignedRequest:
94
+ """
95
+ Generate a request that uploads an object, new or existing, for a
96
+ client without credentials.
97
+
98
+ Unlike :meth:`.BucketObject.generate_url`, it needs no object to exist
99
+ first, and creates none.
100
+
101
+ :type name: ``str``
102
+ :param name: The name of the object to upload.
103
+
104
+ :type expires_in: ``int``
105
+ :param expires_in: Time to live of the request in seconds.
106
+
107
+ :type size: ``int``
108
+ :param size: The object's size in bytes. Where the store can bind it
109
+ into the signature (see :attr:`.PresignedRequest.enforces_size`),
110
+ it refuses an object of another size.
111
+
112
+ :rtype: :class:`.PresignedRequest`
113
+ :return: The request that uploads the object.
114
+ """
115
+ pass
116
+
117
+ @abstractmethod
118
+ def create_multipart_upload(self, name: str) -> MultipartUpload:
119
+ """
120
+ Begin a multipart upload of an object, new or existing.
121
+
122
+ Unlike :meth:`.BucketObject.create_multipart_upload`, it needs no
123
+ object to exist first. The object appears once the upload completes.
124
+
125
+ :type name: ``str``
126
+ :param name: The name of the object to upload.
127
+
128
+ :rtype: :class:`.MultipartUpload`
129
+ :return: An in-progress multipart upload handle.
130
+ """
131
+ pass
132
+
133
+ @abstractmethod
134
+ def get_multipart_upload(self, name: str,
135
+ upload_id: str) -> MultipartUpload:
136
+ """
137
+ A handle for an upload begun earlier, possibly by another process.
138
+
139
+ :type name: ``str``
140
+ :param name: The name of the object being uploaded.
141
+
142
+ :type upload_id: ``str``
143
+ :param upload_id: The upload's :attr:`.MultipartUpload.id`.
144
+
145
+ :rtype: :class:`.MultipartUpload`
146
+ :return: The in-progress multipart upload handle.
147
+ """
148
+ pass
149
+
89
150
 
90
151
  class GatewaySubService(PageableObjectMixin[InternetGateway]):
91
152
  """
@@ -10,12 +10,15 @@ from typing import Any
10
10
  from typing import IO
11
11
  from typing import TYPE_CHECKING
12
12
  from typing import cast
13
+ from urllib.parse import parse_qs
14
+ from urllib.parse import urlparse
13
15
 
14
16
  from botocore.exceptions import ClientError
15
17
 
16
18
  import cloudbridge.base.helpers as cb_helpers
17
19
  from cloudbridge.base.middleware import dispatch
18
20
  from cloudbridge.base.resources import BaseMultipartUpload
21
+ from cloudbridge.base.resources import BasePresignedRequest
19
22
  from cloudbridge.base.resources import BaseUploadPart
20
23
  from cloudbridge.base.resources import ClientPagedResultList
21
24
  from cloudbridge.base.resources import ServerPagedResultList
@@ -47,6 +50,7 @@ from cloudbridge.interfaces.exceptions import DuplicateResourceException
47
50
  from cloudbridge.interfaces.exceptions import \
48
51
  InvalidConfigurationException
49
52
  from cloudbridge.interfaces.exceptions import InvalidParamException
53
+ from cloudbridge.interfaces.exceptions import InvalidPartException
50
54
  from cloudbridge.interfaces.exceptions import InvalidValueException
51
55
  from cloudbridge.interfaces.exceptions import ProviderInternalException
52
56
  from cloudbridge.interfaces.resources import Bucket
@@ -63,6 +67,7 @@ from cloudbridge.interfaces.resources import MachineImage
63
67
  from cloudbridge.interfaces.resources import MultipartUpload
64
68
  from cloudbridge.interfaces.resources import Network
65
69
  from cloudbridge.interfaces.resources import PlacementZone
70
+ from cloudbridge.interfaces.resources import PresignedRequest
66
71
  from cloudbridge.interfaces.resources import Region
67
72
  from cloudbridge.interfaces.resources import ResultList
68
73
  from cloudbridge.interfaces.resources import Router
@@ -706,6 +711,65 @@ class AWSBucketObjectService(BaseBucketObjectService):
706
711
  UploadId=upload.id, PartNumber=part_number, Body=data)
707
712
  return BaseUploadPart(part_number, response['ETag'])
708
713
 
714
+ def _presigned_put(self, client_method: str, params: dict[str, Any],
715
+ expires_in: int,
716
+ size: int | None) -> PresignedRequest:
717
+ if size is not None:
718
+ params['ContentLength'] = size
719
+ client = cast("AWSCloudProvider", self.provider).s3_conn.meta.client
720
+ url = client.generate_presigned_url(client_method, Params=params,
721
+ ExpiresIn=expires_in)
722
+ # SigV4 signs the size into the URL; SigV2 (s3_signature_version
723
+ # 's3') signs no headers.
724
+ signed_headers = parse_qs(urlparse(url).query).get(
725
+ 'X-Amz-SignedHeaders', [''])[0].split(';')
726
+ return BasePresignedRequest(
727
+ 'PUT', url, enforces_size='content-length' in signed_headers)
728
+
729
+ @dispatch(event="provider.storage._bucket_objects.generate_upload_url",
730
+ priority=BaseBucketObjectService.STANDARD_EVENT_PRIORITY)
731
+ def generate_upload_url(self, bucket: Bucket | str, object_name: str,
732
+ expires_in: int,
733
+ size: int | None) -> PresignedRequest:
734
+ return self._presigned_put(
735
+ 'put_object',
736
+ {'Bucket': cast(Any, bucket).name, 'Key': object_name},
737
+ expires_in, size)
738
+
739
+ @dispatch(event="provider.storage._bucket_objects.generate_part_url",
740
+ priority=BaseBucketObjectService.STANDARD_EVENT_PRIORITY)
741
+ def generate_part_url(self, bucket: Bucket | str, upload: MultipartUpload,
742
+ part_number: int, expires_in: int,
743
+ size: int | None) -> PresignedRequest:
744
+ return self._presigned_put(
745
+ 'upload_part',
746
+ {'Bucket': cast(Any, bucket).name, 'Key': upload.object_name,
747
+ 'UploadId': upload.id, 'PartNumber': part_number},
748
+ expires_in, size)
749
+
750
+ @dispatch(event="provider.storage._bucket_objects.reported_part",
751
+ priority=BaseBucketObjectService.STANDARD_EVENT_PRIORITY)
752
+ def reported_part(self, bucket: Bucket | str, upload: MultipartUpload,
753
+ part_number: int, etag: str | None) -> UploadPart:
754
+ if not etag:
755
+ raise InvalidPartException(
756
+ "Part {0} of {1} was reported without the ETag S3 returns for"
757
+ " every part".format(part_number, upload.object_name))
758
+ # S3 compares the ETag with the part it holds when the upload
759
+ # completes.
760
+ return BaseUploadPart(part_number, etag)
761
+
762
+ @dispatch(event="provider.storage._bucket_objects.list_parts",
763
+ priority=BaseBucketObjectService.STANDARD_EVENT_PRIORITY)
764
+ def list_parts(self, bucket: Bucket | str,
765
+ upload: MultipartUpload) -> builtins.list[UploadPart]:
766
+ client = cast("AWSCloudProvider", self.provider).s3_conn.meta.client
767
+ pages = client.get_paginator('list_parts').paginate(
768
+ Bucket=cast(Any, bucket).name, Key=upload.object_name,
769
+ UploadId=upload.id)
770
+ return [BaseUploadPart(part['PartNumber'], part['ETag'], part['Size'])
771
+ for page in pages for part in page.get('Parts', [])]
772
+
709
773
  @dispatch(event="provider.storage._bucket_objects."
710
774
  "complete_multipart_upload",
711
775
  priority=BaseBucketObjectService.STANDARD_EVENT_PRIORITY)
@@ -715,12 +779,18 @@ class AWSBucketObjectService(BaseBucketObjectService):
715
779
  ) -> BucketObject:
716
780
  ordered = sorted(parts, key=lambda p: p.part_number)
717
781
  client = cast("AWSCloudProvider", self.provider).s3_conn.meta.client
718
- client.complete_multipart_upload(
719
- Bucket=cast(Any, bucket).name, Key=upload.object_name,
720
- UploadId=upload.id,
721
- MultipartUpload={'Parts': [
722
- {'PartNumber': p.part_number, 'ETag': p.etag}
723
- for p in ordered]})
782
+ try:
783
+ client.complete_multipart_upload(
784
+ Bucket=cast(Any, bucket).name, Key=upload.object_name,
785
+ UploadId=upload.id,
786
+ MultipartUpload={'Parts': [
787
+ {'PartNumber': p.part_number, 'ETag': p.etag}
788
+ for p in ordered]})
789
+ except ClientError as e:
790
+ # A part S3 doesn't hold, or holds with another ETag.
791
+ if e.response['Error']['Code'] == 'InvalidPart':
792
+ raise InvalidPartException(str(e)) from e
793
+ raise
724
794
  # get() may return None, but the completed object is expected to exist.
725
795
  return cast(BucketObject, self.get(bucket, upload.object_name))
726
796
 
@@ -498,6 +498,12 @@ class AzureClient(object):
498
498
  block_list = [BlobBlock(block_id=block_id) for block_id in block_ids]
499
499
  blob_client.commit_block_list(block_list)
500
500
 
501
+ def get_uncommitted_blocks(self, container_name: str,
502
+ blob_name: str) -> list[BlobBlock]:
503
+ blob_client = self.blob_client(container_name, blob_name)
504
+ _, uncommitted = blob_client.get_block_list('uncommitted')
505
+ return uncommitted
506
+
501
507
  def get_blob(self, container_name: str, blob_name: str) -> Any:
502
508
  blob_client = self.blob_client(container_name, blob_name)
503
509
  return blob_client.get_blob_properties(container_name, blob_name)
@@ -517,23 +523,35 @@ class AzureClient(object):
517
523
  expiry_time: int, writable: bool,
518
524
  content_disposition: str | None = None,
519
525
  content_type: str | None = None) -> str:
520
- now = datetime.datetime.utcnow()
521
- expiry = now + datetime.timedelta(
522
- seconds=expiry_time)
523
- blob_name = blob_name
524
- container_name = container_name.name
526
+ return self._blob_sas_url(
527
+ container_name.name, blob_name, expiry_time,
528
+ BlobSasPermissions(read=True, write=writable),
529
+ content_disposition=content_disposition,
530
+ content_type=content_type)
531
+
532
+ def get_blob_upload_url(self, container_name: str, blob_name: str,
533
+ expiry_time: int) -> str:
534
+ """
535
+ A URL that writes the blob, new or existing, or a block of it, until it
536
+ expires.
537
+ """
538
+ return self._blob_sas_url(container_name, blob_name, expiry_time,
539
+ BlobSasPermissions(write=True))
540
+
541
+ def _blob_sas_url(self, container_name: str, blob_name: str,
542
+ expiry_time: int, permission: BlobSasPermissions,
543
+ **sas_options: Any) -> str:
544
+ now = datetime.datetime.now(datetime.timezone.utc)
545
+ expiry = now + datetime.timedelta(seconds=expiry_time)
525
546
  delegation_key = self.blob_service.get_user_delegation_key(
526
- key_start_time=now, key_expiry_time=expiry
527
- )
547
+ key_start_time=now, key_expiry_time=expiry)
528
548
  sas = generate_blob_sas(
529
549
  self.storage_account, container_name, blob_name,
530
- permission=BlobSasPermissions(read=True, write=writable), expiry=expiry,
531
- user_delegation_key=delegation_key,
532
- content_disposition=content_disposition,
533
- content_type=content_type
534
- )
535
- url = f"https://{self.storage_account}.blob.core.windows.net/{container_name}/{blob_name}?{sas}"
536
- return url
550
+ permission=permission, expiry=expiry,
551
+ user_delegation_key=delegation_key, **sas_options)
552
+ # The blob client's URL escapes the blob's name.
553
+ return "{0}?{1}".format(
554
+ self.blob_client(container_name, blob_name).url, sas)
537
555
 
538
556
  def create_empty_disk(self, disk_name: str, params: dict[str, Any]) -> Any:
539
557
  return self.compute_client.disks.begin_create_or_update(