dcor-shared 0.9.0__tar.gz → 0.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/CHANGELOG +7 -0
  2. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/PKG-INFO +1 -1
  3. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/_version_save.py +1 -1
  4. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/s3.py +149 -67
  5. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/s3cc.py +75 -0
  6. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/testing.py +67 -32
  7. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/PKG-INFO +1 -1
  8. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_s3.py +103 -45
  9. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_s3cc.py +31 -7
  10. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/LICENSE +0 -0
  11. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/MANIFEST.in +0 -0
  12. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/README.rst +0 -0
  13. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/__init__.py +0 -0
  14. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/_version.py +0 -0
  15. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/ckan.py +0 -0
  16. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/data.py +0 -0
  17. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/dcinst.py +0 -0
  18. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/mime.py +0 -0
  19. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/parse.py +0 -0
  20. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/paths.py +0 -0
  21. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/SOURCES.txt +0 -0
  22. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/dependency_links.txt +0 -0
  23. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/requires.txt +0 -0
  24. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/top_level.txt +0 -0
  25. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/setup.cfg +0 -0
  26. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/setup.py +0 -0
  27. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_basic.py +0 -0
  28. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_ckan.py +0 -0
  29. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_data.py +0 -0
  30. {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_testing.py +0 -0
@@ -1,3 +1,10 @@
1
+ 0.10.0
2
+ - BREAKING CHANGE: presigned URLs for uploading resources now work
3
+ differently. For small files (< 1 GiB), a single upload URL for
4
+ PUT is created. For larger files, a multipart upload is started.
5
+ - enh: retry set_object_tagging on failure
6
+ 0.9.1
7
+ - feat: add convenience method `get_s3_attributes_for_artifact`
1
8
  0.9.0
2
9
  - feat: introduce `testing.make_dataset_via_s3` and
3
10
  `testing.make_resource_via_s3` which uploads resources via S3
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcor_shared
3
- Version: 0.9.0
3
+ Version: 0.10.0
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Home-page: https://github.com/DCOR-dev/dcor_shared
6
6
  Author: Paul Müller
@@ -1,3 +1,3 @@
1
1
  #!/usr/bin/env python
2
2
  # This file was created automatically
3
- longversion = '0.9.0'
3
+ longversion = '0.10.0'
@@ -6,6 +6,8 @@ import hashlib
6
6
  import json
7
7
  import pathlib
8
8
  import time
9
+ import traceback
10
+ from typing import List, Tuple
9
11
  import warnings
10
12
 
11
13
  import boto3
@@ -45,9 +47,12 @@ def compute_checksum(bucket_name, object_name, max_size=None):
45
47
  return s3_sha256
46
48
 
47
49
 
48
- def create_presigned_upload_url(bucket_name, object_name,
49
- expiration=86400):
50
- """Create a presigned URL for uploading to S3
50
+ def create_presigned_upload_urls(
51
+ bucket_name: str,
52
+ object_name: str,
53
+ file_size: int,
54
+ expiration: int = 86400) -> Tuple[List, str]:
55
+ """Create presigned URLs for uploading an object to S3
51
56
 
52
57
  Parameters
53
58
  ----------
@@ -55,76 +60,144 @@ def create_presigned_upload_url(bucket_name, object_name,
55
60
  Name of the bucket
56
61
  object_name: str
57
62
  Name of the object
63
+ file_size: int
64
+ Size of the file in bytes; used to determine the number of
65
+ multipart upload parts with a hard-coded part size of
66
+ maximum 1 GiB. Reasons for choosing 1 GiB:
67
+ - Using the default value of 8 or 16 MiB leads to a lot parts
68
+ for a 20 GiB file
69
+ - The maximum allowed part size in S3 is 5 GiB
70
+ - Somebody on the internet said that Amazon is actually not
71
+ concatenating the parts into a new, larger file, but it keeps
72
+ the parts stored and just adds a layer in-between. This has
73
+ performance implications if you are using GET with ranges
74
+ to access partial data.
75
+ - I was originally thinking about 100 MiB, but having the number
76
+ of parts in the ETag as an estimate of the file size in GiB
77
+ seemed like a good idea.
58
78
  expiration: int
59
79
  Time in seconds for the presigned URL to remain valid
60
80
 
61
81
  Returns
62
82
  -------
63
- psurl: str
64
- Presigned upload URL as string (HTTPS://{DOMAIN}/{BUCKET})
65
- fields: dict
66
- Dictionary for `data` to use during upload:
67
- - key: identical to `object_name`
68
- - AWSAccessKeyId: S3 access key name for authentication
69
- - policy: base64-encoded policy (expiration, conditions: bucket, key)
70
- - signature: signature created by the server to verify the request
83
+ upload_urls: list
84
+ List of the presigned URLs required for the upload.
85
+ There will always be the key "urls" containing a list of
86
+ presigned URLs.
87
+ complete_url: str
88
+ If a multipart upload is necessary, this is the presigned URL
89
+ required to finalize the upload. For more information, see
90
+ `https://boto3.amazonaws.com/v1/documentation/api/latest/
91
+ reference/services/s3/client/complete_multipart_upload.html
92
+ #complete-multipart-upload`_ or the example below
71
93
 
72
94
  Notes
73
95
  -----
74
- By default, due to the access design in DCOR, the S3 URL of the
75
- uploaded resource `psurl/object_name` is private. If you would
76
- like to make the resource publicly accessible, you can call
77
- the method :func:`make_object_public`, which will add the `public=true`
78
- tag to the object.
96
+ By default, due to the access design in DCOR, the S3 URLs of the
97
+ uploaded resource is private. If you would like to make the resource
98
+ publicly accessible, you can call the method :func:`make_object_public`,
99
+ which will add the `public=true` tag to the object.
79
100
 
80
101
  Example
81
102
  -------
82
- To upload a file with :mod:`requests` and :mod:`requests_toolbelt`::
103
+ To upload a file with :mod:`requests`::
83
104
 
84
- import pathlib
85
105
  import requests
86
- from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
87
-
88
- # input file
89
- path_to_upload = pathlib.Path("/path/to/file")
90
- bucket_name = "circle-19082eua8sdj82"
91
- object_name = "resources/18/39/01923102983
92
- # obtain upload URL and credentials
93
- psurl, fields = create_presigned_upload_url(bucket_name, object_name)
94
- # callback function for monitoring the upload progress
95
- monitor_callback = lambda mon: print(f"Bytes: {mon.bytes_read}")
96
- # open the input file for streaming
97
- with path_to_upload.open("rb") as fd:
98
- fields["file"] = (fields["key"], fd)
99
- e = MultipartEncoder(fields=fields)
100
- m = MultipartEncoderMonitor(e, monitor_callback)
101
- # Increase the read size to speed-up upload (the default chunk
102
- # size for uploads in urllib is 8k which results in a lot of
103
- # Python code being involved in uploading a 20GB file; Setting
104
- # the chunk size to 4MB should increase the upload speed):
105
- # https://github.com/requests/toolbelt/issues/75
106
- # #issuecomment-237189952
107
- m._read = m.read
108
- m.read = lambda size: m._read(4 * 1024 * 1024)
109
- # perform the actual upload
110
- hrep = requests.post(
111
- psurl,
112
- data=m,
113
- headers={'Content-Type': m.content_type},
114
- verify=True, # verify SSL connection
115
- timeout=27.3, # timeout to avoid freezing
106
+
107
+ with path.open("rb") as fd:
108
+ if len(upload_urls) > 1:
109
+ # Multipart upload
110
+ # Determine the part size for multipart upload
111
+ num_parts = len(upload_urls)
112
+ file_size = path.stat().st_size
113
+ if file_size % num_parts == 0:
114
+ part_size = file_size // num_parts
115
+ else:
116
+ part_size = file_size // num_parts + 1
117
+ # Upload each part
118
+ etags = []
119
+ for psurl in upload_urls:
120
+ respi = requests.put(psurl,
121
+ data=fd.read(part_size),
122
+ timeout=3,
123
+ )
124
+ etag_part = respi.headers.get("ETag").strip("'").strip('"')
125
+ etags.append(etag_part)
126
+ # Finish the multipart upload
127
+ c_xml = "<CompleteMultipartUpload>\n"
128
+ for ii, etag in enumerate(etags):
129
+ c_xml += (f" <Part>\n"
130
+ + f" <PartNumber>{ii + 1}</PartNumber>\n"
131
+ + f" <ETag>{etag}</ETag>\n"
132
+ + f" </Part>\n"
133
+ )
134
+ c_xml += "</CompleteMultipartUpload>"
135
+ resp = requests.post(
136
+ complete_url,
137
+ data=c_xml,
138
+ timeout=3,
116
139
  )
117
- if hrep.status_code != 204:
118
- raise ValueError(
119
- f"Upload failed with {hrep.status_code}: {hrep.reason}")
140
+ etag_full = resp.headers.get("ETag").strip("'").strip('"')
141
+ else:
142
+ # Single file upload
143
+ resp = requests.put(upload_urls[0],
144
+ data=fd,
145
+ timeout=3)
146
+ etag_full = resp.headers.get("ETag").strip("'").strip('"')
120
147
  """
121
148
  require_bucket(bucket_name)
122
149
  s3_client, _, _ = get_s3()
123
- response = s3_client.generate_presigned_post(bucket_name,
124
- object_name,
125
- ExpiresIn=expiration)
126
- # The response contains the presigned URL and required fields
127
- return response["url"], response["fields"]
150
+ fields = s3_client.create_multipart_upload(Bucket=bucket_name,
151
+ Key=object_name)
152
+ upload_id = fields["UploadId"]
153
+
154
+ gb = 1024**3
155
+
156
+ if file_size % gb == 0:
157
+ num_parts = file_size // gb
158
+ else:
159
+ num_parts = file_size // gb + 1
160
+
161
+ upload_urls = []
162
+ if num_parts == 1:
163
+ # The file is uploaded with one PUT request. We don't need
164
+ # multipart upload.
165
+ psurl = s3_client.generate_presigned_url(
166
+ "put_object",
167
+ Params={'Bucket': bucket_name,
168
+ 'Key': object_name,
169
+ },
170
+ ExpiresIn=expiration,
171
+ HttpMethod='PUT',
172
+ )
173
+ upload_urls.append(psurl)
174
+ complete_url = None
175
+ else:
176
+ # Set up multipart upload URLs
177
+ for ii in range(num_parts):
178
+ psurl = s3_client.generate_presigned_url(
179
+ "upload_part",
180
+ Params={'Bucket': bucket_name,
181
+ 'Key': object_name,
182
+ 'UploadId': upload_id,
183
+ 'PartNumber': ii+1
184
+ },
185
+ ExpiresIn=expiration,
186
+ HttpMethod='PUT',
187
+ )
188
+ upload_urls.append(psurl)
189
+ # Set up the complete upload URL
190
+ complete_url = s3_client.generate_presigned_url(
191
+ "complete_multipart_upload",
192
+ Params={'Bucket': bucket_name,
193
+ 'Key': object_name,
194
+ 'UploadId': upload_id,
195
+ },
196
+ ExpiresIn=expiration,
197
+ HttpMethod='POST',
198
+ )
199
+
200
+ return upload_urls, complete_url
128
201
 
129
202
 
130
203
  def create_presigned_url(
@@ -274,18 +347,27 @@ def make_object_public(bucket_name, object_name, missing_ok=False):
274
347
  for item in response["TagSet"]:
275
348
  tags.append(f"{item['Key']}={item['Value']}")
276
349
  if not tags.count("public=true"):
277
- s3_client.put_object_tagging(
278
- Bucket=bucket_name,
279
- Key=object_name,
280
- Tagging={
281
- 'TagSet': [
282
- {
283
- 'Key': 'public',
284
- 'Value': 'true',
350
+ for retry in range(5):
351
+ try:
352
+ s3_client.put_object_tagging(
353
+ Bucket=bucket_name,
354
+ Key=object_name,
355
+ Tagging={
356
+ 'TagSet': [
357
+ {
358
+ 'Key': 'public',
359
+ 'Value': 'true',
360
+ },
361
+ ],
285
362
  },
286
- ],
287
- },
288
- )
363
+ )
364
+ except botocore.exceptions.ClientError:
365
+ trcbck = traceback.format_exc()
366
+ continue
367
+ else:
368
+ break
369
+ else:
370
+ raise ValueError(f"Encountered Error:\n{trcbck}")
289
371
 
290
372
 
291
373
  def object_exists(bucket_name, object_name):
@@ -61,6 +61,81 @@ def create_presigned_url(
61
61
  filename=filename)
62
62
 
63
63
 
64
+ def get_s3_attributes_for_artifact(
65
+ resource_id: str,
66
+ artifact: Literal["condensed", "preview", "resource"] = "resource"):
67
+ """Return all attribute for an artifact in the S3 object store
68
+
69
+ Returns
70
+ -------
71
+ meta: dict
72
+ Metadata dictionary with the keys "etag", "server", "size",
73
+ and "success".
74
+ """
75
+ bucket_name, object_name = get_s3_bucket_object_for_artifact(
76
+ resource_id=resource_id, artifact=artifact)
77
+ s3_client, _, _ = s3.get_s3()
78
+ attr_info = s3_client.head_object(Bucket=bucket_name, Key=object_name)
79
+ # Example output from MinIO::
80
+ #
81
+ # {'AcceptRanges': 'bytes',
82
+ # 'ContentLength': 904729,
83
+ # 'ContentType': 'application/octet-stream',
84
+ # 'ETag': '"108d47e80f3e5f35110493b1fdcd30d5"',
85
+ # 'LastModified': datetime.datetime(2024, 3, 7, 8, 15,
86
+ # tzinfo=tzutc()),
87
+ # 'Metadata': {},
88
+ # 'ResponseMetadata': {
89
+ # 'HTTPHeaders': {
90
+ # 'accept-ranges': 'bytes',
91
+ # 'content-length': '904729',
92
+ # 'content-type': 'application/octet-stream',
93
+ # 'date': 'Thu, 07 Mar 2024 08:15:02 GMT',
94
+ # 'etag': '"108d47e80f3e5f35110493b1fdcd30d5"',
95
+ # 'last-modified': 'Thu, 07 Mar 2024 '
96
+ # '08:15:00 GMT',
97
+ # 'server': 'MinIO',
98
+ # 'strict-transport-security': 'max-age=31536000; '
99
+ # 'includeSubDomains',
100
+ # 'vary': 'Origin, Accept-Encoding',
101
+ # 'x-amz-id-2': 'dd9025bab4ad464b049177c95eb6e...',
102
+ # 'x-amz-request-id': '17BA6D680CB67A2C',
103
+ # 'x-amz-tagging-count': '1',
104
+ # 'x-content-type-options': 'nosniff',
105
+ # 'x-xss-protection': '1; mode=block'},
106
+ # 'HTTPStatusCode': 200,
107
+ # 'HostId': 'dd9025bab4ad464b049177c95eb6ebf3...',
108
+ # 'RequestId': '17BA6D680CB67A2C',
109
+ # 'RetryAttempts': 0}
110
+ # }
111
+ meta = {}
112
+ for key, funcs in [
113
+ ("etag", [lambda m: m.get("ETag"),
114
+ lambda m: m.get("ResponseMetadata",
115
+ {}).get("HTTPHeaders",
116
+ {}).get("etag"),
117
+ ]),
118
+ ("server", [lambda m: m.get("ResponseMetadata",
119
+ {}).get("HTTPHeaders",
120
+ {}).get("server", "unknown")
121
+ ]),
122
+ ("size", [lambda m: m.get("ContentLength"),
123
+ lambda m: m.get("ResponseMetadata",
124
+ {}).get("HTTPHeaders",
125
+ {}).get("content-length"),
126
+ ]),
127
+ ("success", [lambda m: m.get("ResponseMetadata",
128
+ {}).get("HTTPStatusCode", 404) == 200
129
+ ]),
130
+ ]:
131
+ for fn in funcs:
132
+ val = fn(attr_info)
133
+ if val is not None:
134
+ meta[key] = val
135
+ break
136
+ return meta
137
+
138
+
64
139
  def get_s3_bucket_object_for_artifact(
65
140
  resource_id: str,
66
141
  artifact: Literal["condensed", "preview", "resource"] = "resource"):
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  from io import BytesIO
4
4
  import numbers
5
5
  import pathlib
6
- from typing import Dict
6
+ from typing import Dict, List
7
7
  import uuid
8
8
 
9
9
  import ckan.authz
@@ -11,7 +11,6 @@ import ckan.tests.factories as factories
11
11
  import ckan.tests.helpers as helpers
12
12
  from ckan.tests.pytest_ckan.fixtures import FakeFileStorage
13
13
  import requests
14
- from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
15
14
 
16
15
  import pytest
17
16
 
@@ -122,7 +121,7 @@ def make_dataset_via_s3(
122
121
  CKAN dictionary of the owner organization
123
122
  resource_path: str or pathlib.Path
124
123
  path to the resource file to upload. If not specified, then
125
- not resource is created for the dataset
124
+ no resource is created for the dataset
126
125
  activate: bool
127
126
  whether to activate the dataset
128
127
  kwargs: dict
@@ -251,36 +250,72 @@ def synchronous_enqueue_job(job_func, args=None, kwargs=None, title=None,
251
250
  job_func(*args, **kwargs)
252
251
 
253
252
 
254
- def upload_presigned_to_s3(psurl, fields, path_to_upload):
253
+ def upload_presigned_to_s3(
254
+ path: str | pathlib.Path,
255
+ upload_urls: List[str],
256
+ complete_url: str | None):
255
257
  """Helper function for uploading data to S3
256
258
 
257
- This is exactly how DCOR-Aid would be uploading things (with the
258
- requests_toolbelt package). This could have been a little simpler,
259
- but for the sake of reproducibility, we do it the DCOR-Aid way.
259
+ This is how DCOR-Aid would be uploading things.
260
+
261
+ Parameters
262
+ ----------
263
+ path: str or pathlib.Path
264
+ file to upload
265
+ upload_urls: list
266
+ List of the presigned URLs required for the upload.
267
+ There will always be the key "urls" containing a list of
268
+ presigned URLs.
269
+ complete_url: str
270
+ If a multipart upload is necessary, this is the presigned URL
271
+ required to finalize the upload. For more information, see
272
+ `https://boto3.amazonaws.com/v1/documentation/api/latest/
273
+ reference/services/s3/client/complete_multipart_upload.html
274
+ #complete-multipart-upload`_ or the example below
260
275
  """
261
- # callback function for monitoring the upload progress
262
- # open the input file for streaming
263
- with path_to_upload.open("rb") as fd:
264
- fields["file"] = (fields["key"], fd)
265
- e = MultipartEncoder(fields=fields)
266
- m = MultipartEncoderMonitor(
267
- e, lambda monitor: print(f"Bytes: {monitor.bytes_read}"))
268
- # Increase the read size to speed-up upload (the default chunk
269
- # size for uploads in urllib is 8k which results in a lot of
270
- # Python code being involved in uploading a 20GB file; Setting
271
- # the chunk size to 4MB should increase the upload speed):
272
- # https://github.com/requests/toolbelt/issues/75
273
- # #issuecomment-237189952
274
- m._read = m.read
275
- m.read = lambda size: m._read(4 * 1024 * 1024)
276
- # perform the actual upload
277
- hrep = requests.post(
278
- psurl,
279
- data=m,
280
- headers={'Content-Type': m.content_type},
281
- verify=True, # verify SSL connection
282
- timeout=27.3, # timeout to avoid freezing
283
- )
284
- if hrep.status_code != 204:
276
+ path = pathlib.Path(path)
277
+ with path.open("rb") as fd:
278
+ if len(upload_urls) > 1:
279
+ # Multipart upload
280
+ # Determine the part size for multipart upload
281
+ num_parts = len(upload_urls)
282
+ file_size = path.stat().st_size
283
+ if file_size % num_parts == 0:
284
+ part_size = file_size // num_parts
285
+ else:
286
+ part_size = file_size // num_parts + 1
287
+ # Upload each part
288
+ etags = []
289
+ for psurl in upload_urls:
290
+ respi = requests.put(psurl,
291
+ data=fd.read(part_size),
292
+ timeout=3,
293
+ )
294
+ etag_part = respi.headers.get("ETag", "").strip("'").strip('"')
295
+ etags.append(etag_part)
296
+ # Finish the multipart upload
297
+ c_xml = "<CompleteMultipartUpload>\n"
298
+ for ii, etag in enumerate(etags):
299
+ c_xml += (f" <Part>\n"
300
+ + f" <PartNumber>{ii + 1}</PartNumber>\n"
301
+ + f" <ETag>{etag}</ETag>\n"
302
+ + f" </Part>\n"
303
+ )
304
+ c_xml += "</CompleteMultipartUpload>"
305
+ resp = requests.post(
306
+ complete_url,
307
+ data=c_xml,
308
+ timeout=3,
309
+ )
310
+ etag_full = resp.headers.get("ETag", "").strip("'").strip('"')
311
+ else:
312
+ # Single file upload
313
+ resp = requests.put(upload_urls[0],
314
+ data=fd,
315
+ timeout=3)
316
+ etag_full = resp.headers.get("ETag", "").strip("'").strip('"')
317
+ if not etag_full:
285
318
  raise ValueError(
286
- f"Upload failed with {hrep.status_code}: {hrep.reason}")
319
+ f"Upload failed with {resp.status_code}: {resp.reason} "
320
+ f"({resp.headers})")
321
+ return etag_full
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcor-shared
3
- Version: 0.9.0
3
+ Version: 0.10.0
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Home-page: https://github.com/DCOR-dev/dcor_shared
6
6
  Author: Paul Müller
@@ -1,14 +1,14 @@
1
- import base64
2
1
  import hashlib
3
2
  from unittest import mock
4
3
  import pathlib
4
+ import random
5
5
  import uuid
6
6
 
7
7
  import botocore.exceptions
8
8
  import pytest
9
9
  import requests
10
10
 
11
- from dcor_shared import s3, sha256sum
11
+ from dcor_shared import get_ckan_config_option, s3, sha256sum
12
12
  from dcor_shared.testing import upload_presigned_to_s3
13
13
 
14
14
 
@@ -189,14 +189,61 @@ def test_presigned_upload():
189
189
  bucket_name = f"test-circle-{uuid.uuid4()}"
190
190
  rid = str(uuid.uuid4())
191
191
  object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
192
- psurl, fields = s3.create_presigned_upload_url(bucket_name=bucket_name,
193
- object_name=object_name)
192
+ upload_urls, complete_url = s3.create_presigned_upload_urls(
193
+ bucket_name=bucket_name,
194
+ object_name=object_name,
195
+ file_size=path.stat().st_size,
196
+ )
197
+
198
+ assert len(upload_urls) == 1, "no multipart upload"
199
+ assert complete_url is None, "no multipart upload"
194
200
 
195
201
  # This is what DCOR-Aid would do to upload the file
196
- upload_presigned_to_s3(psurl=psurl,
197
- fields=fields,
198
- path_to_upload=path
199
- )
202
+ etag = upload_presigned_to_s3(
203
+ path=path,
204
+ upload_urls=upload_urls,
205
+ complete_url=complete_url,
206
+ )
207
+
208
+ assert hashlib.md5(path.read_bytes()).hexdigest() == etag
209
+
210
+ hash_exp = hashlib.sha256(path.read_bytes()).hexdigest()
211
+ hash_act = s3.compute_checksum(bucket_name=bucket_name,
212
+ object_name=object_name)
213
+ assert hash_exp == hash_act
214
+
215
+
216
+ def test_presigned_upload_multipart(tmp_path):
217
+ path = tmp_path / "calibration_beads_47.rtdc"
218
+ with path.open("wb") as fd:
219
+ for ii in range(20): # 20 MiB
220
+ fd.write(bytearray(1024**2))
221
+
222
+ # This is what would happen on the server when DCOR-Aid requests an
223
+ # upload URL
224
+ bucket_name = f"test-circle-{uuid.uuid4()}"
225
+ rid = str(uuid.uuid4())
226
+ object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
227
+ upload_urls, complete_url = s3.create_presigned_upload_urls(
228
+ bucket_name=bucket_name,
229
+ object_name=object_name,
230
+ # pretend as if we had a huge file
231
+ file_size=int(1.5*1024**3),
232
+ )
233
+
234
+ assert len(upload_urls) == 2, "multipart upload"
235
+ assert complete_url is not None, "multipart upload"
236
+
237
+ # This is what DCOR-Aid would do to upload the file
238
+ etag = upload_presigned_to_s3(
239
+ path=path,
240
+ upload_urls=upload_urls,
241
+ complete_url=complete_url,
242
+ )
243
+ # This is how Amazon computes the ETag for multipart uploads.
244
+ md5part = hashlib.md5(bytearray(1024**2)*10).digest()
245
+ etag_exp = hashlib.md5(md5part+md5part).hexdigest() + "-2"
246
+ assert etag == etag_exp
200
247
 
201
248
  hash_exp = hashlib.sha256(path.read_bytes()).hexdigest()
202
249
  hash_act = s3.compute_checksum(bucket_name=bucket_name,
@@ -205,7 +252,7 @@ def test_presigned_upload():
205
252
 
206
253
 
207
254
  def test_presigned_upload_private_by_default():
208
- """The presigend upload should be private by default"""
255
+ """The presigned upload should be private by default"""
209
256
  path = data_path / "calibration_beads_47.rtdc"
210
257
 
211
258
  # This is what would happen on the server when DCOR-Aid requests an
@@ -213,16 +260,20 @@ def test_presigned_upload_private_by_default():
213
260
  bucket_name = f"test-circle-{uuid.uuid4()}"
214
261
  rid = str(uuid.uuid4())
215
262
  object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
216
- psurl, fields = s3.create_presigned_upload_url(bucket_name=bucket_name,
217
- object_name=object_name)
263
+
264
+ upload_urls, complete_url = s3.create_presigned_upload_urls(
265
+ bucket_name=bucket_name,
266
+ object_name=object_name,
267
+ file_size=path.stat().st_size,
268
+ )
218
269
 
219
270
  # This is what DCOR-Aid would do to upload the file
220
- upload_presigned_to_s3(psurl=psurl,
221
- fields=fields,
222
- path_to_upload=path
271
+ upload_presigned_to_s3(path=path,
272
+ upload_urls=upload_urls,
273
+ complete_url=complete_url,
223
274
  )
224
275
 
225
- s3_url = "/".join([psurl, fields["key"]])
276
+ s3_url = upload_urls[0].split("?")[0]
226
277
 
227
278
  # attempt to download the data
228
279
  response = requests.get(s3_url)
@@ -245,36 +296,34 @@ def test_presigned_upload_wrong_access():
245
296
  bucket_name = f"test-circle-{uuid.uuid4()}"
246
297
  rid = str(uuid.uuid4())
247
298
  object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
248
- psurl, fields = s3.create_presigned_upload_url(bucket_name=bucket_name,
249
- object_name=object_name)
250
- # Try to upload the file under a different object name
251
- # (this tests the S3 access restrictions)
252
- rid2 = str(uuid.uuid4())
253
- object_name_bad = f"resource/{rid2[:3]}/{rid2[3:6]}/{rid2[6:]}"
254
- # replace the old with the bad object name
255
- new_policy = base64.b64encode(
256
- base64.b64decode(fields["policy"])
257
- .decode("utf-8")
258
- .replace(object_name, object_name_bad)
259
- .encode("utf-8")
299
+
300
+ upload_urls, complete_url = s3.create_presigned_upload_urls(
301
+ bucket_name=bucket_name,
302
+ object_name=object_name,
303
+ file_size=path.stat().st_size,
260
304
  )
261
- # sanity check
262
- assert new_policy != fields["policy"]
263
- fields["policy"] = new_policy
264
- fields["key"] = object_name_bad
305
+
306
+ assert len(upload_urls) == 1, "no multipart upload"
307
+ assert complete_url is None, "no multipart upload"
308
+
309
+ # Try to change the signature
310
+ parts = upload_urls[0].split("&")
311
+ for ii, part in enumerate(parts):
312
+ if part.startswith("Signature="):
313
+ new_sig = part[:-10] + part[-10:][::-1]
314
+ parts[ii] = new_sig
315
+ break
316
+ else:
317
+ assert False
318
+ upload_urls[0] = "&".join(parts)
265
319
 
266
320
  with pytest.raises(ValueError, match="Upload failed with 403: Forbidden"):
267
321
  # This is what DCOR-Aid would do to upload the file
268
- upload_presigned_to_s3(psurl=psurl,
269
- fields=fields,
270
- path_to_upload=path
322
+ upload_presigned_to_s3(path=path,
323
+ upload_urls=upload_urls,
324
+ complete_url=complete_url,
271
325
  )
272
326
 
273
- with pytest.raises(botocore.exceptions.ClientError, match="Not Found"):
274
- # Make sure the file does not exist
275
- s3.compute_checksum(bucket_name=bucket_name,
276
- object_name=object_name_bad)
277
-
278
327
 
279
328
  def test_presigned_upload_wrong_key():
280
329
  """Same as `test_presigned_upload_wrong_access` but no policy change"""
@@ -285,19 +334,28 @@ def test_presigned_upload_wrong_key():
285
334
  bucket_name = f"test-circle-{uuid.uuid4()}"
286
335
  rid = str(uuid.uuid4())
287
336
  object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
288
- psurl, fields = s3.create_presigned_upload_url(bucket_name=bucket_name,
289
- object_name=object_name)
337
+
338
+ upload_urls, complete_url = s3.create_presigned_upload_urls(
339
+ bucket_name=bucket_name,
340
+ object_name=object_name,
341
+ file_size=path.stat().st_size,
342
+ )
343
+
344
+ assert len(upload_urls) == 1, "no multipart upload"
345
+ assert complete_url is None, "no multipart upload"
346
+
290
347
  # Try to upload the file under a different object name
291
348
  # (this tests the S3 access restrictions)
292
349
  rid2 = str(uuid.uuid4())
293
350
  object_name_bad = f"resource/{rid2[:3]}/{rid2[3:6]}/{rid2[6:]}"
294
- fields["key"] = object_name_bad
351
+ # replace the old with the bad object name
352
+ upload_urls[0] = upload_urls[0].replace(object_name, object_name_bad)
295
353
 
296
354
  with pytest.raises(ValueError, match="Upload failed with 403: Forbidden"):
297
355
  # This is what DCOR-Aid would do to upload the file
298
- upload_presigned_to_s3(psurl=psurl,
299
- fields=fields,
300
- path_to_upload=path
356
+ upload_presigned_to_s3(path=path,
357
+ upload_urls=upload_urls,
358
+ complete_url=complete_url,
301
359
  )
302
360
 
303
361
  with pytest.raises(botocore.exceptions.ClientError, match="Not Found"):
@@ -23,6 +23,11 @@ data_path = pathlib.Path(__file__).parent / "data"
23
23
 
24
24
  def setup_s3_resource_on_ckan(private=False, resource_path=None):
25
25
  """Create an S3 resource in CKAN"""
26
+ if resource_path is None:
27
+ resource_path = data_path / "calibration_beads_47.rtdc"
28
+
29
+ file_size = resource_path.stat().st_size
30
+
26
31
  user = factories.User()
27
32
  owner_org = factories.Organization(users=[{
28
33
  'name': user['id'],
@@ -34,18 +39,18 @@ def setup_s3_resource_on_ckan(private=False, resource_path=None):
34
39
 
35
40
  # Upload the resource to S3 (note that it is not required that the
36
41
  # dataset exists)
37
- response = helpers.call_action("resource_upload_s3_url",
42
+ response = helpers.call_action("resource_upload_s3_urls",
38
43
  test_context,
39
44
  organization_id=owner_org["id"],
45
+ file_size=file_size,
40
46
  )
41
47
  rid = response["resource_id"]
42
- if resource_path is None:
43
- resource_path = data_path / "calibration_beads_47.rtdc"
44
48
 
45
49
  upload_presigned_to_s3(
46
- psurl=response["url"],
47
- fields=response["fields"],
48
- path_to_upload=resource_path)
50
+ path=resource_path,
51
+ upload_urls=response["upload_urls"],
52
+ complete_url=response["complete_url"],
53
+ )
49
54
 
50
55
  # Create the dataset
51
56
  pkg_dict = helpers.call_action("package_create",
@@ -69,7 +74,7 @@ def setup_s3_resource_on_ckan(private=False, resource_path=None):
69
74
  }],
70
75
  )
71
76
  assert new_pkg_dict["package"]["num_resources"] == 1
72
- s3_url = response["url"] + "/" + response["fields"]["key"]
77
+ s3_url = response["upload_urls"][0].split("?")[0]
73
78
  return rid, s3_url, new_pkg_dict, owner_org
74
79
 
75
80
 
@@ -115,6 +120,25 @@ def test_create_presigned_url(enqueue_job_mock, tmp_path):
115
120
  "490efdf5d9bb4cd4b2a6bcf2fe54d4dc201c38530140bcb168980bf8bf846c73"
116
121
 
117
122
 
123
+ @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
124
+ @pytest.mark.usefixtures('clean_db', 'with_request_context')
125
+ @mock.patch('ckan.plugins.toolkit.enqueue_job',
126
+ side_effect=synchronous_enqueue_job)
127
+ def test_get_s3_attributes_for_artifact(enqueue_job_mock):
128
+ rid, _, _, org_dict = setup_s3_resource_on_ckan()
129
+
130
+ # Make sure the resource exists
131
+ res_dict = helpers.call_action("resource_show", id=rid)
132
+ assert res_dict["id"] == rid, "sanity check"
133
+
134
+ # get the size
135
+ meta = s3cc.get_s3_attributes_for_artifact(rid)
136
+ assert meta["size"] == 904729
137
+ assert meta["success"]
138
+ assert meta["etag"]
139
+ assert meta["server"]
140
+
141
+
118
142
  @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
119
143
  @pytest.mark.usefixtures('clean_db', 'with_request_context')
120
144
  @mock.patch('ckan.plugins.toolkit.enqueue_job',
File without changes
File without changes
File without changes
File without changes
File without changes