dcor-shared 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/CHANGELOG +7 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/PKG-INFO +1 -1
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/_version_save.py +1 -1
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/s3.py +149 -67
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/s3cc.py +75 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/testing.py +67 -32
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/PKG-INFO +1 -1
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_s3.py +103 -45
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_s3cc.py +31 -7
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/LICENSE +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/MANIFEST.in +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/README.rst +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/__init__.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/_version.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/ckan.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/data.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/dcinst.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/mime.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/parse.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared/paths.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/SOURCES.txt +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/dependency_links.txt +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/requires.txt +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/dcor_shared.egg-info/top_level.txt +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/setup.cfg +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/setup.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_basic.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_ckan.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_data.py +0 -0
- {dcor_shared-0.9.0 → dcor_shared-0.10.0}/tests/test_testing.py +0 -0
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
0.10.0
|
|
2
|
+
- BREAKING CHANGE: presigned URLs for uploading resources now work
|
|
3
|
+
differently. For small files (< 1 GiB), a single upload URL for
|
|
4
|
+
PUT is created. For larger files, a multipart upload is started.
|
|
5
|
+
- enh: retry set_object_tagging on failure
|
|
6
|
+
0.9.1
|
|
7
|
+
- feat: add convenience method `get_s3_attributes_for_artifact`
|
|
1
8
|
0.9.0
|
|
2
9
|
- feat: introduce `testing.make_dataset_via_s3` and
|
|
3
10
|
`testing.make_resource_via_s3` which uploads resources via S3
|
|
@@ -6,6 +6,8 @@ import hashlib
|
|
|
6
6
|
import json
|
|
7
7
|
import pathlib
|
|
8
8
|
import time
|
|
9
|
+
import traceback
|
|
10
|
+
from typing import List, Tuple
|
|
9
11
|
import warnings
|
|
10
12
|
|
|
11
13
|
import boto3
|
|
@@ -45,9 +47,12 @@ def compute_checksum(bucket_name, object_name, max_size=None):
|
|
|
45
47
|
return s3_sha256
|
|
46
48
|
|
|
47
49
|
|
|
48
|
-
def
|
|
49
|
-
|
|
50
|
-
|
|
50
|
+
def create_presigned_upload_urls(
|
|
51
|
+
bucket_name: str,
|
|
52
|
+
object_name: str,
|
|
53
|
+
file_size: int,
|
|
54
|
+
expiration: int = 86400) -> Tuple[List, str]:
|
|
55
|
+
"""Create presigned URLs for uploading an object to S3
|
|
51
56
|
|
|
52
57
|
Parameters
|
|
53
58
|
----------
|
|
@@ -55,76 +60,144 @@ def create_presigned_upload_url(bucket_name, object_name,
|
|
|
55
60
|
Name of the bucket
|
|
56
61
|
object_name: str
|
|
57
62
|
Name of the object
|
|
63
|
+
file_size: int
|
|
64
|
+
Size of the file in bytes; used to determine the number of
|
|
65
|
+
multipart upload parts with a hard-coded part size of
|
|
66
|
+
maximum 1 GiB. Reasons for choosing 1 GiB:
|
|
67
|
+
- Using the default value of 8 or 16 MiB leads to a lot parts
|
|
68
|
+
for a 20 GiB file
|
|
69
|
+
- The maximum allowed part size in S3 is 5 GiB
|
|
70
|
+
- Somebody on the internet said that Amazon is actually not
|
|
71
|
+
concatenating the parts into a new, larger file, but it keeps
|
|
72
|
+
the parts stored and just adds a layer in-between. This has
|
|
73
|
+
performance implications if you are using GET with ranges
|
|
74
|
+
to access partial data.
|
|
75
|
+
- I was originally thinking about 100 MiB, but having the number
|
|
76
|
+
of parts in the ETag as an estimate of the file size in GiB
|
|
77
|
+
seemed like a good idea.
|
|
58
78
|
expiration: int
|
|
59
79
|
Time in seconds for the presigned URL to remain valid
|
|
60
80
|
|
|
61
81
|
Returns
|
|
62
82
|
-------
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
83
|
+
upload_urls: list
|
|
84
|
+
List of the presigned URLs required for the upload.
|
|
85
|
+
There will always be the key "urls" containing a list of
|
|
86
|
+
presigned URLs.
|
|
87
|
+
complete_url: str
|
|
88
|
+
If a multipart upload is necessary, this is the presigned URL
|
|
89
|
+
required to finalize the upload. For more information, see
|
|
90
|
+
`https://boto3.amazonaws.com/v1/documentation/api/latest/
|
|
91
|
+
reference/services/s3/client/complete_multipart_upload.html
|
|
92
|
+
#complete-multipart-upload`_ or the example below
|
|
71
93
|
|
|
72
94
|
Notes
|
|
73
95
|
-----
|
|
74
|
-
By default, due to the access design in DCOR, the S3
|
|
75
|
-
uploaded resource
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
tag to the object.
|
|
96
|
+
By default, due to the access design in DCOR, the S3 URLs of the
|
|
97
|
+
uploaded resource is private. If you would like to make the resource
|
|
98
|
+
publicly accessible, you can call the method :func:`make_object_public`,
|
|
99
|
+
which will add the `public=true` tag to the object.
|
|
79
100
|
|
|
80
101
|
Example
|
|
81
102
|
-------
|
|
82
|
-
To upload a file with :mod:`requests
|
|
103
|
+
To upload a file with :mod:`requests`::
|
|
83
104
|
|
|
84
|
-
import pathlib
|
|
85
105
|
import requests
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
106
|
+
|
|
107
|
+
with path.open("rb") as fd:
|
|
108
|
+
if len(upload_urls) > 1:
|
|
109
|
+
# Multipart upload
|
|
110
|
+
# Determine the part size for multipart upload
|
|
111
|
+
num_parts = len(upload_urls)
|
|
112
|
+
file_size = path.stat().st_size
|
|
113
|
+
if file_size % num_parts == 0:
|
|
114
|
+
part_size = file_size // num_parts
|
|
115
|
+
else:
|
|
116
|
+
part_size = file_size // num_parts + 1
|
|
117
|
+
# Upload each part
|
|
118
|
+
etags = []
|
|
119
|
+
for psurl in upload_urls:
|
|
120
|
+
respi = requests.put(psurl,
|
|
121
|
+
data=fd.read(part_size),
|
|
122
|
+
timeout=3,
|
|
123
|
+
)
|
|
124
|
+
etag_part = respi.headers.get("ETag").strip("'").strip('"')
|
|
125
|
+
etags.append(etag_part)
|
|
126
|
+
# Finish the multipart upload
|
|
127
|
+
c_xml = "<CompleteMultipartUpload>\n"
|
|
128
|
+
for ii, etag in enumerate(etags):
|
|
129
|
+
c_xml += (f" <Part>\n"
|
|
130
|
+
+ f" <PartNumber>{ii + 1}</PartNumber>\n"
|
|
131
|
+
+ f" <ETag>{etag}</ETag>\n"
|
|
132
|
+
+ f" </Part>\n"
|
|
133
|
+
)
|
|
134
|
+
c_xml += "</CompleteMultipartUpload>"
|
|
135
|
+
resp = requests.post(
|
|
136
|
+
complete_url,
|
|
137
|
+
data=c_xml,
|
|
138
|
+
timeout=3,
|
|
116
139
|
)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
140
|
+
etag_full = resp.headers.get("ETag").strip("'").strip('"')
|
|
141
|
+
else:
|
|
142
|
+
# Single file upload
|
|
143
|
+
resp = requests.put(upload_urls[0],
|
|
144
|
+
data=fd,
|
|
145
|
+
timeout=3)
|
|
146
|
+
etag_full = resp.headers.get("ETag").strip("'").strip('"')
|
|
120
147
|
"""
|
|
121
148
|
require_bucket(bucket_name)
|
|
122
149
|
s3_client, _, _ = get_s3()
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
150
|
+
fields = s3_client.create_multipart_upload(Bucket=bucket_name,
|
|
151
|
+
Key=object_name)
|
|
152
|
+
upload_id = fields["UploadId"]
|
|
153
|
+
|
|
154
|
+
gb = 1024**3
|
|
155
|
+
|
|
156
|
+
if file_size % gb == 0:
|
|
157
|
+
num_parts = file_size // gb
|
|
158
|
+
else:
|
|
159
|
+
num_parts = file_size // gb + 1
|
|
160
|
+
|
|
161
|
+
upload_urls = []
|
|
162
|
+
if num_parts == 1:
|
|
163
|
+
# The file is uploaded with one PUT request. We don't need
|
|
164
|
+
# multipart upload.
|
|
165
|
+
psurl = s3_client.generate_presigned_url(
|
|
166
|
+
"put_object",
|
|
167
|
+
Params={'Bucket': bucket_name,
|
|
168
|
+
'Key': object_name,
|
|
169
|
+
},
|
|
170
|
+
ExpiresIn=expiration,
|
|
171
|
+
HttpMethod='PUT',
|
|
172
|
+
)
|
|
173
|
+
upload_urls.append(psurl)
|
|
174
|
+
complete_url = None
|
|
175
|
+
else:
|
|
176
|
+
# Set up multipart upload URLs
|
|
177
|
+
for ii in range(num_parts):
|
|
178
|
+
psurl = s3_client.generate_presigned_url(
|
|
179
|
+
"upload_part",
|
|
180
|
+
Params={'Bucket': bucket_name,
|
|
181
|
+
'Key': object_name,
|
|
182
|
+
'UploadId': upload_id,
|
|
183
|
+
'PartNumber': ii+1
|
|
184
|
+
},
|
|
185
|
+
ExpiresIn=expiration,
|
|
186
|
+
HttpMethod='PUT',
|
|
187
|
+
)
|
|
188
|
+
upload_urls.append(psurl)
|
|
189
|
+
# Set up the complete upload URL
|
|
190
|
+
complete_url = s3_client.generate_presigned_url(
|
|
191
|
+
"complete_multipart_upload",
|
|
192
|
+
Params={'Bucket': bucket_name,
|
|
193
|
+
'Key': object_name,
|
|
194
|
+
'UploadId': upload_id,
|
|
195
|
+
},
|
|
196
|
+
ExpiresIn=expiration,
|
|
197
|
+
HttpMethod='POST',
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
return upload_urls, complete_url
|
|
128
201
|
|
|
129
202
|
|
|
130
203
|
def create_presigned_url(
|
|
@@ -274,18 +347,27 @@ def make_object_public(bucket_name, object_name, missing_ok=False):
|
|
|
274
347
|
for item in response["TagSet"]:
|
|
275
348
|
tags.append(f"{item['Key']}={item['Value']}")
|
|
276
349
|
if not tags.count("public=true"):
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
{
|
|
283
|
-
'
|
|
284
|
-
|
|
350
|
+
for retry in range(5):
|
|
351
|
+
try:
|
|
352
|
+
s3_client.put_object_tagging(
|
|
353
|
+
Bucket=bucket_name,
|
|
354
|
+
Key=object_name,
|
|
355
|
+
Tagging={
|
|
356
|
+
'TagSet': [
|
|
357
|
+
{
|
|
358
|
+
'Key': 'public',
|
|
359
|
+
'Value': 'true',
|
|
360
|
+
},
|
|
361
|
+
],
|
|
285
362
|
},
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
363
|
+
)
|
|
364
|
+
except botocore.exceptions.ClientError:
|
|
365
|
+
trcbck = traceback.format_exc()
|
|
366
|
+
continue
|
|
367
|
+
else:
|
|
368
|
+
break
|
|
369
|
+
else:
|
|
370
|
+
raise ValueError(f"Encountered Error:\n{trcbck}")
|
|
289
371
|
|
|
290
372
|
|
|
291
373
|
def object_exists(bucket_name, object_name):
|
|
@@ -61,6 +61,81 @@ def create_presigned_url(
|
|
|
61
61
|
filename=filename)
|
|
62
62
|
|
|
63
63
|
|
|
64
|
+
def get_s3_attributes_for_artifact(
|
|
65
|
+
resource_id: str,
|
|
66
|
+
artifact: Literal["condensed", "preview", "resource"] = "resource"):
|
|
67
|
+
"""Return all attribute for an artifact in the S3 object store
|
|
68
|
+
|
|
69
|
+
Returns
|
|
70
|
+
-------
|
|
71
|
+
meta: dict
|
|
72
|
+
Metadata dictionary with the keys "etag", "server", "size",
|
|
73
|
+
and "success".
|
|
74
|
+
"""
|
|
75
|
+
bucket_name, object_name = get_s3_bucket_object_for_artifact(
|
|
76
|
+
resource_id=resource_id, artifact=artifact)
|
|
77
|
+
s3_client, _, _ = s3.get_s3()
|
|
78
|
+
attr_info = s3_client.head_object(Bucket=bucket_name, Key=object_name)
|
|
79
|
+
# Example output from MinIO::
|
|
80
|
+
#
|
|
81
|
+
# {'AcceptRanges': 'bytes',
|
|
82
|
+
# 'ContentLength': 904729,
|
|
83
|
+
# 'ContentType': 'application/octet-stream',
|
|
84
|
+
# 'ETag': '"108d47e80f3e5f35110493b1fdcd30d5"',
|
|
85
|
+
# 'LastModified': datetime.datetime(2024, 3, 7, 8, 15,
|
|
86
|
+
# tzinfo=tzutc()),
|
|
87
|
+
# 'Metadata': {},
|
|
88
|
+
# 'ResponseMetadata': {
|
|
89
|
+
# 'HTTPHeaders': {
|
|
90
|
+
# 'accept-ranges': 'bytes',
|
|
91
|
+
# 'content-length': '904729',
|
|
92
|
+
# 'content-type': 'application/octet-stream',
|
|
93
|
+
# 'date': 'Thu, 07 Mar 2024 08:15:02 GMT',
|
|
94
|
+
# 'etag': '"108d47e80f3e5f35110493b1fdcd30d5"',
|
|
95
|
+
# 'last-modified': 'Thu, 07 Mar 2024 '
|
|
96
|
+
# '08:15:00 GMT',
|
|
97
|
+
# 'server': 'MinIO',
|
|
98
|
+
# 'strict-transport-security': 'max-age=31536000; '
|
|
99
|
+
# 'includeSubDomains',
|
|
100
|
+
# 'vary': 'Origin, Accept-Encoding',
|
|
101
|
+
# 'x-amz-id-2': 'dd9025bab4ad464b049177c95eb6e...',
|
|
102
|
+
# 'x-amz-request-id': '17BA6D680CB67A2C',
|
|
103
|
+
# 'x-amz-tagging-count': '1',
|
|
104
|
+
# 'x-content-type-options': 'nosniff',
|
|
105
|
+
# 'x-xss-protection': '1; mode=block'},
|
|
106
|
+
# 'HTTPStatusCode': 200,
|
|
107
|
+
# 'HostId': 'dd9025bab4ad464b049177c95eb6ebf3...',
|
|
108
|
+
# 'RequestId': '17BA6D680CB67A2C',
|
|
109
|
+
# 'RetryAttempts': 0}
|
|
110
|
+
# }
|
|
111
|
+
meta = {}
|
|
112
|
+
for key, funcs in [
|
|
113
|
+
("etag", [lambda m: m.get("ETag"),
|
|
114
|
+
lambda m: m.get("ResponseMetadata",
|
|
115
|
+
{}).get("HTTPHeaders",
|
|
116
|
+
{}).get("etag"),
|
|
117
|
+
]),
|
|
118
|
+
("server", [lambda m: m.get("ResponseMetadata",
|
|
119
|
+
{}).get("HTTPHeaders",
|
|
120
|
+
{}).get("server", "unknown")
|
|
121
|
+
]),
|
|
122
|
+
("size", [lambda m: m.get("ContentLength"),
|
|
123
|
+
lambda m: m.get("ResponseMetadata",
|
|
124
|
+
{}).get("HTTPHeaders",
|
|
125
|
+
{}).get("content-length"),
|
|
126
|
+
]),
|
|
127
|
+
("success", [lambda m: m.get("ResponseMetadata",
|
|
128
|
+
{}).get("HTTPStatusCode", 404) == 200
|
|
129
|
+
]),
|
|
130
|
+
]:
|
|
131
|
+
for fn in funcs:
|
|
132
|
+
val = fn(attr_info)
|
|
133
|
+
if val is not None:
|
|
134
|
+
meta[key] = val
|
|
135
|
+
break
|
|
136
|
+
return meta
|
|
137
|
+
|
|
138
|
+
|
|
64
139
|
def get_s3_bucket_object_for_artifact(
|
|
65
140
|
resource_id: str,
|
|
66
141
|
artifact: Literal["condensed", "preview", "resource"] = "resource"):
|
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
from io import BytesIO
|
|
4
4
|
import numbers
|
|
5
5
|
import pathlib
|
|
6
|
-
from typing import Dict
|
|
6
|
+
from typing import Dict, List
|
|
7
7
|
import uuid
|
|
8
8
|
|
|
9
9
|
import ckan.authz
|
|
@@ -11,7 +11,6 @@ import ckan.tests.factories as factories
|
|
|
11
11
|
import ckan.tests.helpers as helpers
|
|
12
12
|
from ckan.tests.pytest_ckan.fixtures import FakeFileStorage
|
|
13
13
|
import requests
|
|
14
|
-
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
|
|
15
14
|
|
|
16
15
|
import pytest
|
|
17
16
|
|
|
@@ -122,7 +121,7 @@ def make_dataset_via_s3(
|
|
|
122
121
|
CKAN dictionary of the owner organization
|
|
123
122
|
resource_path: str or pathlib.Path
|
|
124
123
|
path to the resource file to upload. If not specified, then
|
|
125
|
-
|
|
124
|
+
no resource is created for the dataset
|
|
126
125
|
activate: bool
|
|
127
126
|
whether to activate the dataset
|
|
128
127
|
kwargs: dict
|
|
@@ -251,36 +250,72 @@ def synchronous_enqueue_job(job_func, args=None, kwargs=None, title=None,
|
|
|
251
250
|
job_func(*args, **kwargs)
|
|
252
251
|
|
|
253
252
|
|
|
254
|
-
def upload_presigned_to_s3(
|
|
253
|
+
def upload_presigned_to_s3(
|
|
254
|
+
path: str | pathlib.Path,
|
|
255
|
+
upload_urls: List[str],
|
|
256
|
+
complete_url: str | None):
|
|
255
257
|
"""Helper function for uploading data to S3
|
|
256
258
|
|
|
257
|
-
This is
|
|
258
|
-
|
|
259
|
-
|
|
259
|
+
This is how DCOR-Aid would be uploading things.
|
|
260
|
+
|
|
261
|
+
Parameters
|
|
262
|
+
----------
|
|
263
|
+
path: str or pathlib.Path
|
|
264
|
+
file to upload
|
|
265
|
+
upload_urls: list
|
|
266
|
+
List of the presigned URLs required for the upload.
|
|
267
|
+
There will always be the key "urls" containing a list of
|
|
268
|
+
presigned URLs.
|
|
269
|
+
complete_url: str
|
|
270
|
+
If a multipart upload is necessary, this is the presigned URL
|
|
271
|
+
required to finalize the upload. For more information, see
|
|
272
|
+
`https://boto3.amazonaws.com/v1/documentation/api/latest/
|
|
273
|
+
reference/services/s3/client/complete_multipart_upload.html
|
|
274
|
+
#complete-multipart-upload`_ or the example below
|
|
260
275
|
"""
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
276
|
+
path = pathlib.Path(path)
|
|
277
|
+
with path.open("rb") as fd:
|
|
278
|
+
if len(upload_urls) > 1:
|
|
279
|
+
# Multipart upload
|
|
280
|
+
# Determine the part size for multipart upload
|
|
281
|
+
num_parts = len(upload_urls)
|
|
282
|
+
file_size = path.stat().st_size
|
|
283
|
+
if file_size % num_parts == 0:
|
|
284
|
+
part_size = file_size // num_parts
|
|
285
|
+
else:
|
|
286
|
+
part_size = file_size // num_parts + 1
|
|
287
|
+
# Upload each part
|
|
288
|
+
etags = []
|
|
289
|
+
for psurl in upload_urls:
|
|
290
|
+
respi = requests.put(psurl,
|
|
291
|
+
data=fd.read(part_size),
|
|
292
|
+
timeout=3,
|
|
293
|
+
)
|
|
294
|
+
etag_part = respi.headers.get("ETag", "").strip("'").strip('"')
|
|
295
|
+
etags.append(etag_part)
|
|
296
|
+
# Finish the multipart upload
|
|
297
|
+
c_xml = "<CompleteMultipartUpload>\n"
|
|
298
|
+
for ii, etag in enumerate(etags):
|
|
299
|
+
c_xml += (f" <Part>\n"
|
|
300
|
+
+ f" <PartNumber>{ii + 1}</PartNumber>\n"
|
|
301
|
+
+ f" <ETag>{etag}</ETag>\n"
|
|
302
|
+
+ f" </Part>\n"
|
|
303
|
+
)
|
|
304
|
+
c_xml += "</CompleteMultipartUpload>"
|
|
305
|
+
resp = requests.post(
|
|
306
|
+
complete_url,
|
|
307
|
+
data=c_xml,
|
|
308
|
+
timeout=3,
|
|
309
|
+
)
|
|
310
|
+
etag_full = resp.headers.get("ETag", "").strip("'").strip('"')
|
|
311
|
+
else:
|
|
312
|
+
# Single file upload
|
|
313
|
+
resp = requests.put(upload_urls[0],
|
|
314
|
+
data=fd,
|
|
315
|
+
timeout=3)
|
|
316
|
+
etag_full = resp.headers.get("ETag", "").strip("'").strip('"')
|
|
317
|
+
if not etag_full:
|
|
285
318
|
raise ValueError(
|
|
286
|
-
f"Upload failed with {
|
|
319
|
+
f"Upload failed with {resp.status_code}: {resp.reason} "
|
|
320
|
+
f"({resp.headers})")
|
|
321
|
+
return etag_full
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import base64
|
|
2
1
|
import hashlib
|
|
3
2
|
from unittest import mock
|
|
4
3
|
import pathlib
|
|
4
|
+
import random
|
|
5
5
|
import uuid
|
|
6
6
|
|
|
7
7
|
import botocore.exceptions
|
|
8
8
|
import pytest
|
|
9
9
|
import requests
|
|
10
10
|
|
|
11
|
-
from dcor_shared import s3, sha256sum
|
|
11
|
+
from dcor_shared import get_ckan_config_option, s3, sha256sum
|
|
12
12
|
from dcor_shared.testing import upload_presigned_to_s3
|
|
13
13
|
|
|
14
14
|
|
|
@@ -189,14 +189,61 @@ def test_presigned_upload():
|
|
|
189
189
|
bucket_name = f"test-circle-{uuid.uuid4()}"
|
|
190
190
|
rid = str(uuid.uuid4())
|
|
191
191
|
object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
|
|
192
|
-
|
|
193
|
-
|
|
192
|
+
upload_urls, complete_url = s3.create_presigned_upload_urls(
|
|
193
|
+
bucket_name=bucket_name,
|
|
194
|
+
object_name=object_name,
|
|
195
|
+
file_size=path.stat().st_size,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
assert len(upload_urls) == 1, "no multipart upload"
|
|
199
|
+
assert complete_url is None, "no multipart upload"
|
|
194
200
|
|
|
195
201
|
# This is what DCOR-Aid would do to upload the file
|
|
196
|
-
upload_presigned_to_s3(
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
202
|
+
etag = upload_presigned_to_s3(
|
|
203
|
+
path=path,
|
|
204
|
+
upload_urls=upload_urls,
|
|
205
|
+
complete_url=complete_url,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
assert hashlib.md5(path.read_bytes()).hexdigest() == etag
|
|
209
|
+
|
|
210
|
+
hash_exp = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
211
|
+
hash_act = s3.compute_checksum(bucket_name=bucket_name,
|
|
212
|
+
object_name=object_name)
|
|
213
|
+
assert hash_exp == hash_act
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def test_presigned_upload_multipart(tmp_path):
|
|
217
|
+
path = tmp_path / "calibration_beads_47.rtdc"
|
|
218
|
+
with path.open("wb") as fd:
|
|
219
|
+
for ii in range(20): # 20 MiB
|
|
220
|
+
fd.write(bytearray(1024**2))
|
|
221
|
+
|
|
222
|
+
# This is what would happen on the server when DCOR-Aid requests an
|
|
223
|
+
# upload URL
|
|
224
|
+
bucket_name = f"test-circle-{uuid.uuid4()}"
|
|
225
|
+
rid = str(uuid.uuid4())
|
|
226
|
+
object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
|
|
227
|
+
upload_urls, complete_url = s3.create_presigned_upload_urls(
|
|
228
|
+
bucket_name=bucket_name,
|
|
229
|
+
object_name=object_name,
|
|
230
|
+
# pretend as if we had a huge file
|
|
231
|
+
file_size=int(1.5*1024**3),
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
assert len(upload_urls) == 2, "multipart upload"
|
|
235
|
+
assert complete_url is not None, "multipart upload"
|
|
236
|
+
|
|
237
|
+
# This is what DCOR-Aid would do to upload the file
|
|
238
|
+
etag = upload_presigned_to_s3(
|
|
239
|
+
path=path,
|
|
240
|
+
upload_urls=upload_urls,
|
|
241
|
+
complete_url=complete_url,
|
|
242
|
+
)
|
|
243
|
+
# This is how Amazon computes the ETag for multipart uploads.
|
|
244
|
+
md5part = hashlib.md5(bytearray(1024**2)*10).digest()
|
|
245
|
+
etag_exp = hashlib.md5(md5part+md5part).hexdigest() + "-2"
|
|
246
|
+
assert etag == etag_exp
|
|
200
247
|
|
|
201
248
|
hash_exp = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
202
249
|
hash_act = s3.compute_checksum(bucket_name=bucket_name,
|
|
@@ -205,7 +252,7 @@ def test_presigned_upload():
|
|
|
205
252
|
|
|
206
253
|
|
|
207
254
|
def test_presigned_upload_private_by_default():
|
|
208
|
-
"""The
|
|
255
|
+
"""The presigned upload should be private by default"""
|
|
209
256
|
path = data_path / "calibration_beads_47.rtdc"
|
|
210
257
|
|
|
211
258
|
# This is what would happen on the server when DCOR-Aid requests an
|
|
@@ -213,16 +260,20 @@ def test_presigned_upload_private_by_default():
|
|
|
213
260
|
bucket_name = f"test-circle-{uuid.uuid4()}"
|
|
214
261
|
rid = str(uuid.uuid4())
|
|
215
262
|
object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
|
|
216
|
-
|
|
217
|
-
|
|
263
|
+
|
|
264
|
+
upload_urls, complete_url = s3.create_presigned_upload_urls(
|
|
265
|
+
bucket_name=bucket_name,
|
|
266
|
+
object_name=object_name,
|
|
267
|
+
file_size=path.stat().st_size,
|
|
268
|
+
)
|
|
218
269
|
|
|
219
270
|
# This is what DCOR-Aid would do to upload the file
|
|
220
|
-
upload_presigned_to_s3(
|
|
221
|
-
|
|
222
|
-
|
|
271
|
+
upload_presigned_to_s3(path=path,
|
|
272
|
+
upload_urls=upload_urls,
|
|
273
|
+
complete_url=complete_url,
|
|
223
274
|
)
|
|
224
275
|
|
|
225
|
-
s3_url =
|
|
276
|
+
s3_url = upload_urls[0].split("?")[0]
|
|
226
277
|
|
|
227
278
|
# attempt to download the data
|
|
228
279
|
response = requests.get(s3_url)
|
|
@@ -245,36 +296,34 @@ def test_presigned_upload_wrong_access():
|
|
|
245
296
|
bucket_name = f"test-circle-{uuid.uuid4()}"
|
|
246
297
|
rid = str(uuid.uuid4())
|
|
247
298
|
object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
object_name_bad = f"resource/{rid2[:3]}/{rid2[3:6]}/{rid2[6:]}"
|
|
254
|
-
# replace the old with the bad object name
|
|
255
|
-
new_policy = base64.b64encode(
|
|
256
|
-
base64.b64decode(fields["policy"])
|
|
257
|
-
.decode("utf-8")
|
|
258
|
-
.replace(object_name, object_name_bad)
|
|
259
|
-
.encode("utf-8")
|
|
299
|
+
|
|
300
|
+
upload_urls, complete_url = s3.create_presigned_upload_urls(
|
|
301
|
+
bucket_name=bucket_name,
|
|
302
|
+
object_name=object_name,
|
|
303
|
+
file_size=path.stat().st_size,
|
|
260
304
|
)
|
|
261
|
-
|
|
262
|
-
assert
|
|
263
|
-
|
|
264
|
-
|
|
305
|
+
|
|
306
|
+
assert len(upload_urls) == 1, "no multipart upload"
|
|
307
|
+
assert complete_url is None, "no multipart upload"
|
|
308
|
+
|
|
309
|
+
# Try to change the signature
|
|
310
|
+
parts = upload_urls[0].split("&")
|
|
311
|
+
for ii, part in enumerate(parts):
|
|
312
|
+
if part.startswith("Signature="):
|
|
313
|
+
new_sig = part[:-10] + part[-10:][::-1]
|
|
314
|
+
parts[ii] = new_sig
|
|
315
|
+
break
|
|
316
|
+
else:
|
|
317
|
+
assert False
|
|
318
|
+
upload_urls[0] = "&".join(parts)
|
|
265
319
|
|
|
266
320
|
with pytest.raises(ValueError, match="Upload failed with 403: Forbidden"):
|
|
267
321
|
# This is what DCOR-Aid would do to upload the file
|
|
268
|
-
upload_presigned_to_s3(
|
|
269
|
-
|
|
270
|
-
|
|
322
|
+
upload_presigned_to_s3(path=path,
|
|
323
|
+
upload_urls=upload_urls,
|
|
324
|
+
complete_url=complete_url,
|
|
271
325
|
)
|
|
272
326
|
|
|
273
|
-
with pytest.raises(botocore.exceptions.ClientError, match="Not Found"):
|
|
274
|
-
# Make sure the file does not exist
|
|
275
|
-
s3.compute_checksum(bucket_name=bucket_name,
|
|
276
|
-
object_name=object_name_bad)
|
|
277
|
-
|
|
278
327
|
|
|
279
328
|
def test_presigned_upload_wrong_key():
|
|
280
329
|
"""Same as `test_presigned_upload_wrong_access` but no policy change"""
|
|
@@ -285,19 +334,28 @@ def test_presigned_upload_wrong_key():
|
|
|
285
334
|
bucket_name = f"test-circle-{uuid.uuid4()}"
|
|
286
335
|
rid = str(uuid.uuid4())
|
|
287
336
|
object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
|
|
288
|
-
|
|
289
|
-
|
|
337
|
+
|
|
338
|
+
upload_urls, complete_url = s3.create_presigned_upload_urls(
|
|
339
|
+
bucket_name=bucket_name,
|
|
340
|
+
object_name=object_name,
|
|
341
|
+
file_size=path.stat().st_size,
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
assert len(upload_urls) == 1, "no multipart upload"
|
|
345
|
+
assert complete_url is None, "no multipart upload"
|
|
346
|
+
|
|
290
347
|
# Try to upload the file under a different object name
|
|
291
348
|
# (this tests the S3 access restrictions)
|
|
292
349
|
rid2 = str(uuid.uuid4())
|
|
293
350
|
object_name_bad = f"resource/{rid2[:3]}/{rid2[3:6]}/{rid2[6:]}"
|
|
294
|
-
|
|
351
|
+
# replace the old with the bad object name
|
|
352
|
+
upload_urls[0] = upload_urls[0].replace(object_name, object_name_bad)
|
|
295
353
|
|
|
296
354
|
with pytest.raises(ValueError, match="Upload failed with 403: Forbidden"):
|
|
297
355
|
# This is what DCOR-Aid would do to upload the file
|
|
298
|
-
upload_presigned_to_s3(
|
|
299
|
-
|
|
300
|
-
|
|
356
|
+
upload_presigned_to_s3(path=path,
|
|
357
|
+
upload_urls=upload_urls,
|
|
358
|
+
complete_url=complete_url,
|
|
301
359
|
)
|
|
302
360
|
|
|
303
361
|
with pytest.raises(botocore.exceptions.ClientError, match="Not Found"):
|
|
@@ -23,6 +23,11 @@ data_path = pathlib.Path(__file__).parent / "data"
|
|
|
23
23
|
|
|
24
24
|
def setup_s3_resource_on_ckan(private=False, resource_path=None):
|
|
25
25
|
"""Create an S3 resource in CKAN"""
|
|
26
|
+
if resource_path is None:
|
|
27
|
+
resource_path = data_path / "calibration_beads_47.rtdc"
|
|
28
|
+
|
|
29
|
+
file_size = resource_path.stat().st_size
|
|
30
|
+
|
|
26
31
|
user = factories.User()
|
|
27
32
|
owner_org = factories.Organization(users=[{
|
|
28
33
|
'name': user['id'],
|
|
@@ -34,18 +39,18 @@ def setup_s3_resource_on_ckan(private=False, resource_path=None):
|
|
|
34
39
|
|
|
35
40
|
# Upload the resource to S3 (note that it is not required that the
|
|
36
41
|
# dataset exists)
|
|
37
|
-
response = helpers.call_action("
|
|
42
|
+
response = helpers.call_action("resource_upload_s3_urls",
|
|
38
43
|
test_context,
|
|
39
44
|
organization_id=owner_org["id"],
|
|
45
|
+
file_size=file_size,
|
|
40
46
|
)
|
|
41
47
|
rid = response["resource_id"]
|
|
42
|
-
if resource_path is None:
|
|
43
|
-
resource_path = data_path / "calibration_beads_47.rtdc"
|
|
44
48
|
|
|
45
49
|
upload_presigned_to_s3(
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
50
|
+
path=resource_path,
|
|
51
|
+
upload_urls=response["upload_urls"],
|
|
52
|
+
complete_url=response["complete_url"],
|
|
53
|
+
)
|
|
49
54
|
|
|
50
55
|
# Create the dataset
|
|
51
56
|
pkg_dict = helpers.call_action("package_create",
|
|
@@ -69,7 +74,7 @@ def setup_s3_resource_on_ckan(private=False, resource_path=None):
|
|
|
69
74
|
}],
|
|
70
75
|
)
|
|
71
76
|
assert new_pkg_dict["package"]["num_resources"] == 1
|
|
72
|
-
s3_url = response["
|
|
77
|
+
s3_url = response["upload_urls"][0].split("?")[0]
|
|
73
78
|
return rid, s3_url, new_pkg_dict, owner_org
|
|
74
79
|
|
|
75
80
|
|
|
@@ -115,6 +120,25 @@ def test_create_presigned_url(enqueue_job_mock, tmp_path):
|
|
|
115
120
|
"490efdf5d9bb4cd4b2a6bcf2fe54d4dc201c38530140bcb168980bf8bf846c73"
|
|
116
121
|
|
|
117
122
|
|
|
123
|
+
@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
|
|
124
|
+
@pytest.mark.usefixtures('clean_db', 'with_request_context')
|
|
125
|
+
@mock.patch('ckan.plugins.toolkit.enqueue_job',
|
|
126
|
+
side_effect=synchronous_enqueue_job)
|
|
127
|
+
def test_get_s3_attributes_for_artifact(enqueue_job_mock):
|
|
128
|
+
rid, _, _, org_dict = setup_s3_resource_on_ckan()
|
|
129
|
+
|
|
130
|
+
# Make sure the resource exists
|
|
131
|
+
res_dict = helpers.call_action("resource_show", id=rid)
|
|
132
|
+
assert res_dict["id"] == rid, "sanity check"
|
|
133
|
+
|
|
134
|
+
# get the size
|
|
135
|
+
meta = s3cc.get_s3_attributes_for_artifact(rid)
|
|
136
|
+
assert meta["size"] == 904729
|
|
137
|
+
assert meta["success"]
|
|
138
|
+
assert meta["etag"]
|
|
139
|
+
assert meta["server"]
|
|
140
|
+
|
|
141
|
+
|
|
118
142
|
@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
|
|
119
143
|
@pytest.mark.usefixtures('clean_db', 'with_request_context')
|
|
120
144
|
@mock.patch('ckan.plugins.toolkit.enqueue_job',
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|