caltechdata-api 2.2.3__tar.gz → 2.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/PKG-INFO +2 -2
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/caltechdata_edit.py +35 -9
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/caltechdata_write.py +11 -5
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/download_from_record.py +60 -7
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api.egg-info/PKG-INFO +2 -2
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api.egg-info/SOURCES.txt +3 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/pyproject.toml +2 -1
- caltechdata_api-2.2.4/tests/test_accept_verify.py +53 -0
- caltechdata_api-2.2.4/tests/test_download_verify.py +148 -0
- caltechdata_api-2.2.4/tests/test_edit_verify.py +149 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/LICENSE +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/README.md +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/__init__.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/cli.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/customize_schema.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/download_file.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/get_metadata.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/md_to_json.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/utils.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/date_types.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/description_types.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/identifier_types.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/licenses.csv +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/relation_types.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/resource_types.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/roles.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/title_types.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies.yaml +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api.egg-info/dependency_links.txt +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api.egg-info/entry_points.txt +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api.egg-info/requires.txt +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api.egg-info/top_level.txt +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/setup.cfg +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/tests/test_download.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/tests/test_rdm.py +0 -0
- {caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/tests/test_unit.py +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: caltechdata_api
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.4
|
|
4
4
|
Summary: Python wrapper for CaltechDATA API.
|
|
5
5
|
Author: Rohan Bhattarai, Elizabeth Won
|
|
6
|
-
Author-email: Thomas E Morrell <tmorrell@caltech.edu>, Alexander A Abakah <aabakah@caltech.edu>, Kshemaahna Nagi <knagi@caltech.edu>
|
|
6
|
+
Author-email: Thomas E Morrell <tmorrell@caltech.edu>, Alexander A Abakah <aabakah@caltech.edu>, Kshemaahna Nagi <knagi@caltech.edu>, Robert Doiel <rsdoiel@caltech.edu>
|
|
7
7
|
Maintainer-email: Thomas E Morrell <tmorrell@caltech.edu>
|
|
8
8
|
License-Expression: BSD-3-Clause
|
|
9
9
|
Project-URL: Homepage, https://github.com/caltechlibrary/caltechdata_api
|
|
@@ -14,7 +14,7 @@ def caltechdata_unembargo(token, ids, production=False):
|
|
|
14
14
|
print("caltechdaua_unembargo is not yet re-implemented")
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def caltechdata_accept(ids, token=None, production=False):
|
|
17
|
+
def caltechdata_accept(ids, token=None, production=False, verify=True):
|
|
18
18
|
# Accept a record into a community. Only accepts the first community
|
|
19
19
|
# request
|
|
20
20
|
|
|
@@ -34,11 +34,15 @@ def caltechdata_accept(ids, token=None, production=False):
|
|
|
34
34
|
|
|
35
35
|
for idv in ids:
|
|
36
36
|
result = requests.get(
|
|
37
|
-
url + "/api/records/" + idv + "/draft/review",
|
|
37
|
+
url + "/api/records/" + idv + "/draft/review",
|
|
38
|
+
headers=headers,
|
|
39
|
+
verify=verify,
|
|
38
40
|
)
|
|
39
41
|
if result.status_code != 200:
|
|
40
42
|
result = requests.get(
|
|
41
|
-
url + "/api/records/" + idv + "/requests",
|
|
43
|
+
url + "/api/records/" + idv + "/requests",
|
|
44
|
+
headers=headers,
|
|
45
|
+
verify=verify,
|
|
42
46
|
)
|
|
43
47
|
if result.status_code != 200:
|
|
44
48
|
raise Exception(result.text)
|
|
@@ -51,7 +55,7 @@ def caltechdata_accept(ids, token=None, production=False):
|
|
|
51
55
|
"format": "html",
|
|
52
56
|
}
|
|
53
57
|
}
|
|
54
|
-
result = requests.post(accept_link, json=data, headers=headers)
|
|
58
|
+
result = requests.post(accept_link, json=data, headers=headers, verify=verify)
|
|
55
59
|
if result.status_code != 200:
|
|
56
60
|
raise Exception(result.text)
|
|
57
61
|
|
|
@@ -117,6 +121,7 @@ def caltechdata_edit(
|
|
|
117
121
|
keepfiles=False,
|
|
118
122
|
return_id=False,
|
|
119
123
|
local=False,
|
|
124
|
+
verify=True,
|
|
120
125
|
):
|
|
121
126
|
# Make a copy of the metadata to make sure our local changes don't leak
|
|
122
127
|
metadata = copy.deepcopy(metadata)
|
|
@@ -159,28 +164,32 @@ def caltechdata_edit(
|
|
|
159
164
|
url + "/api/records/" + idv + "/communities",
|
|
160
165
|
headers=headers,
|
|
161
166
|
data=json.dumps({"communities": [{"id": community}]}),
|
|
167
|
+
verify=verify,
|
|
162
168
|
)
|
|
163
169
|
if result.status_code != 200:
|
|
164
170
|
print(result.url)
|
|
165
171
|
raise Exception(result.text)
|
|
166
|
-
caltechdata_accept([idv], token, production)
|
|
172
|
+
caltechdata_accept([idv], token, production, verify=verify)
|
|
167
173
|
|
|
168
174
|
# Check status
|
|
169
175
|
existing = requests.get(
|
|
170
176
|
url + "/api/records/" + idv,
|
|
171
177
|
headers=headers,
|
|
178
|
+
verify=verify,
|
|
172
179
|
)
|
|
173
180
|
if existing.status_code != 200:
|
|
174
181
|
# Might have a draft
|
|
175
182
|
existing = requests.get(
|
|
176
183
|
url + "/api/records/" + idv + "/draft",
|
|
177
184
|
headers=headers,
|
|
185
|
+
verify=verify,
|
|
178
186
|
)
|
|
179
187
|
if existing.status_code != 200:
|
|
180
188
|
# Try back again to the record
|
|
181
189
|
existing = requests.get(
|
|
182
190
|
url + "/api/records/" + idv,
|
|
183
191
|
headers=headers,
|
|
192
|
+
verify=verify,
|
|
184
193
|
)
|
|
185
194
|
|
|
186
195
|
if existing.status_code != 200:
|
|
@@ -202,6 +211,7 @@ def caltechdata_edit(
|
|
|
202
211
|
result = requests.post(
|
|
203
212
|
url + "/api/records/" + idv + "/versions",
|
|
204
213
|
headers=headers,
|
|
214
|
+
verify=verify,
|
|
205
215
|
)
|
|
206
216
|
if result.status_code != 201:
|
|
207
217
|
raise Exception(result.text)
|
|
@@ -304,12 +314,14 @@ def caltechdata_edit(
|
|
|
304
314
|
result = requests.get(
|
|
305
315
|
url + "/api/records/" + idv + "/draft",
|
|
306
316
|
headers=headers,
|
|
317
|
+
verify=verify,
|
|
307
318
|
)
|
|
308
319
|
if result.status_code != 200:
|
|
309
320
|
# We make a draft
|
|
310
321
|
result = requests.post(
|
|
311
322
|
url + "/api/records/" + idv + "/draft",
|
|
312
323
|
headers=headers,
|
|
324
|
+
verify=verify,
|
|
313
325
|
)
|
|
314
326
|
if result.status_code != 201:
|
|
315
327
|
raise Exception(result.text)
|
|
@@ -324,16 +336,28 @@ def caltechdata_edit(
|
|
|
324
336
|
url + "/api/records/" + idv + "/draft",
|
|
325
337
|
headers=headers,
|
|
326
338
|
json=data,
|
|
339
|
+
verify=verify,
|
|
327
340
|
)
|
|
328
341
|
if result.status_code != 200:
|
|
329
342
|
raise Exception(result.text)
|
|
330
343
|
file_upload_link = result.json()["links"]["files"]
|
|
331
344
|
if files:
|
|
332
345
|
write_files_rdm(
|
|
333
|
-
files,
|
|
346
|
+
files,
|
|
347
|
+
file_upload_link,
|
|
348
|
+
headers,
|
|
349
|
+
f_headers,
|
|
350
|
+
keepfiles=keepfiles,
|
|
351
|
+
verify=verify,
|
|
334
352
|
)
|
|
335
353
|
if file_links:
|
|
336
|
-
add_file_links(
|
|
354
|
+
add_file_links(
|
|
355
|
+
file_upload_link,
|
|
356
|
+
file_links,
|
|
357
|
+
headers,
|
|
358
|
+
keepfiles=keepfiles,
|
|
359
|
+
verify=verify,
|
|
360
|
+
)
|
|
337
361
|
|
|
338
362
|
else:
|
|
339
363
|
# We want files to stay the same as the existing record
|
|
@@ -345,6 +369,7 @@ def caltechdata_edit(
|
|
|
345
369
|
url + "/api/records/" + idv + "/draft",
|
|
346
370
|
headers=headers,
|
|
347
371
|
json=data,
|
|
372
|
+
verify=verify,
|
|
348
373
|
)
|
|
349
374
|
if result.status_code != 200:
|
|
350
375
|
time.sleep(3)
|
|
@@ -352,16 +377,17 @@ def caltechdata_edit(
|
|
|
352
377
|
url + "/api/records/" + idv + "/draft",
|
|
353
378
|
headers=headers,
|
|
354
379
|
json=data,
|
|
380
|
+
verify=verify,
|
|
355
381
|
)
|
|
356
382
|
if result.status_code != 200:
|
|
357
383
|
raise Exception(result.text)
|
|
358
384
|
|
|
359
385
|
if publish:
|
|
360
386
|
publish_link = f"{url}/api/records/{idv}/draft/actions/publish"
|
|
361
|
-
result = requests.post(publish_link, headers=headers)
|
|
387
|
+
result = requests.post(publish_link, headers=headers, verify=verify)
|
|
362
388
|
if result.status_code != 202:
|
|
363
389
|
time.sleep(3)
|
|
364
|
-
result = requests.post(publish_link, headers=headers)
|
|
390
|
+
result = requests.post(publish_link, headers=headers, verify=verify)
|
|
365
391
|
if result.status_code != 202:
|
|
366
392
|
raise Exception(result.text)
|
|
367
393
|
if return_id:
|
|
@@ -67,7 +67,7 @@ def write_files_rdm(
|
|
|
67
67
|
raise Exception(result.text)
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
def add_file_links(file_upload_link, file_links, headers, keepfiles=False):
|
|
70
|
+
def add_file_links(file_upload_link, file_links, headers, keepfiles=False, verify=True):
|
|
71
71
|
# Currently configured for S3 links, assuming all are at the same endpoint
|
|
72
72
|
# Links are urls, or (file name, url) pairs when the url file names are not
|
|
73
73
|
# unique across the bucket
|
|
@@ -90,21 +90,27 @@ def add_file_links(file_upload_link, file_links, headers, keepfiles=False):
|
|
|
90
90
|
{"key": file, "size": size, "transfer": {"type": "R", "url": link}}
|
|
91
91
|
)
|
|
92
92
|
# Now we see if any existing files need to be replaced
|
|
93
|
-
result = requests.get(file_upload_link, headers=headers)
|
|
93
|
+
result = requests.get(file_upload_link, headers=headers, verify=verify)
|
|
94
94
|
if result.status_code == 200:
|
|
95
95
|
ex_files = result.json()["entries"]
|
|
96
96
|
for ex in ex_files:
|
|
97
97
|
if ex["key"] in f_list:
|
|
98
|
-
result = requests.delete(
|
|
98
|
+
result = requests.delete(
|
|
99
|
+
ex["links"]["self"], headers=headers, verify=verify
|
|
100
|
+
)
|
|
99
101
|
if result.status_code != 204:
|
|
100
102
|
raise Exception(result.text)
|
|
101
103
|
else:
|
|
102
104
|
if keepfiles == False:
|
|
103
|
-
result = requests.delete(
|
|
105
|
+
result = requests.delete(
|
|
106
|
+
ex["links"]["self"], headers=headers, verify=verify
|
|
107
|
+
)
|
|
104
108
|
if result.status_code != 204:
|
|
105
109
|
raise Exception(result.text)
|
|
106
110
|
|
|
107
|
-
result = requests.post(
|
|
111
|
+
result = requests.post(
|
|
112
|
+
file_upload_link, headers=headers, json=file_json, verify=verify
|
|
113
|
+
)
|
|
108
114
|
if result.status_code != 201:
|
|
109
115
|
raise Exception(result.text)
|
|
110
116
|
|
|
@@ -19,7 +19,9 @@ from urllib.parse import urlsplit
|
|
|
19
19
|
from typing import Any, Dict, Optional, Sequence
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
def get_base_url(
|
|
22
|
+
def get_base_url(
|
|
23
|
+
production: bool = True, authors: bool = False, local: bool = False
|
|
24
|
+
) -> str:
|
|
23
25
|
"""Get the base URL of the repository to download from.
|
|
24
26
|
|
|
25
27
|
Parameters
|
|
@@ -31,6 +33,11 @@ def get_base_url(production: bool = True, authors: bool = False) -> str:
|
|
|
31
33
|
authors
|
|
32
34
|
Whether to use CaltechAUTHORS instead of CaltechDATA.
|
|
33
35
|
|
|
36
|
+
local
|
|
37
|
+
Whether to use a local instance, at ``https://127.0.0.1:5000``,
|
|
38
|
+
instead of the test (development) instance. Ignored when
|
|
39
|
+
``production`` is ``True``, as in :func:`caltechdata_write`.
|
|
40
|
+
|
|
34
41
|
Returns
|
|
35
42
|
-------
|
|
36
43
|
str
|
|
@@ -40,6 +47,8 @@ def get_base_url(production: bool = True, authors: bool = False) -> str:
|
|
|
40
47
|
if authors:
|
|
41
48
|
return "https://authors.library.caltech.edu"
|
|
42
49
|
return "https://data.caltech.edu"
|
|
50
|
+
elif local:
|
|
51
|
+
return "https://127.0.0.1:5000"
|
|
43
52
|
else:
|
|
44
53
|
if authors:
|
|
45
54
|
return "https://authors.caltechlibrary.dev"
|
|
@@ -51,6 +60,8 @@ def get_files_from_record(
|
|
|
51
60
|
production: bool = True,
|
|
52
61
|
authors: bool = False,
|
|
53
62
|
token: Optional[str] = None,
|
|
63
|
+
local: bool = False,
|
|
64
|
+
verify: bool = True,
|
|
54
65
|
) -> Dict[str, Any]:
|
|
55
66
|
"""Get a dictionary of files associated with a record.
|
|
56
67
|
|
|
@@ -74,25 +85,37 @@ def get_files_from_record(
|
|
|
74
85
|
token is given and the published record is not found, the
|
|
75
86
|
record's draft is tried as well.
|
|
76
87
|
|
|
88
|
+
local
|
|
89
|
+
Whether to query a local instance, at ``https://127.0.0.1:5000``,
|
|
90
|
+
instead of the test (development) instance.
|
|
91
|
+
|
|
92
|
+
verify
|
|
93
|
+
Whether to verify the server's TLS certificate. Use ``False``
|
|
94
|
+
for a local instance with a self-signed certificate.
|
|
95
|
+
|
|
77
96
|
Returns
|
|
78
97
|
-------
|
|
79
98
|
dict
|
|
80
99
|
A dictionary with the file names as keys and the metadata
|
|
81
100
|
associated with those files as values.
|
|
82
101
|
"""
|
|
83
|
-
base_url = get_base_url(production=production, authors=authors)
|
|
102
|
+
base_url = get_base_url(production=production, authors=authors, local=local)
|
|
84
103
|
headers = {}
|
|
85
104
|
if token is not None:
|
|
86
105
|
headers["Authorization"] = "Bearer %s" % token
|
|
87
106
|
|
|
88
|
-
r = requests.get(
|
|
107
|
+
r = requests.get(
|
|
108
|
+
f"{base_url}/api/records/{record_id}/files", headers=headers, verify=verify
|
|
109
|
+
)
|
|
89
110
|
try:
|
|
90
111
|
if r.status_code == 404 and token is not None:
|
|
91
112
|
# An unpublished record has no published version to return,
|
|
92
113
|
# so fall back to its draft, which requires authentication.
|
|
93
114
|
r.close()
|
|
94
115
|
r = requests.get(
|
|
95
|
-
f"{base_url}/api/records/{record_id}/draft/files",
|
|
116
|
+
f"{base_url}/api/records/{record_id}/draft/files",
|
|
117
|
+
headers=headers,
|
|
118
|
+
verify=verify,
|
|
96
119
|
)
|
|
97
120
|
|
|
98
121
|
r.raise_for_status()
|
|
@@ -114,6 +137,8 @@ def download_files_from_record(
|
|
|
114
137
|
production: bool = True,
|
|
115
138
|
authors: bool = False,
|
|
116
139
|
token: Optional[str] = None,
|
|
140
|
+
local: bool = False,
|
|
141
|
+
verify: bool = True,
|
|
117
142
|
):
|
|
118
143
|
"""Download one or more files from a record.
|
|
119
144
|
|
|
@@ -157,12 +182,28 @@ def download_files_from_record(
|
|
|
157
182
|
files from restricted records. If omitted, the requests are
|
|
158
183
|
made without authentication and only public records are
|
|
159
184
|
available.
|
|
185
|
+
|
|
186
|
+
local
|
|
187
|
+
Whether to download from a local instance, at
|
|
188
|
+
``https://127.0.0.1:5000``, instead of the test (development)
|
|
189
|
+
instance.
|
|
190
|
+
|
|
191
|
+
verify
|
|
192
|
+
Whether to verify the server's TLS certificate, for requests to
|
|
193
|
+
the instance itself. Use ``False`` for a local instance with a
|
|
194
|
+
self-signed certificate. Redirects to other hosts are always
|
|
195
|
+
verified.
|
|
160
196
|
"""
|
|
161
197
|
if not os.path.isdir(output_path):
|
|
162
198
|
raise IOError(f"{output_path} is not an extant directory")
|
|
163
199
|
|
|
164
200
|
files = get_files_from_record(
|
|
165
|
-
record_id,
|
|
201
|
+
record_id,
|
|
202
|
+
production=production,
|
|
203
|
+
authors=authors,
|
|
204
|
+
token=token,
|
|
205
|
+
local=local,
|
|
206
|
+
verify=verify,
|
|
166
207
|
)
|
|
167
208
|
if filenames is None:
|
|
168
209
|
filenames = sorted(files.keys())
|
|
@@ -181,7 +222,11 @@ def download_files_from_record(
|
|
|
181
222
|
|
|
182
223
|
content_url = entry["links"]["content"]
|
|
183
224
|
download_content(
|
|
184
|
-
content_url,
|
|
225
|
+
content_url,
|
|
226
|
+
output_file,
|
|
227
|
+
max_redirects=max_redirects,
|
|
228
|
+
token=token,
|
|
229
|
+
verify=verify,
|
|
185
230
|
)
|
|
186
231
|
|
|
187
232
|
|
|
@@ -190,6 +235,7 @@ def download_content(
|
|
|
190
235
|
fname: os.PathLike,
|
|
191
236
|
max_redirects=5,
|
|
192
237
|
token: Optional[str] = None,
|
|
238
|
+
verify: bool = True,
|
|
193
239
|
):
|
|
194
240
|
"""Download the contents of a file.
|
|
195
241
|
|
|
@@ -216,6 +262,11 @@ def download_content(
|
|
|
216
262
|
content from restricted records. If omitted, the request is
|
|
217
263
|
made without authentication. The token is only sent to the
|
|
218
264
|
host of ``content_url``, not to hosts we are redirected to.
|
|
265
|
+
|
|
266
|
+
verify
|
|
267
|
+
Whether to verify the server's TLS certificate. Like the token,
|
|
268
|
+
it applies only to the host of ``content_url``: a redirect to
|
|
269
|
+
another host is always verified.
|
|
219
270
|
"""
|
|
220
271
|
url = content_url
|
|
221
272
|
host = urlsplit(content_url).netloc
|
|
@@ -228,7 +279,9 @@ def download_content(
|
|
|
228
279
|
if token is not None and urlsplit(url).netloc == host:
|
|
229
280
|
headers["Authorization"] = "Bearer %s" % token
|
|
230
281
|
|
|
231
|
-
|
|
282
|
+
# A redirect to another host (the file store) is always verified
|
|
283
|
+
verify_url = verify if urlsplit(url).netloc == host else True
|
|
284
|
+
with requests.get(url, stream=True, headers=headers, verify=verify_url) as r:
|
|
232
285
|
r.raise_for_status()
|
|
233
286
|
if "Location" in r.headers:
|
|
234
287
|
# Redirection - follow to the next URL
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: caltechdata_api
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.4
|
|
4
4
|
Summary: Python wrapper for CaltechDATA API.
|
|
5
5
|
Author: Rohan Bhattarai, Elizabeth Won
|
|
6
|
-
Author-email: Thomas E Morrell <tmorrell@caltech.edu>, Alexander A Abakah <aabakah@caltech.edu>, Kshemaahna Nagi <knagi@caltech.edu>
|
|
6
|
+
Author-email: Thomas E Morrell <tmorrell@caltech.edu>, Alexander A Abakah <aabakah@caltech.edu>, Kshemaahna Nagi <knagi@caltech.edu>, Robert Doiel <rsdoiel@caltech.edu>
|
|
7
7
|
Maintainer-email: Thomas E Morrell <tmorrell@caltech.edu>
|
|
8
8
|
License-Expression: BSD-3-Clause
|
|
9
9
|
Project-URL: Homepage, https://github.com/caltechlibrary/caltechdata_api
|
|
@@ -26,6 +26,9 @@ caltechdata_api/vocabularies/relation_types.yaml
|
|
|
26
26
|
caltechdata_api/vocabularies/resource_types.yaml
|
|
27
27
|
caltechdata_api/vocabularies/roles.yaml
|
|
28
28
|
caltechdata_api/vocabularies/title_types.yaml
|
|
29
|
+
tests/test_accept_verify.py
|
|
29
30
|
tests/test_download.py
|
|
31
|
+
tests/test_download_verify.py
|
|
32
|
+
tests/test_edit_verify.py
|
|
30
33
|
tests/test_rdm.py
|
|
31
34
|
tests/test_unit.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "caltechdata_api"
|
|
7
|
-
version = "2.2.
|
|
7
|
+
version = "2.2.4"
|
|
8
8
|
description = "Python wrapper for CaltechDATA API."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -16,6 +16,7 @@ authors = [
|
|
|
16
16
|
{name = "Elizabeth Won"},
|
|
17
17
|
{name = "Alexander A Abakah", email = "aabakah@caltech.edu"},
|
|
18
18
|
{name = "Kshemaahna Nagi", email = "knagi@caltech.edu"},
|
|
19
|
+
{name = "Robert Doiel", email = "rsdoiel@caltech.edu"},
|
|
19
20
|
]
|
|
20
21
|
maintainers = [
|
|
21
22
|
{name = "Thomas E Morrell", email = "tmorrell@caltech.edu"},
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""caltechdata_accept passes its verify argument to every request it makes.
|
|
2
|
+
|
|
3
|
+
No network is used: requests is replaced by a fake that records the keyword
|
|
4
|
+
arguments of each call.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from types import SimpleNamespace
|
|
8
|
+
from unittest import mock
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
import requests
|
|
12
|
+
|
|
13
|
+
from caltechdata_api import caltechdata_accept
|
|
14
|
+
|
|
15
|
+
ACCEPT = "https://data.caltechlibrary.dev/api/requests/r1/actions/accept"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.fixture
|
|
19
|
+
def calls():
|
|
20
|
+
"""Replace requests.get/post; each call is recorded as (method, url, kwargs).
|
|
21
|
+
|
|
22
|
+
The draft's review link is missing (404) so the request list is read too,
|
|
23
|
+
which makes the run use every call caltechdata_accept has.
|
|
24
|
+
"""
|
|
25
|
+
seen = []
|
|
26
|
+
|
|
27
|
+
def get(url, *args, **kwargs):
|
|
28
|
+
seen.append(("GET", url, kwargs))
|
|
29
|
+
if url.endswith("/draft/review"):
|
|
30
|
+
return SimpleNamespace(status_code=404, text="no review")
|
|
31
|
+
hits = {"hits": {"hits": [{"links": {"actions": {"accept": ACCEPT}}}]}}
|
|
32
|
+
return SimpleNamespace(status_code=200, json=lambda: hits)
|
|
33
|
+
|
|
34
|
+
def post(url, *args, **kwargs):
|
|
35
|
+
seen.append(("POST", url, kwargs))
|
|
36
|
+
return SimpleNamespace(status_code=200, json=lambda: {}, text="")
|
|
37
|
+
|
|
38
|
+
with mock.patch.multiple(requests, get=get, post=post):
|
|
39
|
+
yield seen
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
43
|
+
def test_verify_reaches_every_request(calls, verify):
|
|
44
|
+
caltechdata_accept(["abcde-12345"], token="t", verify=verify)
|
|
45
|
+
assert [m for m, _, _ in calls] == ["GET", "GET", "POST"]
|
|
46
|
+
for method, url, kwargs in calls:
|
|
47
|
+
assert kwargs.get("verify", True) is verify, f"{method} {url}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_verify_defaults_to_true(calls):
|
|
51
|
+
caltechdata_accept(["abcde-12345"], token="t")
|
|
52
|
+
for method, url, kwargs in calls:
|
|
53
|
+
assert kwargs.get("verify", True) is True, f"{method} {url}"
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""download_from_record can target a local instance and skip TLS verification.
|
|
2
|
+
|
|
3
|
+
A test instance with a self-signed certificate (local=True) needs verify=False
|
|
4
|
+
on its requests. A redirect to another host (the file store) is still
|
|
5
|
+
verified. No network is used: requests.get is replaced by a fake that records
|
|
6
|
+
the keyword arguments of each call.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from unittest import mock
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
from caltechdata_api.download_from_record import (
|
|
15
|
+
download_content,
|
|
16
|
+
download_files_from_record,
|
|
17
|
+
get_base_url,
|
|
18
|
+
get_files_from_record,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
LOCAL = "https://127.0.0.1:5000"
|
|
22
|
+
STORE = "https://store.example.org/object"
|
|
23
|
+
RECORD = "abcde-12345"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Response:
|
|
27
|
+
def __init__(self, status=200, payload=None, headers=None, body=b""):
|
|
28
|
+
self.status_code = status
|
|
29
|
+
self._payload = payload
|
|
30
|
+
self.headers = headers or {}
|
|
31
|
+
self._body = body
|
|
32
|
+
|
|
33
|
+
def json(self):
|
|
34
|
+
return self._payload
|
|
35
|
+
|
|
36
|
+
def raise_for_status(self):
|
|
37
|
+
if self.status_code >= 400:
|
|
38
|
+
raise requests.HTTPError(str(self.status_code))
|
|
39
|
+
|
|
40
|
+
def iter_content(self, chunk_size=1024):
|
|
41
|
+
yield self._body
|
|
42
|
+
|
|
43
|
+
def close(self):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
def __enter__(self):
|
|
47
|
+
return self
|
|
48
|
+
|
|
49
|
+
def __exit__(self, *exc):
|
|
50
|
+
self.close()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def entries():
|
|
54
|
+
content = f"{LOCAL}/api/records/{RECORD}/files/a.txt/content"
|
|
55
|
+
return {"entries": [{"key": "a.txt", "links": {"content": content}}]}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@pytest.fixture
|
|
59
|
+
def calls():
|
|
60
|
+
"""Replace requests.get; each call is recorded as (url, kwargs)."""
|
|
61
|
+
seen = []
|
|
62
|
+
|
|
63
|
+
def get(url, *args, **kwargs):
|
|
64
|
+
seen.append((url, kwargs))
|
|
65
|
+
if url.endswith("/draft/files"):
|
|
66
|
+
return Response(200, entries())
|
|
67
|
+
if url.endswith("/files"):
|
|
68
|
+
return Response(404)
|
|
69
|
+
if url.endswith("/content"):
|
|
70
|
+
return Response(200, headers={"Location": STORE})
|
|
71
|
+
if url == STORE:
|
|
72
|
+
return Response(200, headers={"content-length": "2"}, body=b"hi")
|
|
73
|
+
raise AssertionError(f"unexpected GET {url}")
|
|
74
|
+
|
|
75
|
+
with mock.patch.object(requests, "get", get):
|
|
76
|
+
yield seen
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_get_base_url_local():
|
|
80
|
+
assert get_base_url(production=False, authors=True, local=True) == LOCAL
|
|
81
|
+
assert get_base_url(production=False, authors=False, local=True) == LOCAL
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_get_base_url_production_wins_over_local():
|
|
85
|
+
assert get_base_url(production=True, authors=True, local=True) == (
|
|
86
|
+
"https://authors.library.caltech.edu"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_get_base_url_unchanged_without_local():
|
|
91
|
+
assert get_base_url(production=False, authors=True) == (
|
|
92
|
+
"https://authors.caltechlibrary.dev"
|
|
93
|
+
)
|
|
94
|
+
assert get_base_url(production=True) == "https://data.caltech.edu"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
98
|
+
def test_get_files_uses_the_local_host_and_verify(calls, verify):
|
|
99
|
+
# the published files are not found, so the draft is tried as well
|
|
100
|
+
files = get_files_from_record(
|
|
101
|
+
RECORD, production=False, authors=True, local=True, token="t", verify=verify
|
|
102
|
+
)
|
|
103
|
+
assert list(files) == ["a.txt"]
|
|
104
|
+
assert [u for u, _ in calls] == [
|
|
105
|
+
f"{LOCAL}/api/records/{RECORD}/files",
|
|
106
|
+
f"{LOCAL}/api/records/{RECORD}/draft/files",
|
|
107
|
+
]
|
|
108
|
+
for url, kwargs in calls:
|
|
109
|
+
assert kwargs.get("verify", True) is verify, url
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
113
|
+
def test_download_verifies_other_hosts(calls, tmp_path, verify):
|
|
114
|
+
download_files_from_record(
|
|
115
|
+
RECORD,
|
|
116
|
+
tmp_path,
|
|
117
|
+
production=False,
|
|
118
|
+
authors=True,
|
|
119
|
+
local=True,
|
|
120
|
+
token="t",
|
|
121
|
+
verify=verify,
|
|
122
|
+
)
|
|
123
|
+
assert (tmp_path / "a.txt").read_bytes() == b"hi"
|
|
124
|
+
by_url = {url: kwargs.get("verify", True) for url, kwargs in calls}
|
|
125
|
+
# the instance's own requests follow verify
|
|
126
|
+
assert by_url[f"{LOCAL}/api/records/{RECORD}/files"] is verify
|
|
127
|
+
assert by_url[f"{LOCAL}/api/records/{RECORD}/files/a.txt/content"] is verify
|
|
128
|
+
# a redirect to another host is always verified
|
|
129
|
+
assert by_url[STORE] is True
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_download_content_verify_false_stays_on_the_first_host(calls, tmp_path):
|
|
133
|
+
download_content(
|
|
134
|
+
f"{LOCAL}/api/records/{RECORD}/files/a.txt/content",
|
|
135
|
+
tmp_path / "a.txt",
|
|
136
|
+
verify=False,
|
|
137
|
+
)
|
|
138
|
+
by_url = {url: kwargs.get("verify", True) for url, kwargs in calls}
|
|
139
|
+
assert by_url[f"{LOCAL}/api/records/{RECORD}/files/a.txt/content"] is False
|
|
140
|
+
assert by_url[STORE] is True
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def test_defaults_are_unchanged(calls):
|
|
144
|
+
get_files_from_record(RECORD, production=False, authors=True, token="t")
|
|
145
|
+
dev = "https://authors.caltechlibrary.dev"
|
|
146
|
+
assert calls[0][0] == f"{dev}/api/records/{RECORD}/files"
|
|
147
|
+
for _, kwargs in calls:
|
|
148
|
+
assert kwargs.get("verify", True) is True
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""caltechdata_edit passes its verify argument to every request it makes.
|
|
2
|
+
|
|
3
|
+
A test instance with a self-signed certificate (local=True) needs verify=False
|
|
4
|
+
on every call, including the ones made by write_files_rdm and add_file_links.
|
|
5
|
+
No network is used: requests is replaced by a router that answers the calls an
|
|
6
|
+
edit of a published record makes, and records the keyword arguments.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from types import SimpleNamespace
|
|
10
|
+
from unittest import mock
|
|
11
|
+
from urllib.parse import urlsplit
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
from caltechdata_api import caltechdata_edit
|
|
17
|
+
|
|
18
|
+
BASE = "https://127.0.0.1:5000"
|
|
19
|
+
RECORD = "abcde-12345"
|
|
20
|
+
NEW = "fghij-67890"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def respond(method, url):
|
|
24
|
+
"""Return (status, payload) for the calls an edit makes."""
|
|
25
|
+
path = urlsplit(url).path
|
|
26
|
+
if method == "GET":
|
|
27
|
+
if path.endswith("/draft/review"):
|
|
28
|
+
accept = "https://data.caltechlibrary.dev/api/requests/r1/actions/accept"
|
|
29
|
+
return 200, {"links": {"actions": {"accept": accept}}}
|
|
30
|
+
if path.endswith("/draft/files"):
|
|
31
|
+
return 200, {"entries": []}
|
|
32
|
+
if path.endswith("/draft"):
|
|
33
|
+
return 200, {}
|
|
34
|
+
return 200, {"status": "published", "files": {"enabled": False}}
|
|
35
|
+
if method == "POST":
|
|
36
|
+
if path.endswith("/communities") or path.endswith("/actions/accept"):
|
|
37
|
+
return 200, {}
|
|
38
|
+
if path.endswith("/versions"):
|
|
39
|
+
return 201, {"id": NEW}
|
|
40
|
+
if path.endswith("/draft/files"):
|
|
41
|
+
link = f"{BASE}/api/records/{NEW}/draft/files/a.txt"
|
|
42
|
+
return 201, {
|
|
43
|
+
"entries": [
|
|
44
|
+
{
|
|
45
|
+
"key": "a.txt",
|
|
46
|
+
"links": {
|
|
47
|
+
"self": link,
|
|
48
|
+
"content": link + "/content",
|
|
49
|
+
"commit": link + "/commit",
|
|
50
|
+
},
|
|
51
|
+
}
|
|
52
|
+
]
|
|
53
|
+
}
|
|
54
|
+
if path.endswith("/commit"):
|
|
55
|
+
return 200, {}
|
|
56
|
+
if path.endswith("/actions/publish"):
|
|
57
|
+
return 202, {"id": NEW, "pids": {"doi": {"identifier": "10.1/x"}}}
|
|
58
|
+
if method == "PUT":
|
|
59
|
+
if path.endswith("/content"):
|
|
60
|
+
return 200, {}
|
|
61
|
+
return 200, {"links": {"files": f"{BASE}/api/records/{NEW}/draft/files"}}
|
|
62
|
+
raise AssertionError(f"unexpected {method} {url}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@pytest.fixture
|
|
66
|
+
def calls():
|
|
67
|
+
"""Replace requests.get/post/put/delete; the list records (method, kwargs)."""
|
|
68
|
+
seen = []
|
|
69
|
+
|
|
70
|
+
def make(method):
|
|
71
|
+
def fake(url, *args, **kwargs):
|
|
72
|
+
seen.append((method, url, kwargs))
|
|
73
|
+
status, payload = respond(method, url)
|
|
74
|
+
return SimpleNamespace(
|
|
75
|
+
status_code=status, json=lambda: payload, text=str(payload), url=url
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
return fake
|
|
79
|
+
|
|
80
|
+
with mock.patch.multiple(
|
|
81
|
+
requests, **{m.lower(): make(m) for m in ("GET", "POST", "PUT", "DELETE")}
|
|
82
|
+
):
|
|
83
|
+
yield seen
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@pytest.fixture
|
|
87
|
+
def s3():
|
|
88
|
+
"""Stand in for s3fs, which add_file_links uses to size each linked file."""
|
|
89
|
+
fs = mock.Mock()
|
|
90
|
+
fs.return_value.info.return_value = {"size": 1}
|
|
91
|
+
with mock.patch("caltechdata_api.caltechdata_write.s3fs.S3FileSystem", fs):
|
|
92
|
+
yield fs
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def edit(**kwargs):
|
|
96
|
+
return caltechdata_edit(
|
|
97
|
+
RECORD, token="t", authors=True, local=True, publish=True, **kwargs
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def assert_all(calls, expected):
|
|
102
|
+
assert calls, "no requests were made"
|
|
103
|
+
for method, url, kwargs in calls:
|
|
104
|
+
assert (
|
|
105
|
+
kwargs.get("verify", True) is expected
|
|
106
|
+
), f"{method} {url} did not get verify={expected}"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
110
|
+
def test_metadata_only_edit(calls, verify):
|
|
111
|
+
edit(verify=verify)
|
|
112
|
+
assert_all(calls, verify)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
116
|
+
def test_new_version(calls, verify):
|
|
117
|
+
edit(new_version=True, verify=verify)
|
|
118
|
+
assert_all(calls, verify)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
122
|
+
def test_file_upload(calls, tmp_path, verify):
|
|
123
|
+
path = tmp_path / "a.txt"
|
|
124
|
+
path.write_text("x")
|
|
125
|
+
edit(files=[str(path)], verify=verify)
|
|
126
|
+
assert_all(calls, verify)
|
|
127
|
+
# the upload and its commit were part of the run
|
|
128
|
+
assert any(m == "PUT" and u.endswith("/content") for m, u, _ in calls)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
132
|
+
def test_file_links(calls, s3, verify):
|
|
133
|
+
edit(file_links=["https://s3.example.org/bucket/a.txt"], verify=verify)
|
|
134
|
+
assert_all(calls, verify)
|
|
135
|
+
assert any(m == "POST" and u.endswith("/draft/files") for m, u, _ in calls)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_verify_defaults_to_true(calls):
|
|
139
|
+
edit()
|
|
140
|
+
assert_all(calls, True)
|
|
141
|
+
assert all(kwargs.get("verify", True) is True for _, _, kwargs in calls)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@pytest.mark.parametrize("verify", [True, False])
|
|
145
|
+
def test_community_accept(calls, verify):
|
|
146
|
+
# adding to a community accepts the request through caltechdata_accept
|
|
147
|
+
edit(community="c-1", verify=verify)
|
|
148
|
+
assert_all(calls, verify)
|
|
149
|
+
assert any(m == "POST" and u.endswith("/actions/accept") for m, u, _ in calls)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/date_types.yaml
RENAMED
|
File without changes
|
{caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/description_types.yaml
RENAMED
|
File without changes
|
{caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/identifier_types.yaml
RENAMED
|
File without changes
|
|
File without changes
|
{caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/relation_types.yaml
RENAMED
|
File without changes
|
{caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/resource_types.yaml
RENAMED
|
File without changes
|
|
File without changes
|
{caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api/vocabularies/title_types.yaml
RENAMED
|
File without changes
|
|
File without changes
|
{caltechdata_api-2.2.3 → caltechdata_api-2.2.4}/caltechdata_api.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|