deriva 1.7.5__py3-none-any.whl → 1.7.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deriva/core/__init__.py +1 -1
- deriva/core/base_cli.py +5 -0
- deriva/core/hatrac_store.py +6 -1
- deriva/transfer/__init__.py +1 -0
- deriva/transfer/download/deriva_download.py +5 -10
- deriva/transfer/download/deriva_export.py +254 -0
- deriva/transfer/download/processors/query/base_query_processor.py +4 -4
- {deriva-1.7.5.dist-info → deriva-1.7.7.dist-info}/METADATA +1 -1
- {deriva-1.7.5.dist-info → deriva-1.7.7.dist-info}/RECORD +13 -12
- {deriva-1.7.5.dist-info → deriva-1.7.7.dist-info}/entry_points.txt +1 -0
- {deriva-1.7.5.dist-info → deriva-1.7.7.dist-info}/LICENSE +0 -0
- {deriva-1.7.5.dist-info → deriva-1.7.7.dist-info}/WHEEL +0 -0
- {deriva-1.7.5.dist-info → deriva-1.7.7.dist-info}/top_level.txt +0 -0
deriva/core/__init__.py
CHANGED
deriva/core/base_cli.py
CHANGED
|
@@ -53,6 +53,11 @@ class BaseCLI(object):
|
|
|
53
53
|
|
|
54
54
|
return args
|
|
55
55
|
|
|
56
|
+
# Function to convert comma-separated CLI input into a tuple
|
|
57
|
+
@staticmethod
|
|
58
|
+
def parse_tuple(value):
|
|
59
|
+
# Split the input by commas and convert to a tuple
|
|
60
|
+
return tuple(map(float, value.split(',')))
|
|
56
61
|
|
|
57
62
|
class KeyValuePairArgs(argparse.Action):
|
|
58
63
|
def __init__(self, option_strings, dest, nargs=None, **kwargs):
|
deriva/core/hatrac_store.py
CHANGED
|
@@ -184,6 +184,7 @@ class HatracStore(DerivaBinding):
|
|
|
184
184
|
|
|
185
185
|
headers = headers.copy()
|
|
186
186
|
|
|
187
|
+
file_opened = False
|
|
187
188
|
if hasattr(data, 'read') and hasattr(data, 'seek'):
|
|
188
189
|
data.seek(0, os.SEEK_END)
|
|
189
190
|
file_size = data.tell()
|
|
@@ -192,6 +193,7 @@ class HatracStore(DerivaBinding):
|
|
|
192
193
|
else:
|
|
193
194
|
file_size = os.path.getsize(data)
|
|
194
195
|
f = open(data, 'rb')
|
|
196
|
+
file_opened = True
|
|
195
197
|
|
|
196
198
|
if not (md5 or sha256):
|
|
197
199
|
md5 = hu.compute_hashes(f, hashes=['md5'])['md5'][1]
|
|
@@ -208,7 +210,8 @@ class HatracStore(DerivaBinding):
|
|
|
208
210
|
if (md5 and r.headers.get('Content-MD5') == md5 or
|
|
209
211
|
sha256 and r.headers.get('Content-SHA256') == sha256):
|
|
210
212
|
# object already has same content so skip upload
|
|
211
|
-
|
|
213
|
+
if file_opened:
|
|
214
|
+
f.close()
|
|
212
215
|
return r.headers.get('Content-Location')
|
|
213
216
|
elif not allow_versioning:
|
|
214
217
|
raise NotModified("The data cannot be uploaded because content already exists for this object "
|
|
@@ -232,6 +235,8 @@ class HatracStore(DerivaBinding):
|
|
|
232
235
|
url = '%s%s' % (url.rstrip("/") if url.endswith("/") else url,
|
|
233
236
|
"" if not parents else "?parents=%s" % str(parents).lower())
|
|
234
237
|
r = self._session.put(url, data=f, headers=headers)
|
|
238
|
+
if file_opened:
|
|
239
|
+
f.close()
|
|
235
240
|
self._response_raise_for_status(r)
|
|
236
241
|
loc = r.text.strip() or r.url
|
|
237
242
|
if loc.startswith(self._server_uri):
|
deriva/transfer/__init__.py
CHANGED
|
@@ -2,6 +2,7 @@ from deriva.transfer.download.deriva_download import DerivaDownload, GenericDown
|
|
|
2
2
|
DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, \
|
|
3
3
|
DerivaDownloadBaggingError
|
|
4
4
|
from deriva.transfer.download.deriva_download_cli import DerivaDownloadCLI
|
|
5
|
+
from deriva.transfer.download.deriva_export import DerivaExport, DerivaExportCLI
|
|
5
6
|
|
|
6
7
|
from deriva.transfer.upload.deriva_upload import DerivaUpload, GenericUploader, DerivaUploadError, DerivaUploadError, \
|
|
7
8
|
DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError, \
|
|
@@ -8,8 +8,8 @@ import requests
|
|
|
8
8
|
from requests.exceptions import HTTPError
|
|
9
9
|
from bdbag import bdbag_api as bdb, bdbag_ro as ro, BAG_PROFILE_TAG, BDBAG_RO_PROFILE_ID
|
|
10
10
|
from bdbag.bdbagit import BagValidationError
|
|
11
|
-
from deriva.core import ErmrestCatalog, HatracStore, format_exception, get_credential,
|
|
12
|
-
|
|
11
|
+
from deriva.core import DerivaServer, ErmrestCatalog, HatracStore, format_exception, get_credential, \
|
|
12
|
+
format_credential, read_config, stob, Megabyte, __version__ as VERSION
|
|
13
13
|
from deriva.core.utils.version_utils import get_installed_version
|
|
14
14
|
from deriva.transfer.download.processors import find_query_processor, find_transform_processor, find_post_processor
|
|
15
15
|
from deriva.transfer.download.processors.base_processor import LOCAL_PATH_KEY, REMOTE_PATHS_KEY, SERVICE_URL_KEY, \
|
|
@@ -76,14 +76,9 @@ class DerivaDownload(object):
|
|
|
76
76
|
password=password)
|
|
77
77
|
|
|
78
78
|
# catalog and file store initialization
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
self.
|
|
82
|
-
protocol, self.hostname, catalog_id, self.credentials, session_config=session_config)
|
|
83
|
-
if self.store:
|
|
84
|
-
del self.store
|
|
85
|
-
self.store = HatracStore(
|
|
86
|
-
protocol, self.hostname, self.credentials, session_config=session_config)
|
|
79
|
+
server = DerivaServer(protocol, self.hostname, credentials=self.credentials, session_config=session_config)
|
|
80
|
+
self.catalog = server.connect_ermrest(catalog_id)
|
|
81
|
+
self.store = HatracStore(protocol, self.hostname, self.credentials, session_config=session_config)
|
|
87
82
|
|
|
88
83
|
# init dcctx cid
|
|
89
84
|
self.set_dcctx_cid(kwargs.get("dcctx_cid", "api/" + self.__class__.__name__))
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import json
|
|
4
|
+
import traceback
|
|
5
|
+
import requests
|
|
6
|
+
import argparse
|
|
7
|
+
import logging
|
|
8
|
+
import certifi
|
|
9
|
+
import datetime
|
|
10
|
+
from collections.abc import Mapping, Iterable
|
|
11
|
+
from requests.exceptions import HTTPError, ConnectionError, Timeout
|
|
12
|
+
from deriva.core.deriva_binding import DerivaClientContext
|
|
13
|
+
from deriva.core.utils.mime_utils import parse_content_disposition
|
|
14
|
+
from deriva.core import BaseCLI, KeyValuePairArgs, get_new_requests_session, get_transfer_summary, get_credential, \
|
|
15
|
+
format_credential, format_exception, urlsplit, DEFAULT_SESSION_CONFIG, DEFAULT_CHUNK_SIZE
|
|
16
|
+
from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
|
|
17
|
+
DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
EXPORT_SERVICE_PATH = "/deriva/export/%s"
|
|
22
|
+
|
|
23
|
+
"""
|
|
24
|
+
Client tool for interacting with DERIVA Export service.
|
|
25
|
+
|
|
26
|
+
:param host (str): The host server for the export operation.
|
|
27
|
+
:param config_file (str): Path to an export configuration file.
|
|
28
|
+
:param credential (dict): Authentication credential (returned from get_credential()) for the export process. Optional.
|
|
29
|
+
:param envars (dict): A dictionary of variables used for template substitution. Optional.
|
|
30
|
+
:param output_dir (str): The directory where exported data will be stored (default: "."). Optional.
|
|
31
|
+
:param defer_download (bool): Whether to defer the actual data download. Optional.
|
|
32
|
+
:param timeout (tuple) OR (float): Timeout value as a tuple of floats in seconds for (connect,read) export operations.
|
|
33
|
+
If a single float value is passed, it will apply to both connect and read operations. Optional.
|
|
34
|
+
:param export_type (str): The type of export to perform (default: "bdbag"). Optional.
|
|
35
|
+
|
|
36
|
+
:return: The full path to the downloaded file. If "defer_download" is True, the URL(s) where the export can be downloaded.
|
|
37
|
+
"""
|
|
38
|
+
class DerivaExport:
|
|
39
|
+
def __init__(self, **kwargs):
|
|
40
|
+
self.host = kwargs.get("host")
|
|
41
|
+
self.config_file = kwargs.get("config_file")
|
|
42
|
+
self.envars = kwargs.get("envars", dict())
|
|
43
|
+
self.credential = kwargs.get("credential")
|
|
44
|
+
self.output_dir = kwargs.get("output_dir", ".")
|
|
45
|
+
self.defer_download = kwargs.get("defer_download")
|
|
46
|
+
self.timeout = kwargs.get("timeout")
|
|
47
|
+
self.export_type = kwargs.get("export_type", "bdbag")
|
|
48
|
+
self.base_server_uri = "https://" + self.host
|
|
49
|
+
self.service_url = self.base_server_uri + EXPORT_SERVICE_PATH % self.export_type
|
|
50
|
+
self.session_config = DEFAULT_SESSION_CONFIG.copy()
|
|
51
|
+
if isinstance(self.timeout, tuple):
|
|
52
|
+
if len(self.timeout) == 2:
|
|
53
|
+
self.session_config["timeout"] = self.timeout
|
|
54
|
+
else:
|
|
55
|
+
self.session_config["timeout"] = float(self.timeout[0])
|
|
56
|
+
elif self.timeout is not None:
|
|
57
|
+
try:
|
|
58
|
+
self.session_config["timeout"] = float(self.timeout)
|
|
59
|
+
except ValueError:
|
|
60
|
+
logger.warning("Unparseable timeout value: %r. Defaults will be used: %r." %
|
|
61
|
+
(self.timeout, self.session_config["timeout"]))
|
|
62
|
+
self.session = get_new_requests_session(self.service_url, self.session_config)
|
|
63
|
+
self.dcctx = DerivaClientContext()
|
|
64
|
+
self.dcctx['cid'] = kwargs.get("dcctx_cid", "api/" + self.__class__.__name__)
|
|
65
|
+
self.session.headers.update({'deriva-client-context': self.dcctx.encoded()})
|
|
66
|
+
|
|
67
|
+
# credential initialization
|
|
68
|
+
if self.credential is None:
|
|
69
|
+
token = kwargs.get("token")
|
|
70
|
+
oauth2_token = kwargs.get("oauth2_token")
|
|
71
|
+
credential_file = kwargs.get("credential_file")
|
|
72
|
+
if token or oauth2_token:
|
|
73
|
+
self.credential = format_credential(token=token, oauth2_token=oauth2_token)
|
|
74
|
+
else:
|
|
75
|
+
self.credential = get_credential(self.host, credential_file)
|
|
76
|
+
|
|
77
|
+
if self.credential is None:
|
|
78
|
+
raise DerivaDownloadAuthenticationError(
|
|
79
|
+
"The requested service requires authentication and a valid login credential could "
|
|
80
|
+
"not be found (or was not provided) for the specified host.")
|
|
81
|
+
if 'bearer-token' in self.credential:
|
|
82
|
+
self.session.headers.update(
|
|
83
|
+
{'Authorization': 'Bearer {token}'.format(token=self.credential['bearer-token'])})
|
|
84
|
+
elif 'cookie' in self.credential:
|
|
85
|
+
cname, cval = self.credential['cookie'].split('=', 1)
|
|
86
|
+
self.session.cookies.set(cname, cval, domain=self.host, path='/')
|
|
87
|
+
|
|
88
|
+
def validate_authn_session(self):
|
|
89
|
+
url = self.base_server_uri + "/authn/session"
|
|
90
|
+
r = self.session.get(url)
|
|
91
|
+
if r.status_code == requests.codes.not_found or r.status_code == requests.codes.unauthorized:
|
|
92
|
+
logger.warning("Unable to authenticate. Check for missing or expired credentials.")
|
|
93
|
+
r.raise_for_status()
|
|
94
|
+
return r.json()
|
|
95
|
+
|
|
96
|
+
def recursive_format(self, d, **kwargs):
|
|
97
|
+
"""
|
|
98
|
+
Recursively apply str.format to all string-based values in a dictionary.
|
|
99
|
+
Supports nested dictionaries and lists.
|
|
100
|
+
|
|
101
|
+
:param d: Dictionary or iterable containing values to be formatted
|
|
102
|
+
:param kwargs: Formatting arguments
|
|
103
|
+
:return: New dictionary or iterable with formatted strings
|
|
104
|
+
"""
|
|
105
|
+
if isinstance(d, Mapping):
|
|
106
|
+
return {k: self.recursive_format(v, **kwargs) for k, v in d.items()}
|
|
107
|
+
elif isinstance(d, str):
|
|
108
|
+
return d.format(**kwargs)
|
|
109
|
+
elif isinstance(d, Iterable) and not isinstance(d, (str, bytes)):
|
|
110
|
+
return type(d)(self.recursive_format(v, **kwargs) for v in d)
|
|
111
|
+
else:
|
|
112
|
+
return d
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def retrieve_file(self, url):
|
|
116
|
+
content_disposition = None
|
|
117
|
+
try:
|
|
118
|
+
head = self.session.head(url)
|
|
119
|
+
if head.ok:
|
|
120
|
+
content_disposition = head.headers.get("Content-Disposition") if head.ok else None
|
|
121
|
+
if not content_disposition:
|
|
122
|
+
raise DerivaDownloadError("HEAD response missing Content-Disposition header.")
|
|
123
|
+
except requests.HTTPError as e:
|
|
124
|
+
raise DerivaDownloadError("HEAD request for [%s] failed: %s" % (url, e))
|
|
125
|
+
|
|
126
|
+
filename = parse_content_disposition(content_disposition)
|
|
127
|
+
output_path = os.path.abspath(os.path.join(self.output_dir, filename))
|
|
128
|
+
with self.session.get(url, stream=True, verify=certifi.where()) as r:
|
|
129
|
+
if r.status_code != 200:
|
|
130
|
+
file_error = "File [%s] transfer failed." % output_path
|
|
131
|
+
url_error = 'HTTP GET Failed for url: %s' % url
|
|
132
|
+
host_error = "Host %s responded:\n\n%s" % (urlsplit(url).netloc, r.text)
|
|
133
|
+
raise DerivaDownloadError('%s\n\n%s\n%s' % (file_error, url_error, host_error))
|
|
134
|
+
else:
|
|
135
|
+
total = 0
|
|
136
|
+
start = datetime.datetime.now()
|
|
137
|
+
logging.debug("Transferring file %s to %s" % (url, output_path))
|
|
138
|
+
with open(output_path, 'wb') as data_file:
|
|
139
|
+
for chunk in r.iter_content(chunk_size=DEFAULT_CHUNK_SIZE):
|
|
140
|
+
data_file.write(chunk)
|
|
141
|
+
total += len(chunk)
|
|
142
|
+
elapsed = datetime.datetime.now() - start
|
|
143
|
+
summary = get_transfer_summary(total, elapsed)
|
|
144
|
+
logging.info("File [%s] transfer successful. %s" % (output_path, summary))
|
|
145
|
+
return output_path
|
|
146
|
+
|
|
147
|
+
def export(self):
|
|
148
|
+
try:
|
|
149
|
+
auth = self.validate_authn_session()
|
|
150
|
+
logger.debug("Authenticated session established. Session attributes: %s" % auth)
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
logger.info("Processing export config file: %s" % self.config_file)
|
|
154
|
+
with open(self.config_file, encoding='utf-8') as cf:
|
|
155
|
+
config = json.loads(cf.read())
|
|
156
|
+
env = config.get("env", {})
|
|
157
|
+
env = self.recursive_format(env, **self.envars)
|
|
158
|
+
config.update({"env": env})
|
|
159
|
+
except Exception as e:
|
|
160
|
+
raise DerivaDownloadConfigurationError("Error processing export config file: %s" % format_exception(e))
|
|
161
|
+
|
|
162
|
+
logger.info("Requesting %s export at: %s" % (self.export_type, self.service_url))
|
|
163
|
+
response = self.session.post(self.service_url, json=config)
|
|
164
|
+
response.raise_for_status()
|
|
165
|
+
result_urls = response.text.split('\n')
|
|
166
|
+
logger.info("Export successful. Service responded with URL list: %s" % result_urls)
|
|
167
|
+
if not self.defer_download:
|
|
168
|
+
if self.export_type == "bdbag":
|
|
169
|
+
result_url = result_urls[1] if len(result_urls) > 1 else result_urls[0]
|
|
170
|
+
logger.info("Downloading exported bag content from %s to directory: %s" %
|
|
171
|
+
(result_url, os.path.abspath(self.output_dir)))
|
|
172
|
+
return self.retrieve_file(result_url)
|
|
173
|
+
elif self.export_type == "file":
|
|
174
|
+
for result_url in result_urls:
|
|
175
|
+
self.retrieve_file(result_url)
|
|
176
|
+
logger.info("Downloading exported file content from %s to directory: %s" %
|
|
177
|
+
(result_url, os.path.abspath(self.output_dir)))
|
|
178
|
+
else:
|
|
179
|
+
pass
|
|
180
|
+
else:
|
|
181
|
+
return result_urls
|
|
182
|
+
except ConnectionError as e:
|
|
183
|
+
raise DerivaDownloadError("Connection error occurred. %s" % format_exception(e))
|
|
184
|
+
except Timeout as e:
|
|
185
|
+
raise DerivaDownloadTimeoutError("Connection timeout occurred. %s" % format_exception(e))
|
|
186
|
+
except HTTPError as e:
|
|
187
|
+
if e.response.status_code == requests.codes.unauthorized:
|
|
188
|
+
raise DerivaDownloadAuthenticationError(
|
|
189
|
+
"The requested service requires authentication and a valid login session could "
|
|
190
|
+
"not be found for the specified host. Server responded: %s" % format_exception(e))
|
|
191
|
+
elif e.response.status_code == requests.codes.forbidden:
|
|
192
|
+
raise DerivaDownloadAuthorizationError(
|
|
193
|
+
"A requested operation was forbidden. Server responded: %s" % format_exception(e))
|
|
194
|
+
else:
|
|
195
|
+
raise DerivaDownloadError(format_exception(e))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class DerivaExportCLI(BaseCLI):
|
|
199
|
+
def __init__(self, description, epilog, **kwargs):
|
|
200
|
+
|
|
201
|
+
BaseCLI.__init__(self, description, epilog, **kwargs)
|
|
202
|
+
self.parser.add_argument("--defer-download", action="store_true",
|
|
203
|
+
help="Do not download exported file(s). Default: False")
|
|
204
|
+
self.parser.add_argument("--timeout", metavar="<connect,read>", type=BaseCLI.parse_tuple,
|
|
205
|
+
help="Timeout value(s) in seconds (int or float) for connect and read operations. "
|
|
206
|
+
"Separate using commas. If a single value is provided it will be used for both "
|
|
207
|
+
"connect and read timeouts.")
|
|
208
|
+
self.parser.add_argument("--export-type", choices=["bdbag", "file"], default="bdbag",
|
|
209
|
+
help="Export type: {bdbag|file}. Default is bdbag.",)
|
|
210
|
+
self.parser.add_argument("--output-dir", metavar="<output dir>", default=".",
|
|
211
|
+
help="Path to an output directory. Default is current directory.")
|
|
212
|
+
self.parser.add_argument("envars", metavar="[key=value key=value ...]",
|
|
213
|
+
nargs=argparse.REMAINDER, action=KeyValuePairArgs, default={},
|
|
214
|
+
help="Variable length of whitespace-delimited key=value pair arguments used for "
|
|
215
|
+
"string interpolation in specific parts of the configuration file. "
|
|
216
|
+
"For example: key1=value1 key2=value2")
|
|
217
|
+
|
|
218
|
+
def main(self):
|
|
219
|
+
try:
|
|
220
|
+
args = self.parse_cli()
|
|
221
|
+
except ValueError as e:
|
|
222
|
+
sys.stderr.write(str(e))
|
|
223
|
+
return 2
|
|
224
|
+
if not args.quiet:
|
|
225
|
+
sys.stderr.write("\n")
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
exporter = DerivaExport(**vars(args), dcctx_cid="cli/" + self.__class__.__name__)
|
|
229
|
+
exporter.export()
|
|
230
|
+
except (DerivaDownloadError, DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError,
|
|
231
|
+
DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError) as e:
|
|
232
|
+
sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e))
|
|
233
|
+
if args.debug:
|
|
234
|
+
traceback.print_exc()
|
|
235
|
+
return 1
|
|
236
|
+
except:
|
|
237
|
+
sys.stderr.write("An unexpected error occurred.")
|
|
238
|
+
traceback.print_exc()
|
|
239
|
+
return 1
|
|
240
|
+
finally:
|
|
241
|
+
if not args.quiet:
|
|
242
|
+
sys.stderr.write("\n\n")
|
|
243
|
+
return 0
|
|
244
|
+
|
|
245
|
+
DESC = "Deriva Export Service Download Utility - CLI"
|
|
246
|
+
INFO = "For more information see: https://github.com/informatics-isi-edu/deriva-py"
|
|
247
|
+
|
|
248
|
+
def main():
|
|
249
|
+
cli = DerivaExportCLI(DESC, INFO, hostname_required=True, config_file_required=True)
|
|
250
|
+
return cli.main()
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
if __name__ == '__main__':
|
|
254
|
+
sys.exit(main())
|
|
@@ -2,7 +2,7 @@ import os
|
|
|
2
2
|
import errno
|
|
3
3
|
import certifi
|
|
4
4
|
import requests
|
|
5
|
-
from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, DEFAULT_SESSION_CONFIG
|
|
5
|
+
from deriva.core import urlsplit, get_new_requests_session, stob, make_dirs, format_exception, DEFAULT_SESSION_CONFIG
|
|
6
6
|
from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
|
|
7
7
|
DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError
|
|
8
8
|
from deriva.transfer.download.processors.base_processor import BaseProcessor, \
|
|
@@ -83,12 +83,12 @@ class BaseQueryProcessor(BaseProcessor):
|
|
|
83
83
|
return self.catalog.get(self.query, headers=headers).json()
|
|
84
84
|
except requests.HTTPError as e:
|
|
85
85
|
if e.response.status_code == 401:
|
|
86
|
-
raise DerivaDownloadAuthenticationError(e)
|
|
86
|
+
raise DerivaDownloadAuthenticationError(format_exception(e))
|
|
87
87
|
if e.response.status_code == 403:
|
|
88
|
-
raise DerivaDownloadAuthorizationError(e)
|
|
88
|
+
raise DerivaDownloadAuthorizationError(format_exception(e))
|
|
89
89
|
if as_file:
|
|
90
90
|
os.remove(self.output_abspath)
|
|
91
|
-
raise DerivaDownloadError("Error executing catalog query: %s" % e)
|
|
91
|
+
raise DerivaDownloadError("Error executing catalog query: %s" % format_exception(e))
|
|
92
92
|
except Exception:
|
|
93
93
|
if as_file:
|
|
94
94
|
os.remove(self.output_abspath)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: deriva
|
|
3
|
-
Version: 1.7.
|
|
3
|
+
Version: 1.7.7
|
|
4
4
|
Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
|
|
5
5
|
Home-page: https://github.com/informatics-isi-edu/deriva-py
|
|
6
6
|
Author: USC Information Sciences Institute, Informatics Systems Research Division
|
|
@@ -8,9 +8,9 @@ deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIe
|
|
|
8
8
|
deriva/config/rollback_annotation.py,sha256=vqrIcen-KZX8LDpu2OVNivzIHpQoQgWkZAChZJctvtk,3015
|
|
9
9
|
deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
|
|
10
10
|
deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
|
|
11
|
-
deriva/core/__init__.py,sha256=
|
|
11
|
+
deriva/core/__init__.py,sha256=D7aya8r3BTbewaSUmhbb4DB-Ltwe2aXV10CMfPo13rI,4945
|
|
12
12
|
deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
|
|
13
|
-
deriva/core/base_cli.py,sha256=
|
|
13
|
+
deriva/core/base_cli.py,sha256=78Ilf3_f2xREQb3IIj6q0jwWAiXSObZszG0JURs36lA,2902
|
|
14
14
|
deriva/core/catalog_cli.py,sha256=-6Bo6GLWFWap7y3VxkzPs73HAe_XzRXIJMW-Ri84m3M,23273
|
|
15
15
|
deriva/core/datapath.py,sha256=w8LvPAd_DuknKxHc_YS6NUHhTY6XpCSkfa0m_xQUdZE,89068
|
|
16
16
|
deriva/core/deriva_binding.py,sha256=_sA9HGrcVRqT-OhrneMDMOquyVOFOxLq3WzBQhasLIM,12970
|
|
@@ -18,7 +18,7 @@ deriva/core/deriva_server.py,sha256=nsW3gwg1sIaHl3BTf-nL41AkSj3dEpcEBlatvjvN8CQ,
|
|
|
18
18
|
deriva/core/ermrest_catalog.py,sha256=_eqQg16i1aA95R99B7tLZxHWQlYk-rLpN_0zghfNWRc,54991
|
|
19
19
|
deriva/core/ermrest_model.py,sha256=NdvrMLkmNTY4ncodzojpyV1XjgSlod7x-SXyUu65Z0Q,124921
|
|
20
20
|
deriva/core/hatrac_cli.py,sha256=l9QmneLRHSMiG_z9S83ea0QGVhTS3Wq1KGPEKEDpecM,14522
|
|
21
|
-
deriva/core/hatrac_store.py,sha256=
|
|
21
|
+
deriva/core/hatrac_store.py,sha256=NcVuO4h4hswbEAct8tTKZ1pNtXBn74Nn9TelKN_jr8Q,22323
|
|
22
22
|
deriva/core/mmo.py,sha256=dcB8akgsqbYMi22ClbVpOKVL6so8FjSDjSb6gP4_jFo,17852
|
|
23
23
|
deriva/core/polling_ermrest_catalog.py,sha256=KsjiFqPQaHWnJZCVF5i77sdzfubqZHgMBbQ1p8V8D3s,10351
|
|
24
24
|
deriva/core/schemas/app_links.schema.json,sha256=AxrkC2scxomM6N7jyjtdYA73BbZzPrmuqU8PYWe7okI,954
|
|
@@ -53,15 +53,16 @@ deriva/core/utils/webauthn_utils.py,sha256=rD0HQZAjUKp4NfqHQG1FhH3x7uKog2et7w7LB
|
|
|
53
53
|
deriva/seo/__init__.py,sha256=dYn48A7blbeYf40b4T3KVofrQK4u5K5MfxXWfIGloig,54
|
|
54
54
|
deriva/seo/sitemap_builder.py,sha256=Ht_AbodEERDofIoCcd4kPlrl1pVW670WN5dT4cc05LQ,13948
|
|
55
55
|
deriva/seo/sitemap_cli.py,sha256=miCqRfpSj5Dx5BfJGSd8Pi2e4OOQjotDzP_JubukhCM,2654
|
|
56
|
-
deriva/transfer/__init__.py,sha256=
|
|
56
|
+
deriva/transfer/__init__.py,sha256=3a01U6e68kBEnFUp0dRMFLRDj2p50iIJF9RZaG3TXVI,1253
|
|
57
57
|
deriva/transfer/backup/__init__.py,sha256=vxsZiDLMTJQPybXT89G-07GsUoLhnItTCbLdXcDSyeA,465
|
|
58
58
|
deriva/transfer/backup/__main__.py,sha256=dT12--8C6sKGEtMhsYuy013ebXKpVnBJfhcQNlVtv6Y,361
|
|
59
59
|
deriva/transfer/backup/deriva_backup.py,sha256=IO9Tmzx6jHfUCkP-41nSsAeOFLn9T-0HwQcpRLpM_zs,5228
|
|
60
60
|
deriva/transfer/backup/deriva_backup_cli.py,sha256=T0tvPKWniRinMQt0qG7FI8AoK3GgtlT6EyBZmZCAjL8,2157
|
|
61
61
|
deriva/transfer/download/__init__.py,sha256=Pr7Zud4AFsIWwopTxeC_pupslgCG_lzycO9w9Xyh88Q,350
|
|
62
62
|
deriva/transfer/download/__main__.py,sha256=YUg7AZ07t_xaOgtfJnU_l1nkEHCCPR8sU5X-l1An6SY,363
|
|
63
|
-
deriva/transfer/download/deriva_download.py,sha256=
|
|
63
|
+
deriva/transfer/download/deriva_download.py,sha256=ulFrHHEDj3oJA2pAo7MGvGyDF9rA3l8yCUoK3FMvHEk,17100
|
|
64
64
|
deriva/transfer/download/deriva_download_cli.py,sha256=wN8tyQDv1AIE_aDqjECbmkoEWN050vlEdJyteYbdgSs,3940
|
|
65
|
+
deriva/transfer/download/deriva_export.py,sha256=R6f08eJSt1iBeDNNQhThN3W1l4UwE0xzy2DjXbqNskE,13085
|
|
65
66
|
deriva/transfer/download/processors/__init__.py,sha256=evLp36tZn-Z_AMshdfV3JJO8w1es5owsnRN0IFJUwIo,4507
|
|
66
67
|
deriva/transfer/download/processors/base_processor.py,sha256=R6IIHSa_euv4X2Dyhd8fvQAiVYDGJTWMQtPoukHQn-Q,3837
|
|
67
68
|
deriva/transfer/download/processors/postprocess/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -70,7 +71,7 @@ deriva/transfer/download/processors/postprocess/transfer_post_processor.py,sha25
|
|
|
70
71
|
deriva/transfer/download/processors/postprocess/url_post_processor.py,sha256=s68iIYqQSZHtbv4y-fCG8pjhApAeMEG6hYcKx2Pvf5Y,2745
|
|
71
72
|
deriva/transfer/download/processors/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
72
73
|
deriva/transfer/download/processors/query/bag_fetch_query_processor.py,sha256=tiQtfuy01YgOFFD5b_sP7TGjMnt0Jqcg2gp1KNWqeLE,5645
|
|
73
|
-
deriva/transfer/download/processors/query/base_query_processor.py,sha256=
|
|
74
|
+
deriva/transfer/download/processors/query/base_query_processor.py,sha256=0tGxTJKCEqm7ecdOcfd8szA94muc2jbYwbqYvTGwAac,10465
|
|
74
75
|
deriva/transfer/download/processors/query/file_download_query_processor.py,sha256=Hg1NbKsaGJh9cB86yIyL7Fm7ywSNVop837Dv8aFXUes,7257
|
|
75
76
|
deriva/transfer/download/processors/transform/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
76
77
|
deriva/transfer/download/processors/transform/base_transform_processor.py,sha256=Ddw5gsNpDANeuLvUaF4utp8psaxOtAzlgXtOg8gb-Pc,4109
|
|
@@ -107,9 +108,9 @@ tests/deriva/core/mmo/test_mmo_find.py,sha256=PcUN76sik68B3XKg0G3wHVpKcPEld_6Rtb
|
|
|
107
108
|
tests/deriva/core/mmo/test_mmo_prune.py,sha256=4pYtYL8g1BgadlewNPVpVA5lT_gV6SPTDYf04ZKzBTA,6851
|
|
108
109
|
tests/deriva/core/mmo/test_mmo_rename.py,sha256=4oSR1G3Od701Ss3AnolI1Z7CbMxKuQF2uSr2_IcoR6s,8512
|
|
109
110
|
tests/deriva/core/mmo/test_mmo_replace.py,sha256=w-66LWyiQ_ajC7Ipmhc4kAKwIloPdQELeUPsvelTdX8,8439
|
|
110
|
-
deriva-1.7.
|
|
111
|
-
deriva-1.7.
|
|
112
|
-
deriva-1.7.
|
|
113
|
-
deriva-1.7.
|
|
114
|
-
deriva-1.7.
|
|
115
|
-
deriva-1.7.
|
|
111
|
+
deriva-1.7.7.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
|
|
112
|
+
deriva-1.7.7.dist-info/METADATA,sha256=V1dOlM-ESgLHfPMX_EOZ-kI6jvouKBIIzBcwwZ_bnnM,1623
|
|
113
|
+
deriva-1.7.7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
114
|
+
deriva-1.7.7.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
|
|
115
|
+
deriva-1.7.7.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
|
|
116
|
+
deriva-1.7.7.dist-info/RECORD,,
|
|
@@ -7,6 +7,7 @@ deriva-annotation-validate = deriva.config.annotation_validate:main
|
|
|
7
7
|
deriva-backup-cli = deriva.transfer.backup.__main__:main
|
|
8
8
|
deriva-catalog-cli = deriva.core.catalog_cli:main
|
|
9
9
|
deriva-download-cli = deriva.transfer.download.__main__:main
|
|
10
|
+
deriva-export-cli = deriva.transfer.download.deriva_export:main
|
|
10
11
|
deriva-globus-auth-utils = deriva.core.utils.globus_auth_utils:main
|
|
11
12
|
deriva-hatrac-cli = deriva.core.hatrac_cli:main
|
|
12
13
|
deriva-restore-cli = deriva.transfer.restore.__main__:main
|
|
File without changes
|
|
File without changes
|
|
File without changes
|