deriva 1.7.8__tar.gz → 1.7.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deriva-1.7.8/deriva.egg-info → deriva-1.7.9}/PKG-INFO +17 -4
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/acl_config.py +2 -1
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/__init__.py +2 -2
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/catalog_cli.py +12 -6
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/deriva_binding.py +7 -7
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/ermrest_catalog.py +20 -6
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/ermrest_model.py +1 -1
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/utils/core_utils.py +11 -6
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/utils/globus_auth_utils.py +41 -4
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/deriva_export.py +11 -3
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/query/base_query_processor.py +12 -12
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/query/file_download_query_processor.py +2 -2
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/deriva_upload.py +19 -15
- {deriva-1.7.8 → deriva-1.7.9/deriva.egg-info}/PKG-INFO +17 -4
- {deriva-1.7.8 → deriva-1.7.9}/deriva.egg-info/requires.txt +2 -2
- {deriva-1.7.8 → deriva-1.7.9}/setup.py +2 -2
- {deriva-1.7.8 → deriva-1.7.9}/.gitignore +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/CHANGELOG.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/LICENSE +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/README.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/annotation_config.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/annotation_validate.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/base_config.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/dump_catalog_annotations.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/examples/group_owner_policy.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/examples/self_serve_policy.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/config/rollback_annotation.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/annotation.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/base_cli.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/datapath.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/deriva_server.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/hatrac_cli.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/hatrac_store.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/mmo.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/polling_ermrest_catalog.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/app_links.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/asset.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/bulk_upload.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/chaise_config.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/citation.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/column_display.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/display.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/export.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/export_2019.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/foreign_key.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/generated.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/immutable.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/indexing_preferences.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/key_display.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/non_deletable.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/required.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/source_definitions.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/table_alternatives.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/table_display.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/visible_columns.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/schemas/visible_foreign_keys.schema.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/tests/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/utils/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/utils/hash_utils.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/utils/mime_utils.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/utils/version_utils.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/core/utils/webauthn_utils.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/seo/README.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/seo/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/seo/sitemap_builder.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/seo/sitemap_cli.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/backup/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/backup/__main__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/backup/deriva_backup.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/backup/deriva_backup_cli.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/__main__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/deriva_download.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/deriva_download_cli.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/base_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/postprocess/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/postprocess/identifier_post_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/postprocess/transfer_post_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/postprocess/url_post_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/query/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/query/bag_fetch_query_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/transform/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/transform/base_transform_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/transform/column_transform_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/transform/fasta_transform_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/transform/format_transform_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/transform/geo_transform_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/transform/string_transform_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test1.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test10.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test11.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test12.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test13.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test14.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test15.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test16.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test19.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test2.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test20.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test3.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test4.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test5.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test6.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test7.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test8.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/tests/test9.json +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/restore/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/restore/__main__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/restore/deriva_restore.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/restore/deriva_restore_cli.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/__main__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/deriva_upload_cli.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/processors/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/processors/archive_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/processors/base_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/processors/logging_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/processors/metadata_update_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/processors/rename_processor.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/upload/tests/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva/utils/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva.egg-info/SOURCES.txt +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva.egg-info/dependency_links.txt +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva.egg-info/entry_points.txt +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/deriva.egg-info/top_level.txt +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/BUILD.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/Makefile +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/README.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/_static/README.txt +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.config.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.core.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.core.utils.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.seo.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.backup.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.download.processors.postprocess.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.download.processors.query.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.download.processors.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.download.processors.transform.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.download.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.restore.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/api/deriva.transfer.upload.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/commands.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-acl-config.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-annotation-config.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-annotation-validate.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-backup-cli.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-download-cli.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-hatrac-cli.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-restore-cli.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/cli/deriva-sitemap-cli.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/conf.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/derivapy-catalog-snapshot.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/derivapy-catalog.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/derivapy-datapath-example-1.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/derivapy-datapath-example-2.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/derivapy-datapath-example-3.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/derivapy-datapath-example-4.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/derivapy-datapath-update.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/get-started.ipynb +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/index.rst +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/install.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/make.bat +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/project-tutorial.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/docs/using-r.md +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/requirements_dev.txt +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/setup.cfg +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/mmo/__init__.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/mmo/base.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/mmo/test_mmo_drop.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/mmo/test_mmo_find.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/mmo/test_mmo_prune.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/mmo/test_mmo_rename.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/mmo/test_mmo_replace.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/test_datapath.py +0 -0
- {deriva-1.7.8 → deriva-1.7.9}/tests/deriva/core/test_ermrest_model.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: deriva
|
|
3
|
-
Version: 1.7.
|
|
3
|
+
Version: 1.7.9
|
|
4
4
|
Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
|
|
5
5
|
Home-page: https://github.com/informatics-isi-edu/deriva-py
|
|
6
6
|
Author: USC Information Sciences Institute, Informatics Systems Research Division
|
|
@@ -26,12 +26,25 @@ License-File: LICENSE
|
|
|
26
26
|
Requires-Dist: packaging
|
|
27
27
|
Requires-Dist: requests
|
|
28
28
|
Requires-Dist: pika
|
|
29
|
-
Requires-Dist: urllib3<3,>=1.26
|
|
29
|
+
Requires-Dist: urllib3<3,>=1.26.20
|
|
30
30
|
Requires-Dist: portalocker>=1.2.1
|
|
31
|
-
Requires-Dist: bdbag>=1.7.
|
|
31
|
+
Requires-Dist: bdbag>=1.7.5
|
|
32
32
|
Requires-Dist: globus_sdk<4,>=3
|
|
33
33
|
Requires-Dist: fair-research-login>=0.3.1
|
|
34
34
|
Requires-Dist: fair-identifiers-client>=0.5.1
|
|
35
35
|
Requires-Dist: jsonschema>=3.1
|
|
36
|
+
Dynamic: author
|
|
37
|
+
Dynamic: author-email
|
|
38
|
+
Dynamic: classifier
|
|
39
|
+
Dynamic: description
|
|
40
|
+
Dynamic: description-content-type
|
|
41
|
+
Dynamic: home-page
|
|
42
|
+
Dynamic: license
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
Dynamic: maintainer
|
|
45
|
+
Dynamic: maintainer-email
|
|
46
|
+
Dynamic: requires-dist
|
|
47
|
+
Dynamic: requires-python
|
|
48
|
+
Dynamic: summary
|
|
36
49
|
|
|
37
50
|
For further information, visit the project [homepage](https://github.com/informatics-isi-edu/deriva-py).
|
|
@@ -291,7 +291,8 @@ class AclConfig:
|
|
|
291
291
|
elif group.startswith(self.ROBOT_PREFIX_FORMAT.format(server=self.server)):
|
|
292
292
|
self.validate_webauthn_robot(group)
|
|
293
293
|
else:
|
|
294
|
-
|
|
294
|
+
if self.verbose:
|
|
295
|
+
warnings.warn("Can't determine format of group '{g}'".format(g=group))
|
|
295
296
|
|
|
296
297
|
def validate_globus_group(self, group):
|
|
297
298
|
guid = group[len(self.GLOBUS_PREFIX):]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
__version__ = "1.7.
|
|
1
|
+
__version__ = "1.7.9"
|
|
2
2
|
|
|
3
3
|
from deriva.core.utils.core_utils import *
|
|
4
4
|
from deriva.core.base_cli import BaseCLI, KeyValuePairArgs
|
|
@@ -55,7 +55,7 @@ def get_credential(host,
|
|
|
55
55
|
scope_map = globus_client.hosts_to_scope_map(hosts=[host], match_scope_tag=match_scope_tag,
|
|
56
56
|
force_refresh=force_scope_lookup,
|
|
57
57
|
warn_on_discovery_failure=True if not creds else False)
|
|
58
|
-
tokens = globus_client.is_logged_in(exclude_defaults=True)
|
|
58
|
+
tokens = globus_client.is_logged_in(exclude_defaults=True, hosts_to_scope_map=scope_map)
|
|
59
59
|
if tokens:
|
|
60
60
|
# 1. look for the explicitly requested scope in the token store, if specified
|
|
61
61
|
token = globus_client.find_access_token_for_scope(requested_scope, tokens)
|
|
@@ -6,7 +6,7 @@ import traceback
|
|
|
6
6
|
from pprint import pp
|
|
7
7
|
from requests.exceptions import HTTPError, ConnectionError
|
|
8
8
|
from deriva.core import __version__ as VERSION, BaseCLI, KeyValuePairArgs, DerivaServer, DerivaPathError, \
|
|
9
|
-
get_credential, format_credential, format_exception, DEFAULT_HEADERS
|
|
9
|
+
get_credential, format_credential, format_exception, read_config, DEFAULT_SESSION_CONFIG, DEFAULT_HEADERS
|
|
10
10
|
from deriva.core.ermrest_model import nochange
|
|
11
11
|
from deriva.core.utils import eprint
|
|
12
12
|
|
|
@@ -191,6 +191,11 @@ class DerivaCatalogCLI (BaseCLI):
|
|
|
191
191
|
else:
|
|
192
192
|
return get_credential(host_name)
|
|
193
193
|
|
|
194
|
+
@staticmethod
|
|
195
|
+
def _get_session_config():
|
|
196
|
+
config = read_config()
|
|
197
|
+
return config.get("session", DEFAULT_SESSION_CONFIG)
|
|
198
|
+
|
|
194
199
|
def _post_parser_init(self, args):
|
|
195
200
|
"""Shared initialization for all sub-commands.
|
|
196
201
|
"""
|
|
@@ -202,7 +207,8 @@ class DerivaCatalogCLI (BaseCLI):
|
|
|
202
207
|
credentials=DerivaCatalogCLI._get_credential(
|
|
203
208
|
self.host,
|
|
204
209
|
token=args.token,
|
|
205
|
-
oauth2_token=args.oauth2_token)
|
|
210
|
+
oauth2_token=args.oauth2_token),
|
|
211
|
+
session_config=DerivaCatalogCLI._get_session_config())
|
|
206
212
|
|
|
207
213
|
@staticmethod
|
|
208
214
|
def _decorate_headers(headers, file_format, method="get"):
|
|
@@ -279,10 +285,10 @@ class DerivaCatalogCLI (BaseCLI):
|
|
|
279
285
|
catalog = self.server.connect_ermrest(args.id)
|
|
280
286
|
try:
|
|
281
287
|
if args.output_file:
|
|
282
|
-
catalog.
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
288
|
+
catalog.get_as_file(args.path,
|
|
289
|
+
destfilename=args.output_file,
|
|
290
|
+
headers=headers,
|
|
291
|
+
delete_if_empty=args.auto_delete)
|
|
286
292
|
else:
|
|
287
293
|
pp(catalog.get(args.path, headers=headers).json())
|
|
288
294
|
except HTTPError as e:
|
|
@@ -139,9 +139,10 @@ class DerivaBinding (object):
|
|
|
139
139
|
server
|
|
140
140
|
)
|
|
141
141
|
self._server_uri = self._base_server_uri
|
|
142
|
+
self._auth_uri = self._base_server_uri + "/authn/session"
|
|
142
143
|
|
|
143
|
-
self.session_config = DEFAULT_SESSION_CONFIG if not session_config else session_config
|
|
144
144
|
self._session = None
|
|
145
|
+
self.session_config = DEFAULT_SESSION_CONFIG if not session_config else session_config
|
|
145
146
|
self._get_new_session(self.session_config)
|
|
146
147
|
|
|
147
148
|
self._caching = caching
|
|
@@ -160,9 +161,6 @@ class DerivaBinding (object):
|
|
|
160
161
|
self._close_session()
|
|
161
162
|
self._session = get_new_requests_session(self._server_uri + '/',
|
|
162
163
|
session_config if session_config else self.session_config)
|
|
163
|
-
# allow loopback requests to bypass SSL cert verification
|
|
164
|
-
if "https://localhost" in self._server_uri:
|
|
165
|
-
self._session.verify = False
|
|
166
164
|
|
|
167
165
|
def _pre_get(self, path, headers):
|
|
168
166
|
self.check_path(path)
|
|
@@ -222,19 +220,21 @@ class DerivaBinding (object):
|
|
|
222
220
|
self._session.headers.update({'Authorization': 'Bearer {token}'.format(token=credentials['bearer-token'])})
|
|
223
221
|
elif 'cookie' in credentials:
|
|
224
222
|
cname, cval = credentials['cookie'].split('=', 1)
|
|
225
|
-
|
|
223
|
+
# Fix for cookielib domain rewrite to *.local when no "." in hostname. In this case, don't set the domain.
|
|
224
|
+
# Covers "localhost" and other dev/test scenarios. See "https://github.com/psf/requests/issues/5388"
|
|
225
|
+
self._session.cookies.set(cname, cval, domain="" if "." not in server else server, path='/')
|
|
226
226
|
elif 'username' in credentials and 'password' in credentials:
|
|
227
227
|
self.post_authn_session(credentials)
|
|
228
228
|
|
|
229
229
|
def get_authn_session(self):
|
|
230
230
|
headers = { 'deriva-client-context': self.dcctx.encoded() }
|
|
231
|
-
r = self._session.get(self.
|
|
231
|
+
r = self._session.get(self._auth_uri, headers=headers)
|
|
232
232
|
_response_raise_for_status(r)
|
|
233
233
|
return r
|
|
234
234
|
|
|
235
235
|
def post_authn_session(self, credentials):
|
|
236
236
|
headers = { 'deriva-client-context': self.dcctx.encoded() }
|
|
237
|
-
r = self._session.post(self.
|
|
237
|
+
r = self._session.post(self._auth_uri, data=credentials, headers=headers)
|
|
238
238
|
_response_raise_for_status(r)
|
|
239
239
|
return r
|
|
240
240
|
|
|
@@ -481,12 +481,26 @@ class ErmrestCatalog(DerivaBinding):
|
|
|
481
481
|
def getAsFile(self,
|
|
482
482
|
path,
|
|
483
483
|
destfilename,
|
|
484
|
-
headers=DEFAULT_HEADERS,
|
|
485
|
-
callback=None,
|
|
486
|
-
delete_if_empty=False,
|
|
487
|
-
paged=False,
|
|
488
|
-
page_size=DEFAULT_PAGE_SIZE,
|
|
489
|
-
page_sort_columns=frozenset(["RID"])):
|
|
484
|
+
headers = DEFAULT_HEADERS,
|
|
485
|
+
callback = None,
|
|
486
|
+
delete_if_empty = False,
|
|
487
|
+
paged = False,
|
|
488
|
+
page_size = DEFAULT_PAGE_SIZE,
|
|
489
|
+
page_sort_columns = frozenset(["RID"])):
|
|
490
|
+
"""
|
|
491
|
+
Deprecated, call `get_as_file` instead.
|
|
492
|
+
"""
|
|
493
|
+
self.get_as_file(path, destfilename, headers, callback, delete_if_empty, paged, page_size, page_sort_columns)
|
|
494
|
+
|
|
495
|
+
def get_as_file(self,
|
|
496
|
+
path,
|
|
497
|
+
destfilename,
|
|
498
|
+
headers=DEFAULT_HEADERS,
|
|
499
|
+
callback=None,
|
|
500
|
+
delete_if_empty=False,
|
|
501
|
+
paged=False,
|
|
502
|
+
page_size=DEFAULT_PAGE_SIZE,
|
|
503
|
+
page_sort_columns=frozenset(["RID"])):
|
|
490
504
|
"""
|
|
491
505
|
Retrieve catalog data streamed to destination file.
|
|
492
506
|
Caller is responsible to clean up file even on error, when the file may or may not exist.
|
|
@@ -188,7 +188,7 @@ def equivalent(doc1, doc2, method=None):
|
|
|
188
188
|
def canon_cat_acls(d):
|
|
189
189
|
return {
|
|
190
190
|
k: d.get(k, [])
|
|
191
|
-
for k in {'owner', '
|
|
191
|
+
for k in {'owner', 'enumerate', 'write', 'select', 'insert', 'update', 'delete'}
|
|
192
192
|
}
|
|
193
193
|
return equivalent(canon_cat_acls(doc1), canon_cat_acls(doc2), method='acls')
|
|
194
194
|
elif method == 'foreign_key_acls':
|
|
@@ -48,7 +48,8 @@ DEFAULT_SESSION_CONFIG = {
|
|
|
48
48
|
"allow_retry_on_all_methods": False,
|
|
49
49
|
"cookie_jar": DEFAULT_COOKIE_JAR_FILE,
|
|
50
50
|
"max_request_size": DEFAULT_MAX_REQUEST_SIZE,
|
|
51
|
-
"max_chunk_limit": DEFAULT_MAX_CHUNK_LIMIT
|
|
51
|
+
"max_chunk_limit": DEFAULT_MAX_CHUNK_LIMIT,
|
|
52
|
+
"bypass_cert_verify_host_list": []
|
|
52
53
|
}
|
|
53
54
|
OAUTH2_SCOPES_KEY = "oauth2_scopes"
|
|
54
55
|
DEFAULT_CONFIG = {
|
|
@@ -189,11 +190,15 @@ def get_new_requests_session(url=None, session_config=DEFAULT_SESSION_CONFIG):
|
|
|
189
190
|
not session_config.get("allow_retry_on_all_methods", False) else False,
|
|
190
191
|
raise_on_status=True)
|
|
191
192
|
adapter = TimeoutHTTPAdapter(timeout=session_config.get("timeout", DEFAULT_REQUESTS_TIMEOUT), max_retries=retries)
|
|
193
|
+
session.mount('http://', adapter)
|
|
194
|
+
session.mount('https://', adapter)
|
|
195
|
+
|
|
192
196
|
if url:
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
+
# allow whitelisted hosts to bypass SSL cert verification
|
|
198
|
+
upr = urlparse(url)
|
|
199
|
+
bypass_cert_verify_host_list = session_config.get("bypass_cert_verify_host_list", [])
|
|
200
|
+
if upr.scheme == "https" and upr.hostname in bypass_cert_verify_host_list:
|
|
201
|
+
session.verify = False
|
|
197
202
|
|
|
198
203
|
return session
|
|
199
204
|
|
|
@@ -301,7 +306,7 @@ def get_oauth_scopes_for_host(host,
|
|
|
301
306
|
result = scopes
|
|
302
307
|
break
|
|
303
308
|
if not result or force_refresh:
|
|
304
|
-
session = get_new_requests_session(session_config=DEFAULT_SESSION_CONFIG)
|
|
309
|
+
session = get_new_requests_session(url, session_config=config.get("session", DEFAULT_SESSION_CONFIG))
|
|
305
310
|
try:
|
|
306
311
|
r = session.get(url, headers=DEFAULT_HEADERS)
|
|
307
312
|
r.raise_for_status()
|
|
@@ -5,7 +5,7 @@ import json
|
|
|
5
5
|
import logging
|
|
6
6
|
import platform
|
|
7
7
|
import traceback
|
|
8
|
-
import
|
|
8
|
+
import time
|
|
9
9
|
import datetime
|
|
10
10
|
import tzlocal
|
|
11
11
|
import webbrowser
|
|
@@ -15,7 +15,8 @@ from bdbag.fetch.auth import keychain as bdbkc
|
|
|
15
15
|
from deriva.core import __version__ as VERSION, DEFAULT_CONFIG_PATH, DEFAULT_GLOBUS_CREDENTIAL_FILE, urlparse, urljoin,\
|
|
16
16
|
read_config, format_exception, BaseCLI, get_oauth_scopes_for_host, get_new_requests_session
|
|
17
17
|
from deriva.core.utils import eprint
|
|
18
|
-
from globus_sdk import ConfidentialAppAuthClient, AuthClient, AccessTokenAuthorizer,
|
|
18
|
+
from globus_sdk import ConfidentialAppAuthClient, AuthClient, AccessTokenAuthorizer, RefreshTokenAuthorizer, \
|
|
19
|
+
GlobusError, GlobusAPIError, AuthAPIError
|
|
19
20
|
from fair_research_login.client import NativeClient, LoadError, NoSavedTokens, TokensExpired
|
|
20
21
|
|
|
21
22
|
NATIVE_APP_CLIENT_ID = "8ef15ba9-2b4a-469c-a163-7fd910c9d111"
|
|
@@ -586,12 +587,47 @@ class GlobusNativeLogin:
|
|
|
586
587
|
if not scopes:
|
|
587
588
|
return None
|
|
588
589
|
try:
|
|
589
|
-
|
|
590
|
+
tokens = self.client.load_tokens(list(scopes))
|
|
591
|
+
return self.refresh_tokens_preemptively(tokens, time_before_expiry=86400)
|
|
590
592
|
except LoadError as e:
|
|
591
593
|
logging.debug("Unable to load or find tokens for specified scopes [%s]: %s" %
|
|
592
594
|
(scopes, format_exception(e)))
|
|
593
595
|
return None
|
|
594
596
|
|
|
597
|
+
def refresh_tokens_preemptively(self, tokens, time_before_expiry=0):
|
|
598
|
+
needs_refresh = {}
|
|
599
|
+
|
|
600
|
+
for rs, ts in {t: ts for t, ts in tokens.items() if bool(ts['refresh_token'])}.items():
|
|
601
|
+
expiry_threshold = int(time.time()) + time_before_expiry
|
|
602
|
+
if ts['expires_at_seconds'] < expiry_threshold:
|
|
603
|
+
needs_refresh.update({rs:ts})
|
|
604
|
+
|
|
605
|
+
if not needs_refresh:
|
|
606
|
+
return tokens
|
|
607
|
+
|
|
608
|
+
for rs, ts in needs_refresh.items():
|
|
609
|
+
authorizer = RefreshTokenAuthorizer(
|
|
610
|
+
ts['refresh_token'],
|
|
611
|
+
self.client.client,
|
|
612
|
+
access_token=ts['access_token'],
|
|
613
|
+
expires_at=int(time.time()),
|
|
614
|
+
# setting the expires_at to the current time is vicious hack but necessary since the globus_sdk
|
|
615
|
+
# client doesn't allow you to refresh an access token unless it has already expired
|
|
616
|
+
)
|
|
617
|
+
try:
|
|
618
|
+
authorizer.ensure_valid_token()
|
|
619
|
+
ts['access_token'] = authorizer.access_token
|
|
620
|
+
ts['expires_at_seconds'] = authorizer.expires_at
|
|
621
|
+
except AuthAPIError as e:
|
|
622
|
+
if e.message == 'invalid_grant':
|
|
623
|
+
logging.warning('Refresh Token expired for resource server: %s', rs)
|
|
624
|
+
|
|
625
|
+
self.client.save_tokens(needs_refresh)
|
|
626
|
+
result = {rs: ts for rs, ts in tokens.items() if rs not in needs_refresh}
|
|
627
|
+
result.update(needs_refresh)
|
|
628
|
+
|
|
629
|
+
return result
|
|
630
|
+
|
|
595
631
|
def hosts_to_scope_map(self,
|
|
596
632
|
hosts,
|
|
597
633
|
match_scope_tag=None,
|
|
@@ -662,6 +698,7 @@ class GlobusNativeLogin:
|
|
|
662
698
|
access_token = token.get("access_token")
|
|
663
699
|
if (token_scope == scope) and (access_token is not None):
|
|
664
700
|
return access_token
|
|
701
|
+
return None
|
|
665
702
|
|
|
666
703
|
def update_bdbag_keychain(self, token=None, host=None, keychain_file=None, allow_redirects=False, delete=False):
|
|
667
704
|
if (token is None) or (host is None):
|
|
@@ -699,7 +736,7 @@ class GlobusNativeLogin:
|
|
|
699
736
|
prefill_named_grant = self.client.app_name + " with requested scopes [%s] " % ", ".join(scopes)
|
|
700
737
|
tokens = self.client.login(no_local_server=no_local_server,
|
|
701
738
|
no_browser=no_browser,
|
|
702
|
-
requested_scopes=scopes,
|
|
739
|
+
requested_scopes=list(scopes),
|
|
703
740
|
refresh_tokens=refresh_tokens,
|
|
704
741
|
prefill_named_grant=prefill_named_grant,
|
|
705
742
|
query_params=additional_params,
|
|
@@ -45,6 +45,7 @@ class DerivaExport:
|
|
|
45
45
|
self.timeout = kwargs.get("timeout")
|
|
46
46
|
self.export_type = kwargs.get("export_type", "bdbag")
|
|
47
47
|
self.base_server_uri = "https://" + self.host
|
|
48
|
+
self.auth_service_url = self.base_server_uri + "/authn/session"
|
|
48
49
|
self.service_url = self.base_server_uri + EXPORT_SERVICE_PATH % self.export_type
|
|
49
50
|
self.session_config = DEFAULT_SESSION_CONFIG.copy()
|
|
50
51
|
if isinstance(self.timeout, tuple):
|
|
@@ -77,16 +78,23 @@ class DerivaExport:
|
|
|
77
78
|
raise DerivaDownloadAuthenticationError(
|
|
78
79
|
"The requested service requires authentication and a valid login credential could "
|
|
79
80
|
"not be found (or was not provided) for the specified host.")
|
|
81
|
+
|
|
80
82
|
if 'bearer-token' in self.credential:
|
|
81
83
|
self.session.headers.update(
|
|
82
84
|
{'Authorization': 'Bearer {token}'.format(token=self.credential['bearer-token'])})
|
|
83
85
|
elif 'cookie' in self.credential:
|
|
84
86
|
cname, cval = self.credential['cookie'].split('=', 1)
|
|
85
|
-
self.session.cookies.set(cname, cval, domain=self.host, path='/')
|
|
87
|
+
self.session.cookies.set(cname, cval, domain="" if "." not in self.host else self.host, path='/')
|
|
88
|
+
else:
|
|
89
|
+
try:
|
|
90
|
+
r = self.session.post(self.auth_service_url, data=self.credential)
|
|
91
|
+
r.raise_for_status()
|
|
92
|
+
except HTTPError as e:
|
|
93
|
+
raise DerivaDownloadAuthenticationError(
|
|
94
|
+
"Exception during POST authentication flow: %s" % format_exception(e))
|
|
86
95
|
|
|
87
96
|
def validate_authn_session(self):
|
|
88
|
-
|
|
89
|
-
r = self.session.get(url)
|
|
97
|
+
r = self.session.get(self.auth_service_url)
|
|
90
98
|
if r.status_code == requests.codes.not_found or r.status_code == requests.codes.unauthorized:
|
|
91
99
|
logger.warning("Unable to authenticate. Check for missing or expired credentials.")
|
|
92
100
|
r.raise_for_status()
|
{deriva-1.7.8 → deriva-1.7.9}/deriva/transfer/download/processors/query/base_query_processor.py
RENAMED
|
@@ -71,13 +71,13 @@ class BaseQueryProcessor(BaseProcessor):
|
|
|
71
71
|
make_dirs(output_dir)
|
|
72
72
|
try:
|
|
73
73
|
if as_file:
|
|
74
|
-
return self.catalog.
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
74
|
+
return self.catalog.get_as_file(self.query, self.output_abspath,
|
|
75
|
+
headers=headers,
|
|
76
|
+
callback=self.callback,
|
|
77
|
+
delete_if_empty=True,
|
|
78
|
+
paged=self.paged_query,
|
|
79
|
+
page_size=self.paged_query_size,
|
|
80
|
+
page_sort_columns=self.paged_query_sort_columns)
|
|
81
81
|
else:
|
|
82
82
|
return self.catalog.get(self.query, headers=headers).json()
|
|
83
83
|
except requests.HTTPError as e:
|
|
@@ -142,21 +142,21 @@ class BaseQueryProcessor(BaseProcessor):
|
|
|
142
142
|
|
|
143
143
|
return url
|
|
144
144
|
|
|
145
|
-
def getExternalSession(self,
|
|
146
|
-
sessions = self.sessions
|
|
145
|
+
def getExternalSession(self, url):
|
|
147
146
|
auth_params = self.kwargs.get("auth_params", dict())
|
|
148
147
|
cookies = auth_params.get("cookies")
|
|
149
148
|
auth_url = auth_params.get("auth_url")
|
|
150
149
|
login_params = auth_params.get("login_params")
|
|
151
150
|
session_config = self.kwargs.get("session_config")
|
|
152
151
|
|
|
153
|
-
|
|
152
|
+
host = urlsplit(url).hostname
|
|
153
|
+
session = self.sessions.get(host)
|
|
154
154
|
if session is not None:
|
|
155
155
|
return session
|
|
156
156
|
|
|
157
157
|
if not session_config:
|
|
158
158
|
session_config = DEFAULT_SESSION_CONFIG
|
|
159
|
-
session = get_new_requests_session(session_config=session_config)
|
|
159
|
+
session = get_new_requests_session(url, session_config=session_config)
|
|
160
160
|
|
|
161
161
|
if cookies:
|
|
162
162
|
session.cookies.update(cookies)
|
|
@@ -166,7 +166,7 @@ class BaseQueryProcessor(BaseProcessor):
|
|
|
166
166
|
raise DerivaDownloadError(
|
|
167
167
|
'GetExternalSession Failed with Status Code: %s\n%s\n' % (r.status_code, r.text))
|
|
168
168
|
|
|
169
|
-
sessions[host] = session
|
|
169
|
+
self.sessions[host] = session
|
|
170
170
|
return session
|
|
171
171
|
|
|
172
172
|
def create_default_paths(self):
|
|
@@ -31,13 +31,12 @@ class FileDownloadQueryProcessor(BaseQueryProcessor):
|
|
|
31
31
|
return self.outputs
|
|
32
32
|
|
|
33
33
|
def getExternalFile(self, url, output_path, headers=None):
|
|
34
|
-
host = urlsplit(url).netloc
|
|
35
34
|
if output_path:
|
|
36
35
|
if not headers:
|
|
37
36
|
headers = self.HEADERS.copy()
|
|
38
37
|
else:
|
|
39
38
|
headers.update(self.HEADERS)
|
|
40
|
-
session = self.getExternalSession(
|
|
39
|
+
session = self.getExternalSession(url)
|
|
41
40
|
with session.get(url, headers=headers, stream=True) as r:
|
|
42
41
|
if r.status_code != 200:
|
|
43
42
|
file_error = "File [%s] transfer failed." % output_path
|
|
@@ -58,6 +57,7 @@ class FileDownloadQueryProcessor(BaseQueryProcessor):
|
|
|
58
57
|
length = int(r.headers.get('Content-Length'))
|
|
59
58
|
content_type = r.headers.get("Content-Type")
|
|
60
59
|
return output_path, length, content_type
|
|
60
|
+
return None
|
|
61
61
|
|
|
62
62
|
def downloadFiles(self, input_manifest):
|
|
63
63
|
logging.info("Attempting to download file(s) based on the results of query: %s" % self.query)
|
|
@@ -346,7 +346,7 @@ class DerivaUpload(object):
|
|
|
346
346
|
return '%s:%s' % (urlquote(schema_name), urlquote(table_name))
|
|
347
347
|
|
|
348
348
|
@staticmethod
|
|
349
|
-
def interpolateDict(src, dst,
|
|
349
|
+
def interpolateDict(src, dst, allow_none_column_list=[]):
|
|
350
350
|
if not (isinstance(src, dict) and isinstance(dst, dict)):
|
|
351
351
|
raise ValueError("Invalid input parameter type(s): (src = %s, dst = %s), expected (dict, dict)" % (
|
|
352
352
|
type(src).__name__, type(dst).__name__))
|
|
@@ -360,26 +360,27 @@ class DerivaUpload(object):
|
|
|
360
360
|
for k, v in dst.items():
|
|
361
361
|
try:
|
|
362
362
|
value = v.format(**src)
|
|
363
|
+
if k in src:
|
|
364
|
+
v_type = type(src[k])
|
|
365
|
+
value = v_type(value)
|
|
363
366
|
except KeyError:
|
|
364
367
|
value = v
|
|
365
368
|
if value:
|
|
366
369
|
if value.startswith('{') and value.endswith('}'):
|
|
367
370
|
value = None
|
|
368
371
|
dst.update({k: value})
|
|
369
|
-
# remove all None valued entries in the dest, if
|
|
370
|
-
|
|
371
|
-
for k in empty:
|
|
372
|
-
if not allow_none or (allow_none and k not in allow_none_column_list):
|
|
373
|
-
del dst[k]
|
|
372
|
+
# remove all None valued entries in the dest, if column is not explicitly allowed
|
|
373
|
+
dst = {k: v for k, v in dst.items() if v is not None or k in allow_none_column_list}
|
|
374
374
|
|
|
375
375
|
return dst
|
|
376
376
|
|
|
377
377
|
@staticmethod
|
|
378
|
-
def pruneDict(src, dst,
|
|
378
|
+
def pruneDict(src, dst, allow_none_column_list=[]):
|
|
379
379
|
dst = dst.copy()
|
|
380
380
|
for k in dst.keys():
|
|
381
381
|
value = src.get(k)
|
|
382
|
-
|
|
382
|
+
if value is not None or (value is None and k in allow_none_column_list):
|
|
383
|
+
dst[k] = value
|
|
383
384
|
return dst
|
|
384
385
|
|
|
385
386
|
def getCurrentConfigFilePath(self):
|
|
@@ -716,7 +717,8 @@ class DerivaUpload(object):
|
|
|
716
717
|
self.metadata["URI"] = versioned_uri
|
|
717
718
|
else:
|
|
718
719
|
self.metadata["URI"] = versioned_uri.rsplit(":")[0]
|
|
719
|
-
|
|
720
|
+
safe_overrides = asset_mapping.get("url_encoding_safe_overrides", {}).get("URI", "")
|
|
721
|
+
self.metadata["URI_urlencoded"] = urlquote(self.metadata["URI"], safe=safe_overrides)
|
|
720
722
|
|
|
721
723
|
# 7. Check for an existing record and create a new one if necessary
|
|
722
724
|
if not record:
|
|
@@ -725,8 +727,7 @@ class DerivaUpload(object):
|
|
|
725
727
|
# 8. Update an existing record, if necessary
|
|
726
728
|
column_map = asset_mapping.get("column_map", {})
|
|
727
729
|
allow_none_col_list = asset_mapping.get("allow_empty_columns_on_update", [])
|
|
728
|
-
|
|
729
|
-
updated_record = self.interpolateDict(self.metadata, column_map, allow_none, allow_none_col_list)
|
|
730
|
+
updated_record = self.interpolateDict(self.metadata, column_map, allow_none_col_list)
|
|
730
731
|
if updated_record != record:
|
|
731
732
|
record_update_template = asset_mapping.get("record_update_template")
|
|
732
733
|
require_record_update_template = stob(asset_mapping.get("require_record_update_template", False))
|
|
@@ -784,6 +785,7 @@ class DerivaUpload(object):
|
|
|
784
785
|
"""
|
|
785
786
|
record = None
|
|
786
787
|
column_map = asset_mapping.get("column_map", {})
|
|
788
|
+
allow_none_col_list = asset_mapping.get("allow_empty_columns_on_update", [])
|
|
787
789
|
rqt = asset_mapping['record_query_template']
|
|
788
790
|
try:
|
|
789
791
|
path = rqt.format(**self.metadata)
|
|
@@ -793,14 +795,14 @@ class DerivaUpload(object):
|
|
|
793
795
|
if result:
|
|
794
796
|
record = result[0]
|
|
795
797
|
self._updateFileMetadata(record, no_overwrite=True)
|
|
796
|
-
return self.pruneDict(record, column_map), record
|
|
798
|
+
return self.pruneDict(record, column_map, allow_none_col_list), record
|
|
797
799
|
else:
|
|
798
800
|
row = self.interpolateDict(self.metadata, column_map)
|
|
799
801
|
result = self._catalogRecordCreate(self.metadata['target_table'], row)
|
|
800
802
|
if result:
|
|
801
803
|
record = result[0]
|
|
802
804
|
self._updateFileMetadata(record)
|
|
803
|
-
return self.interpolateDict(self.metadata, column_map,
|
|
805
|
+
return self.interpolateDict(self.metadata, column_map, allow_none_column_list=allow_none_col_list), record
|
|
804
806
|
|
|
805
807
|
def _urlEncodeMetadata(self, safe_overrides=None):
|
|
806
808
|
urlencoded = dict()
|
|
@@ -892,12 +894,14 @@ class DerivaUpload(object):
|
|
|
892
894
|
def _getFileHatracMetadata(self, asset_mapping):
|
|
893
895
|
try:
|
|
894
896
|
hatrac_templates = asset_mapping["hatrac_templates"]
|
|
897
|
+
# convert None values to empty strings for URI and content-disposition template replacement
|
|
898
|
+
metadata = {k: ('' if v is None else v) for k, v in self.metadata.items()}
|
|
895
899
|
# URI is required
|
|
896
|
-
self.metadata["URI"] = hatrac_templates["hatrac_uri"].format(**
|
|
900
|
+
self.metadata["URI"] = hatrac_templates["hatrac_uri"].format(**metadata)
|
|
897
901
|
# overridden content-disposition is optional
|
|
898
902
|
content_disposition = hatrac_templates.get("content-disposition")
|
|
899
903
|
if content_disposition:
|
|
900
|
-
filename = content_disposition.format(**
|
|
904
|
+
filename = content_disposition.format(**metadata)
|
|
901
905
|
else:
|
|
902
906
|
filename = urlparse(self.metadata["URI"]).path.rsplit("/", 1)[-1]
|
|
903
907
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: deriva
|
|
3
|
-
Version: 1.7.
|
|
3
|
+
Version: 1.7.9
|
|
4
4
|
Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
|
|
5
5
|
Home-page: https://github.com/informatics-isi-edu/deriva-py
|
|
6
6
|
Author: USC Information Sciences Institute, Informatics Systems Research Division
|
|
@@ -26,12 +26,25 @@ License-File: LICENSE
|
|
|
26
26
|
Requires-Dist: packaging
|
|
27
27
|
Requires-Dist: requests
|
|
28
28
|
Requires-Dist: pika
|
|
29
|
-
Requires-Dist: urllib3<3,>=1.26
|
|
29
|
+
Requires-Dist: urllib3<3,>=1.26.20
|
|
30
30
|
Requires-Dist: portalocker>=1.2.1
|
|
31
|
-
Requires-Dist: bdbag>=1.7.
|
|
31
|
+
Requires-Dist: bdbag>=1.7.5
|
|
32
32
|
Requires-Dist: globus_sdk<4,>=3
|
|
33
33
|
Requires-Dist: fair-research-login>=0.3.1
|
|
34
34
|
Requires-Dist: fair-identifiers-client>=0.5.1
|
|
35
35
|
Requires-Dist: jsonschema>=3.1
|
|
36
|
+
Dynamic: author
|
|
37
|
+
Dynamic: author-email
|
|
38
|
+
Dynamic: classifier
|
|
39
|
+
Dynamic: description
|
|
40
|
+
Dynamic: description-content-type
|
|
41
|
+
Dynamic: home-page
|
|
42
|
+
Dynamic: license
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
Dynamic: maintainer
|
|
45
|
+
Dynamic: maintainer-email
|
|
46
|
+
Dynamic: requires-dist
|
|
47
|
+
Dynamic: requires-python
|
|
48
|
+
Dynamic: summary
|
|
36
49
|
|
|
37
50
|
For further information, visit the project [homepage](https://github.com/informatics-isi-edu/deriva-py).
|
|
@@ -65,9 +65,9 @@ setup(
|
|
|
65
65
|
'packaging',
|
|
66
66
|
'requests',
|
|
67
67
|
'pika',
|
|
68
|
-
'urllib3>=1.26,<3',
|
|
68
|
+
'urllib3>=1.26.20,<3',
|
|
69
69
|
'portalocker>=1.2.1',
|
|
70
|
-
'bdbag>=1.7.
|
|
70
|
+
'bdbag>=1.7.5',
|
|
71
71
|
'globus_sdk>=3,<4',
|
|
72
72
|
'fair-research-login>=0.3.1',
|
|
73
73
|
'fair-identifiers-client>=0.5.1',
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|