deriva 1.7.10__py3-none-any.whl → 1.7.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deriva/core/__init__.py +1 -1
- deriva/core/catalog_cli.py +16 -18
- deriva/core/ermrest_catalog.py +1 -1
- deriva/core/ermrest_model.py +92 -1
- deriva/core/hatrac_cli.py +30 -2
- deriva/core/hatrac_store.py +88 -33
- deriva/core/utils/core_utils.py +312 -35
- deriva/core/utils/credenza_auth_utils.py +645 -0
- deriva/core/utils/globus_auth_utils.py +7 -13
- deriva/transfer/backup/deriva_backup.py +2 -2
- deriva/transfer/backup/deriva_backup_cli.py +5 -0
- deriva/transfer/upload/deriva_upload.py +18 -2
- {deriva-1.7.10.dist-info → deriva-1.7.12.dist-info}/METADATA +4 -3
- {deriva-1.7.10.dist-info → deriva-1.7.12.dist-info}/RECORD +18 -28
- {deriva-1.7.10.dist-info → deriva-1.7.12.dist-info}/WHEEL +1 -1
- {deriva-1.7.10.dist-info → deriva-1.7.12.dist-info}/entry_points.txt +1 -0
- {deriva-1.7.10.dist-info → deriva-1.7.12.dist-info}/top_level.txt +0 -1
- tests/deriva/__init__.py +0 -0
- tests/deriva/core/__init__.py +0 -0
- tests/deriva/core/mmo/__init__.py +0 -0
- tests/deriva/core/mmo/base.py +0 -300
- tests/deriva/core/mmo/test_mmo_drop.py +0 -252
- tests/deriva/core/mmo/test_mmo_find.py +0 -90
- tests/deriva/core/mmo/test_mmo_prune.py +0 -196
- tests/deriva/core/mmo/test_mmo_rename.py +0 -222
- tests/deriva/core/mmo/test_mmo_replace.py +0 -180
- tests/deriva/core/test_datapath.py +0 -893
- tests/deriva/core/test_ermrest_model.py +0 -782
- {deriva-1.7.10.dist-info → deriva-1.7.12.dist-info}/licenses/LICENSE +0 -0
|
@@ -25,6 +25,11 @@ class DerivaBackupCLI(DerivaDownloadCLI):
|
|
|
25
25
|
"references to asset files in the bag's \"fetch.txt\" file.")
|
|
26
26
|
self.parser.add_argument("--bag-archiver", choices=['zip', 'tgz', 'bz2'],
|
|
27
27
|
help="Format for compressed bag output.")
|
|
28
|
+
self.parser.add_argument("--ignore-acl", action="store_true",
|
|
29
|
+
help="If no ACL is present in the catalog schema, which may mean the user is not "
|
|
30
|
+
"the catalog owner, proceed anyway. WARNING: this will result in the backup "
|
|
31
|
+
"process running under whatever permissions the user possesses, which may be "
|
|
32
|
+
"limited enough to result in incomplete schema, incomplete data, or both.")
|
|
28
33
|
self.parser.add_argument("--exclude-data", default=list(),
|
|
29
34
|
type=lambda s: [item.strip() for item in s.split(',')],
|
|
30
35
|
metavar="<schema>, <schema:table>, ...",
|
|
@@ -695,6 +695,16 @@ class DerivaUpload(object):
|
|
|
695
695
|
# 6. Perform the Hatrac upload
|
|
696
696
|
self._getFileHatracMetadata(asset_mapping)
|
|
697
697
|
hatrac_options = asset_mapping.get("hatrac_options", {})
|
|
698
|
+
v = hatrac_options.get("chunk_size")
|
|
699
|
+
try:
|
|
700
|
+
chunk_size = int(v) if v not in (None, "") else DEFAULT_CHUNK_SIZE
|
|
701
|
+
except (TypeError, ValueError):
|
|
702
|
+
chunk_size = DEFAULT_CHUNK_SIZE
|
|
703
|
+
if chunk_size <= 0:
|
|
704
|
+
chunk_size = DEFAULT_CHUNK_SIZE
|
|
705
|
+
logger.warning(
|
|
706
|
+
"Specified chunk_size must be a positive integer (> 0) - falling back to default chunk size: %d." %
|
|
707
|
+
DEFAULT_CHUNK_SIZE)
|
|
698
708
|
file_size = self.metadata["file_size"]
|
|
699
709
|
versioned_uri = \
|
|
700
710
|
self._hatracUpload(self.metadata["URI"],
|
|
@@ -704,8 +714,10 @@ class DerivaUpload(object):
|
|
|
704
714
|
content_type=self.guessContentType(file_path),
|
|
705
715
|
content_disposition=self.metadata.get("content-disposition"),
|
|
706
716
|
chunked=True if (file_size > DEFAULT_CHUNK_SIZE or file_size == 0) else False,
|
|
717
|
+
chunk_size=chunk_size,
|
|
707
718
|
create_parents=stob(hatrac_options.get("create_parents", True)),
|
|
708
719
|
allow_versioning=stob(hatrac_options.get("allow_versioning", True)),
|
|
720
|
+
force=stob(hatrac_options.get("force", False)),
|
|
709
721
|
callback=callback)
|
|
710
722
|
logger.debug("Hatrac upload successful. Result object URI: %s" % versioned_uri)
|
|
711
723
|
versioned_uris = True
|
|
@@ -942,9 +954,11 @@ class DerivaUpload(object):
|
|
|
942
954
|
content_type=None,
|
|
943
955
|
content_disposition=None,
|
|
944
956
|
chunked=True,
|
|
957
|
+
chunk_size=DEFAULT_CHUNK_SIZE,
|
|
945
958
|
create_parents=True,
|
|
946
959
|
allow_versioning=True,
|
|
947
|
-
callback=None
|
|
960
|
+
callback=None,
|
|
961
|
+
force=False):
|
|
948
962
|
|
|
949
963
|
# check if there is already an in-progress transfer for this file,
|
|
950
964
|
# and if so, that the local file has not been modified since the original upload job was created
|
|
@@ -979,10 +993,12 @@ class DerivaUpload(object):
|
|
|
979
993
|
content_type=content_type,
|
|
980
994
|
content_disposition=content_disposition,
|
|
981
995
|
chunked=chunked,
|
|
996
|
+
chunk_size=chunk_size,
|
|
982
997
|
create_parents=create_parents,
|
|
983
998
|
allow_versioning=allow_versioning,
|
|
984
999
|
callback=callback,
|
|
985
|
-
cancel_job_on_error=False
|
|
1000
|
+
cancel_job_on_error=False,
|
|
1001
|
+
force=force)
|
|
986
1002
|
|
|
987
1003
|
def _get_catalog_table_columns(self, table):
|
|
988
1004
|
table_columns = set()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deriva
|
|
3
|
-
Version: 1.7.
|
|
3
|
+
Version: 1.7.12
|
|
4
4
|
Summary: Python APIs and CLIs (Command-Line Interfaces) for the DERIVA platform.
|
|
5
5
|
Home-page: https://github.com/informatics-isi-edu/deriva-py
|
|
6
6
|
Author: USC Information Sciences Institute, Informatics Systems Research Division
|
|
@@ -15,11 +15,12 @@ Classifier: Operating System :: POSIX
|
|
|
15
15
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
16
16
|
Classifier: Operating System :: Microsoft :: Windows
|
|
17
17
|
Classifier: Programming Language :: Python
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.9
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
22
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
24
|
Requires-Python: >=3.8, <4
|
|
24
25
|
Description-Content-Type: text/markdown
|
|
25
26
|
License-File: LICENSE
|
|
@@ -28,7 +29,7 @@ Requires-Dist: requests
|
|
|
28
29
|
Requires-Dist: pika
|
|
29
30
|
Requires-Dist: urllib3<3,>=1.26.20
|
|
30
31
|
Requires-Dist: portalocker>=1.2.1
|
|
31
|
-
Requires-Dist: bdbag>=1.
|
|
32
|
+
Requires-Dist: bdbag>=1.8.0
|
|
32
33
|
Requires-Dist: globus_sdk<4,>=3
|
|
33
34
|
Requires-Dist: fair-research-login>=0.3.1
|
|
34
35
|
Requires-Dist: fair-identifiers-client>=0.5.1
|
|
@@ -8,17 +8,17 @@ deriva/config/dump_catalog_annotations.py,sha256=QzaWDLfWIAQ0eWVV11zeceWgwDBOYIe
|
|
|
8
8
|
deriva/config/rollback_annotation.py,sha256=vqrIcen-KZX8LDpu2OVNivzIHpQoQgWkZAChZJctvtk,3015
|
|
9
9
|
deriva/config/examples/group_owner_policy.json,sha256=8v3GWM1F_BWnYD9x_f6Eo4kBDvyy8g7mRqujfoEKLNc,2408
|
|
10
10
|
deriva/config/examples/self_serve_policy.json,sha256=pW-cqWz4rJNNXwY4eVZFkQ8gKCHclC9yDa22ylfcDqY,1676
|
|
11
|
-
deriva/core/__init__.py,sha256=
|
|
11
|
+
deriva/core/__init__.py,sha256=eknMCmzhKiL-URSE8NKY6-D1OLP0oaFZKtiOjEeH3XI,4976
|
|
12
12
|
deriva/core/annotation.py,sha256=PkAkPkxX1brQsb8_drR1Qj5QjQA5mjkpXhkq9NuZ1g8,13432
|
|
13
13
|
deriva/core/base_cli.py,sha256=78Ilf3_f2xREQb3IIj6q0jwWAiXSObZszG0JURs36lA,2902
|
|
14
|
-
deriva/core/catalog_cli.py,sha256=
|
|
14
|
+
deriva/core/catalog_cli.py,sha256=2Hq0yVVYtH7wSkGyXTvDjSJ8hwGfAMk7Z6X3JnwcKDA,23390
|
|
15
15
|
deriva/core/datapath.py,sha256=h9WzLipAYctppoiZehyDNJdFpCnbU9-z-AWVD80z2Pk,90525
|
|
16
16
|
deriva/core/deriva_binding.py,sha256=6yGXIbnayDtb9LU-JVK43zk02-aQwshrKiaLi1pru-8,13086
|
|
17
17
|
deriva/core/deriva_server.py,sha256=nsW3gwg1sIaHl3BTf-nL41AkSj3dEpcEBlatvjvN8CQ,200
|
|
18
|
-
deriva/core/ermrest_catalog.py,sha256
|
|
19
|
-
deriva/core/ermrest_model.py,sha256=
|
|
20
|
-
deriva/core/hatrac_cli.py,sha256=
|
|
21
|
-
deriva/core/hatrac_store.py,sha256=
|
|
18
|
+
deriva/core/ermrest_catalog.py,sha256=-iCYwuzO55HmDLB-BYGfQWDdqKTS4mYAfUlrBmq0KmY,55557
|
|
19
|
+
deriva/core/ermrest_model.py,sha256=sgg9PF-Y1jg-2KUrM9WgpR3A0O8KcQ_wGs0_KG38ckM,128196
|
|
20
|
+
deriva/core/hatrac_cli.py,sha256=G4iDQe7h45UCmd1uM6c1UpgLSu0lKiqUhm-OFirCFCY,15934
|
|
21
|
+
deriva/core/hatrac_store.py,sha256=k783qeX3Wo1285ncN6Mr_Hbn9cSfmEDhSpQGF3dCm_M,24637
|
|
22
22
|
deriva/core/mmo.py,sha256=dcB8akgsqbYMi22ClbVpOKVL6so8FjSDjSb6gP4_jFo,17852
|
|
23
23
|
deriva/core/polling_ermrest_catalog.py,sha256=KsjiFqPQaHWnJZCVF5i77sdzfubqZHgMBbQ1p8V8D3s,10351
|
|
24
24
|
deriva/core/schemas/app_links.schema.json,sha256=AxrkC2scxomM6N7jyjtdYA73BbZzPrmuqU8PYWe7okI,954
|
|
@@ -44,8 +44,9 @@ deriva/core/schemas/visible_columns.schema.json,sha256=-JKqvhfKT-5btKBtzKn3p6EgY
|
|
|
44
44
|
deriva/core/schemas/visible_foreign_keys.schema.json,sha256=K-oa2qzj5EbmJCEyN6mN3vubblHENFxxbZEeQfFV_QQ,3364
|
|
45
45
|
deriva/core/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
46
|
deriva/core/utils/__init__.py,sha256=XSbGaWe44hebxYvoh5huFzZkMY6TSKPOCRSjUOvaY70,124
|
|
47
|
-
deriva/core/utils/core_utils.py,sha256=
|
|
48
|
-
deriva/core/utils/
|
|
47
|
+
deriva/core/utils/core_utils.py,sha256=rSidaI2oY_eAO6MT68QAK9eJTLpibcaPni5Rg5axYuU,29569
|
|
48
|
+
deriva/core/utils/credenza_auth_utils.py,sha256=0LlZeYgivBiWGAnKie7qHtSNdOH13frPyd0DYDDkkBo,25705
|
|
49
|
+
deriva/core/utils/globus_auth_utils.py,sha256=fzMFvNIkaaCHNMB6xXPXWCeo0DblLrVVbek6MzB9EY0,57184
|
|
49
50
|
deriva/core/utils/hash_utils.py,sha256=JqUYVB3jXusCQYX9fkKmweUKBC0WQi8ZI2N8m-uKygQ,2299
|
|
50
51
|
deriva/core/utils/mime_utils.py,sha256=ZT7pMjY2kQWBsgsGC3jY6kjfygdsIyiYW3BKNw_pPyg,1128
|
|
51
52
|
deriva/core/utils/version_utils.py,sha256=HWNUQAZrPXu0oGjhG6gMNm3kWtfa3nR50vfH2uQxBA0,2954
|
|
@@ -56,8 +57,8 @@ deriva/seo/sitemap_cli.py,sha256=miCqRfpSj5Dx5BfJGSd8Pi2e4OOQjotDzP_JubukhCM,265
|
|
|
56
57
|
deriva/transfer/__init__.py,sha256=VRxmElD5SIOGzrB1biUU8bP3_iXPMvJuau-9alU5oy0,1266
|
|
57
58
|
deriva/transfer/backup/__init__.py,sha256=vxsZiDLMTJQPybXT89G-07GsUoLhnItTCbLdXcDSyeA,465
|
|
58
59
|
deriva/transfer/backup/__main__.py,sha256=dT12--8C6sKGEtMhsYuy013ebXKpVnBJfhcQNlVtv6Y,361
|
|
59
|
-
deriva/transfer/backup/deriva_backup.py,sha256=
|
|
60
|
-
deriva/transfer/backup/deriva_backup_cli.py,sha256=
|
|
60
|
+
deriva/transfer/backup/deriva_backup.py,sha256=CgMvi0HwQohf2F7bvPk3LF2sFs3AS8ikvNSStF2VETk,5223
|
|
61
|
+
deriva/transfer/backup/deriva_backup_cli.py,sha256=GcutL_3JjzYMHMQ02pckLby_37b8rR48CXHjeYrPq60,2706
|
|
61
62
|
deriva/transfer/download/__init__.py,sha256=Pr7Zud4AFsIWwopTxeC_pupslgCG_lzycO9w9Xyh88Q,350
|
|
62
63
|
deriva/transfer/download/__main__.py,sha256=YUg7AZ07t_xaOgtfJnU_l1nkEHCCPR8sU5X-l1An6SY,363
|
|
63
64
|
deriva/transfer/download/deriva_download.py,sha256=ulFrHHEDj3oJA2pAo7MGvGyDF9rA3l8yCUoK3FMvHEk,17100
|
|
@@ -87,7 +88,7 @@ deriva/transfer/restore/deriva_restore.py,sha256=s0h7cXit2USSdjrIfrj0dr7BJ0rrHHM
|
|
|
87
88
|
deriva/transfer/restore/deriva_restore_cli.py,sha256=2ViZ1Lyl5ndXPKeJFCHHGnwzkg3DfHhTuRa_bN7eJm8,5603
|
|
88
89
|
deriva/transfer/upload/__init__.py,sha256=4mlc_iUX-v7SpXzlCZmhxQtSiW5JeDGb2FX7bb1E6tY,304
|
|
89
90
|
deriva/transfer/upload/__main__.py,sha256=hqnXtGpRqPthwpO6uvrnf_TQm7McheeyOt960hStSMY,340
|
|
90
|
-
deriva/transfer/upload/deriva_upload.py,sha256=
|
|
91
|
+
deriva/transfer/upload/deriva_upload.py,sha256=UPwg8qt_JJkCjpgN4JpLED0ITallTsCgcR30rOLbOAM,62514
|
|
91
92
|
deriva/transfer/upload/deriva_upload_cli.py,sha256=-Q6xgiYabQziTQcMQdGNDAv-eLxCCHO-BCSo4umbDE4,5082
|
|
92
93
|
deriva/transfer/upload/processors/__init__.py,sha256=sMM5xdJ82UIRdB1lGMKk7ft0BgtjS2oJ0sI4SQSqiIU,2481
|
|
93
94
|
deriva/transfer/upload/processors/archive_processor.py,sha256=ID0lDwDn4vPe5nbxy6m28Ssj_TsZpK4df2xRrM6nJRQ,2015
|
|
@@ -97,20 +98,9 @@ deriva/transfer/upload/processors/metadata_update_processor.py,sha256=Hgu5huZf7Z
|
|
|
97
98
|
deriva/transfer/upload/processors/rename_processor.py,sha256=UQ-JQuQgyYCGT-fU9kHA53kPdQ20kt-2Bb486od7B14,2423
|
|
98
99
|
deriva/transfer/upload/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
99
100
|
deriva/utils/__init__.py,sha256=jv2YF__bseklT3OWEzlqJ5qE24c4aWd5F4r0TTjOrWQ,65
|
|
100
|
-
deriva-1.7.
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
tests/deriva/core/mmo/base.py,sha256=3iLqUDNdiJaERB-8-G-5NEvkJr9fz0YUMvPxl5hMINE,9356
|
|
107
|
-
tests/deriva/core/mmo/test_mmo_drop.py,sha256=R_yl-PIHGx-3kchT5TH8O9TJktepWjgs6hHISOc6d2k,8930
|
|
108
|
-
tests/deriva/core/mmo/test_mmo_find.py,sha256=PcUN76sik68B3XKg0G3wHVpKcPEld_6RtbxeGzkrMQ8,4172
|
|
109
|
-
tests/deriva/core/mmo/test_mmo_prune.py,sha256=4pYtYL8g1BgadlewNPVpVA5lT_gV6SPTDYf04ZKzBTA,6851
|
|
110
|
-
tests/deriva/core/mmo/test_mmo_rename.py,sha256=4oSR1G3Od701Ss3AnolI1Z7CbMxKuQF2uSr2_IcoR6s,8512
|
|
111
|
-
tests/deriva/core/mmo/test_mmo_replace.py,sha256=w-66LWyiQ_ajC7Ipmhc4kAKwIloPdQELeUPsvelTdX8,8439
|
|
112
|
-
deriva-1.7.10.dist-info/METADATA,sha256=OrMw19lEt6tH9VyJq5a8wL4WpeIu-Wr1a8rjrJLKNdk,1891
|
|
113
|
-
deriva-1.7.10.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
|
|
114
|
-
deriva-1.7.10.dist-info/entry_points.txt,sha256=HmYCHlgbjYQ_aZX_j4_4tApH4tDTbYtS66jKlfytbn8,850
|
|
115
|
-
deriva-1.7.10.dist-info/top_level.txt,sha256=_LHDie5-O53wFlexfrxjewpVkf04oydf3CqX5h75DXE,13
|
|
116
|
-
deriva-1.7.10.dist-info/RECORD,,
|
|
101
|
+
deriva-1.7.12.dist-info/licenses/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
|
|
102
|
+
deriva-1.7.12.dist-info/METADATA,sha256=OTDbjqhWLZwSiWGwo9aTcV_Alp9vvYqbN4repIpNo9w,1944
|
|
103
|
+
deriva-1.7.12.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
104
|
+
deriva-1.7.12.dist-info/entry_points.txt,sha256=uS-e7FxTP7lVm-QJdq_ErytmAbXvlRkEvrjab33ctRE,922
|
|
105
|
+
deriva-1.7.12.dist-info/top_level.txt,sha256=OoTnCAuRnf-pPKMre5HA7mgLMIPl_2-lAWoZ63y74-U,7
|
|
106
|
+
deriva-1.7.12.dist-info/RECORD,,
|
|
@@ -6,6 +6,7 @@ deriva-annotation-rollback = deriva.config.rollback_annotation:main
|
|
|
6
6
|
deriva-annotation-validate = deriva.config.annotation_validate:main
|
|
7
7
|
deriva-backup-cli = deriva.transfer.backup.__main__:main
|
|
8
8
|
deriva-catalog-cli = deriva.core.catalog_cli:main
|
|
9
|
+
deriva-credenza-auth-utils = deriva.core.utils.credenza_auth_utils:main
|
|
9
10
|
deriva-download-cli = deriva.transfer.download.__main__:main
|
|
10
11
|
deriva-export-cli = deriva.transfer.download.deriva_export:main
|
|
11
12
|
deriva-globus-auth-utils = deriva.core.utils.globus_auth_utils:main
|
tests/deriva/__init__.py
DELETED
|
File without changes
|
tests/deriva/core/__init__.py
DELETED
|
File without changes
|
|
File without changes
|
tests/deriva/core/mmo/base.py
DELETED
|
@@ -1,300 +0,0 @@
|
|
|
1
|
-
"""Base class for MMO test cases.
|
|
2
|
-
"""
|
|
3
|
-
import logging
|
|
4
|
-
import os
|
|
5
|
-
import unittest
|
|
6
|
-
|
|
7
|
-
from deriva.core import DerivaServer, ErmrestCatalog, get_credential
|
|
8
|
-
from deriva.core.ermrest_model import Schema, Table, Column, Key, ForeignKey, tag, builtin_types
|
|
9
|
-
|
|
10
|
-
logger = logging.getLogger(__name__)
|
|
11
|
-
logger.setLevel(os.getenv('DERIVA_PY_TEST_LOGLEVEL', default=logging.WARNING))
|
|
12
|
-
ermrest_hostname = os.getenv('DERIVA_PY_TEST_HOSTNAME')
|
|
13
|
-
ermrest_catalog_id = os.getenv('DERIVA_PY_TEST_CATALOG')
|
|
14
|
-
catalog = None
|
|
15
|
-
|
|
16
|
-
# baseline annotation doc for `dept` table
|
|
17
|
-
dept_annotations = {
|
|
18
|
-
tag.visible_columns: {
|
|
19
|
-
"compact": [
|
|
20
|
-
["dept_schema", "dept_RID_key"],
|
|
21
|
-
["dept_schema", "dept_dept_no_key"],
|
|
22
|
-
"name"
|
|
23
|
-
],
|
|
24
|
-
"detailed": [
|
|
25
|
-
"RID",
|
|
26
|
-
"RCT",
|
|
27
|
-
{
|
|
28
|
-
"source": "RMT",
|
|
29
|
-
"markdown_name": "Last Modified Time"
|
|
30
|
-
},
|
|
31
|
-
"dept_no",
|
|
32
|
-
"name",
|
|
33
|
-
{
|
|
34
|
-
"source": "street_address",
|
|
35
|
-
"markdown_name": "Number and Street Name"
|
|
36
|
-
},
|
|
37
|
-
"postal_code",
|
|
38
|
-
{
|
|
39
|
-
"sourcekey": "head_count",
|
|
40
|
-
"markdown_name": "Head Count"
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
"display": {
|
|
44
|
-
"wait_for": [
|
|
45
|
-
"personnel"
|
|
46
|
-
],
|
|
47
|
-
"template_engine": "handlebars",
|
|
48
|
-
"markdown_pattern": "{{#each personnel}}{{{this.values.name}}}{{#unless @last}}, {{/unless}}{{/each}}."
|
|
49
|
-
},
|
|
50
|
-
"markdown_name": "Personnel"
|
|
51
|
-
}
|
|
52
|
-
]
|
|
53
|
-
},
|
|
54
|
-
tag.visible_foreign_keys: {
|
|
55
|
-
"*": [
|
|
56
|
-
[
|
|
57
|
-
"person_schema",
|
|
58
|
-
"person_dept_fkey"
|
|
59
|
-
]
|
|
60
|
-
]
|
|
61
|
-
},
|
|
62
|
-
tag.source_definitions: {
|
|
63
|
-
"columns": [
|
|
64
|
-
"dept_no",
|
|
65
|
-
"name",
|
|
66
|
-
"RID",
|
|
67
|
-
"country"
|
|
68
|
-
],
|
|
69
|
-
"sources": {
|
|
70
|
-
"personnel": {
|
|
71
|
-
"source": [
|
|
72
|
-
{
|
|
73
|
-
"inbound": [
|
|
74
|
-
"person_schema",
|
|
75
|
-
"person_dept_fkey"
|
|
76
|
-
]
|
|
77
|
-
},
|
|
78
|
-
"name"
|
|
79
|
-
]
|
|
80
|
-
},
|
|
81
|
-
"head_count": {
|
|
82
|
-
"source": [
|
|
83
|
-
{
|
|
84
|
-
"inbound": [
|
|
85
|
-
"person_schema",
|
|
86
|
-
"person_dept_fkey"
|
|
87
|
-
]
|
|
88
|
-
},
|
|
89
|
-
"RID"
|
|
90
|
-
],
|
|
91
|
-
"entity": False,
|
|
92
|
-
"aggregate": "cnt_d"
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
# baseline annotation doc for `person` table
|
|
99
|
-
person_annotations = {
|
|
100
|
-
tag.visible_columns: {
|
|
101
|
-
"compact": [
|
|
102
|
-
["person_schema", "person_RID_key"],
|
|
103
|
-
"name"
|
|
104
|
-
],
|
|
105
|
-
"detailed": [
|
|
106
|
-
"RID",
|
|
107
|
-
"name",
|
|
108
|
-
["person_schema", "person_dept_fkey"],
|
|
109
|
-
{
|
|
110
|
-
"markdown_name": "Department Name",
|
|
111
|
-
"source": [
|
|
112
|
-
{
|
|
113
|
-
"outbound": [
|
|
114
|
-
"person_schema",
|
|
115
|
-
"person_dept_fkey"
|
|
116
|
-
]
|
|
117
|
-
},
|
|
118
|
-
"name"
|
|
119
|
-
],
|
|
120
|
-
"entity": False
|
|
121
|
-
},
|
|
122
|
-
{
|
|
123
|
-
"sourcekey": "dept_size",
|
|
124
|
-
"markdown_name": "Department Size"
|
|
125
|
-
},
|
|
126
|
-
{
|
|
127
|
-
"sourcekey": "dept_city",
|
|
128
|
-
"markdown_name": "City"
|
|
129
|
-
},
|
|
130
|
-
{
|
|
131
|
-
"markdown_name": "State or Province",
|
|
132
|
-
"source": [
|
|
133
|
-
{
|
|
134
|
-
"outbound": [
|
|
135
|
-
"person_schema",
|
|
136
|
-
"person_dept_fkey"
|
|
137
|
-
]
|
|
138
|
-
},
|
|
139
|
-
"state"
|
|
140
|
-
],
|
|
141
|
-
"entity": False
|
|
142
|
-
},
|
|
143
|
-
]
|
|
144
|
-
},
|
|
145
|
-
tag.source_definitions: {
|
|
146
|
-
"columns": [
|
|
147
|
-
"RID",
|
|
148
|
-
"name",
|
|
149
|
-
"dept"
|
|
150
|
-
],
|
|
151
|
-
"fkeys": [
|
|
152
|
-
["person_schema", "person_dept_fkey"]
|
|
153
|
-
],
|
|
154
|
-
"sources": {
|
|
155
|
-
"dept_size": {
|
|
156
|
-
"source": [
|
|
157
|
-
{
|
|
158
|
-
"outbound": [
|
|
159
|
-
"person_schema",
|
|
160
|
-
"person_dept_fkey"
|
|
161
|
-
]
|
|
162
|
-
},
|
|
163
|
-
{
|
|
164
|
-
"inbound": [
|
|
165
|
-
"person_schema",
|
|
166
|
-
"person_dept_fkey"
|
|
167
|
-
]
|
|
168
|
-
},
|
|
169
|
-
"RID"
|
|
170
|
-
],
|
|
171
|
-
"entity": False,
|
|
172
|
-
"aggregate": "cnt_d"
|
|
173
|
-
},
|
|
174
|
-
"dept_city": {
|
|
175
|
-
"markdown_name": "City",
|
|
176
|
-
"source": [
|
|
177
|
-
{
|
|
178
|
-
"outbound": [
|
|
179
|
-
"person_schema",
|
|
180
|
-
"person_dept_fkey"
|
|
181
|
-
]
|
|
182
|
-
},
|
|
183
|
-
"city"
|
|
184
|
-
],
|
|
185
|
-
"entity": False
|
|
186
|
-
}
|
|
187
|
-
},
|
|
188
|
-
"search-box": {
|
|
189
|
-
"or": [
|
|
190
|
-
{
|
|
191
|
-
"source": "last_name"
|
|
192
|
-
}
|
|
193
|
-
]
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
@unittest.skipUnless(ermrest_hostname, 'ERMrest hostname not defined.')
|
|
200
|
-
class BaseMMOTestCase (unittest.TestCase):
|
|
201
|
-
|
|
202
|
-
@classmethod
|
|
203
|
-
def setUpClass(cls):
|
|
204
|
-
BaseMMOTestCase.setUpCatalog()
|
|
205
|
-
|
|
206
|
-
@classmethod
|
|
207
|
-
def setUpCatalog(cls):
|
|
208
|
-
global catalog
|
|
209
|
-
|
|
210
|
-
# create catalog
|
|
211
|
-
server = DerivaServer('https', ermrest_hostname, credentials=get_credential(ermrest_hostname))
|
|
212
|
-
if ermrest_catalog_id:
|
|
213
|
-
logger.debug(f'Connecting to {ermrest_hostname}/ermrest/catalog/{ermrest_catalog_id}')
|
|
214
|
-
catalog = server.connect_ermrest(ermrest_catalog_id)
|
|
215
|
-
else:
|
|
216
|
-
catalog = server.create_ermrest_catalog()
|
|
217
|
-
logger.debug(f'Created {ermrest_hostname}/ermrest/catalog/{catalog.catalog_id}')
|
|
218
|
-
|
|
219
|
-
# get the model
|
|
220
|
-
model = catalog.getCatalogModel()
|
|
221
|
-
|
|
222
|
-
# drop all schemas (except 'public')
|
|
223
|
-
for sname in [sname for sname in model.schemas if sname != 'public']:
|
|
224
|
-
model.schemas[sname].drop(cascade=True)
|
|
225
|
-
|
|
226
|
-
# recreate schemas
|
|
227
|
-
for sname in ["dept_schema", "person_schema"]:
|
|
228
|
-
model.create_schema(Schema.define(sname))
|
|
229
|
-
|
|
230
|
-
# create `dept` table
|
|
231
|
-
model.schemas["dept_schema"].create_table(
|
|
232
|
-
Table.define(
|
|
233
|
-
'dept',
|
|
234
|
-
column_defs=[
|
|
235
|
-
Column.define('dept_no', builtin_types.int8),
|
|
236
|
-
Column.define('name', builtin_types.text),
|
|
237
|
-
Column.define('street_address', builtin_types.text),
|
|
238
|
-
Column.define('city', builtin_types.text),
|
|
239
|
-
Column.define('state', builtin_types.text),
|
|
240
|
-
Column.define('country', builtin_types.text),
|
|
241
|
-
Column.define('postal_code', builtin_types.int8)
|
|
242
|
-
],
|
|
243
|
-
key_defs=[
|
|
244
|
-
Key.define(['dept_no'])
|
|
245
|
-
],
|
|
246
|
-
annotations=dept_annotations
|
|
247
|
-
)
|
|
248
|
-
)
|
|
249
|
-
|
|
250
|
-
# create `person` table
|
|
251
|
-
model.schemas["person_schema"].create_table(
|
|
252
|
-
Table.define(
|
|
253
|
-
'person',
|
|
254
|
-
column_defs=[
|
|
255
|
-
Column.define('name', builtin_types.text),
|
|
256
|
-
Column.define('dept', builtin_types.int8),
|
|
257
|
-
Column.define('last_name', builtin_types.text)
|
|
258
|
-
],
|
|
259
|
-
fkey_defs=[
|
|
260
|
-
ForeignKey.define(['dept'], "dept_schema", 'dept', ['dept_no'])
|
|
261
|
-
],
|
|
262
|
-
annotations=person_annotations
|
|
263
|
-
)
|
|
264
|
-
)
|
|
265
|
-
|
|
266
|
-
# populate for good measure (though not necessary for current set of tests)
|
|
267
|
-
pbuilder = catalog.getPathBuilder()
|
|
268
|
-
|
|
269
|
-
pbuilder.dept_schema.dept.insert([
|
|
270
|
-
{'dept_no': 1, 'name': 'Dept A', 'street_address': '123 Main St', 'city': 'Anywhere', 'state': 'CA', 'country': 'US', 'postal_code': 98765},
|
|
271
|
-
{'dept_no': 2, 'name': 'Dept B', 'street_address': '777 Oak Ave', 'city': 'Somewhere', 'state': 'NY', 'country': 'US', 'postal_code': 12345}
|
|
272
|
-
])
|
|
273
|
-
|
|
274
|
-
pbuilder.person_schema.person.insert([
|
|
275
|
-
{'name': 'John', 'dept': 1},
|
|
276
|
-
{'name': 'Helena', 'dept': 1},
|
|
277
|
-
{'name': 'Ben', 'dept': 1},
|
|
278
|
-
{'name': 'Sonia', 'dept': 2},
|
|
279
|
-
{'name': 'Rafael', 'dept': 2},
|
|
280
|
-
])
|
|
281
|
-
|
|
282
|
-
@classmethod
|
|
283
|
-
def tearDownClass(cls):
|
|
284
|
-
BaseMMOTestCase.tearDownCatalog()
|
|
285
|
-
|
|
286
|
-
@classmethod
|
|
287
|
-
def tearDownCatalog(cls):
|
|
288
|
-
global catalog
|
|
289
|
-
if not ermrest_catalog_id and isinstance(catalog, ErmrestCatalog) and int(catalog.catalog_id) > 1000:
|
|
290
|
-
# note: the '... > 1000' clause is intended to safeguard against accidental deletion of production catalogs in the usual (lower) range
|
|
291
|
-
catalog.delete_ermrest_catalog(really=True)
|
|
292
|
-
catalog = None
|
|
293
|
-
|
|
294
|
-
def setUp(self):
|
|
295
|
-
# reset annotations to baseline
|
|
296
|
-
assert isinstance(catalog, ErmrestCatalog)
|
|
297
|
-
self.model = catalog.getCatalogModel()
|
|
298
|
-
|
|
299
|
-
def tearDown(self):
|
|
300
|
-
pass
|