synapseclient 2.6.0__tar.gz → 2.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {synapseclient-2.6.0/synapseclient.egg-info → synapseclient-2.7.1}/PKG-INFO +2 -2
- synapseclient-2.7.1/pyproject.toml +14 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/setup.py +51 -48
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/__init__.py +16 -7
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/__main__.py +21 -3
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/client.py +154 -20
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/entity.py +8 -1
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/synapsePythonClient +1 -1
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/table.py +247 -6
- {synapseclient-2.6.0 → synapseclient-2.7.1/synapseclient.egg-info}/PKG-INFO +2 -2
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient.egg-info/SOURCES.txt +1 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient.egg-info/requires.txt +2 -1
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseutils/copy_functions.py +18 -15
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseutils/describe_functions.py +5 -4
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseutils/sync.py +19 -11
- {synapseclient-2.6.0 → synapseclient-2.7.1}/LICENSE +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/MANIFEST.in +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/README.md +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/setup.cfg +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/.synapseConfig +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/activity.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/annotations.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/__init__.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/cache.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/config.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/constants/__init__.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/constants/concrete_types.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/constants/config_file_constants.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/constants/limits.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/credentials/__init__.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/credentials/cached_sessions.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/credentials/cred_data.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/credentials/credential_provider.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/cumulative_transfer_progress.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/dozer.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/exceptions.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/lock.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/logging_setup.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/models/__init__.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/models/custom_json.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/models/dict_object.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/multithread_download/__init__.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/multithread_download/download_threads.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/pool_provider.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/remote_file_storage_wrappers.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/retry.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/sts_transfer.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/upload/__init__.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/upload/multipart_upload.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/upload/upload_functions.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/utils.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/core/version_check.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/evaluation.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/team.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient/wiki.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient.egg-info/dependency_links.txt +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient.egg-info/entry_points.txt +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient.egg-info/not-zip-safe +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseclient.egg-info/top_level.txt +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseutils/__init__.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseutils/migrate_functions.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseutils/monitor.py +0 -0
- {synapseclient-2.6.0 → synapseclient-2.7.1}/synapseutils/walk_functions.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: synapseclient
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.1
|
|
4
4
|
Summary: A client for Synapse, a collaborative compute space that allows scientists to share and analyze data together.
|
|
5
5
|
Home-page: https://www.synapse.org
|
|
6
6
|
Author: The Synapse Engineering Team
|
|
@@ -206,7 +206,7 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
206
206
|
Classifier: Topic :: Software Development :: Libraries
|
|
207
207
|
Classifier: Topic :: Scientific/Engineering
|
|
208
208
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
209
|
-
Requires-Python: >=3.7
|
|
209
|
+
Requires-Python: >=3.7
|
|
210
210
|
Description-Content-Type: text/markdown
|
|
211
211
|
Provides-Extra: pandas
|
|
212
212
|
Provides-Extra: pysftp
|
|
@@ -9,23 +9,30 @@ import json
|
|
|
9
9
|
# check Python version, before we do anything
|
|
10
10
|
if sys.version_info.major < 3 and sys.version_info.minor < 7:
|
|
11
11
|
sys.stderr.write("The Synapse Client for Python requires Python 3.7+\n")
|
|
12
|
-
sys.stderr.write(
|
|
12
|
+
sys.stderr.write(
|
|
13
|
+
"Your Python appears to be version %d.%d.%d\n" % sys.version_info[:3]
|
|
14
|
+
)
|
|
13
15
|
sys.exit(-1)
|
|
14
16
|
|
|
15
17
|
# figure out the version
|
|
16
|
-
__version__ = json.loads(open(
|
|
18
|
+
__version__ = json.loads(open("synapseclient/synapsePythonClient").read())[
|
|
19
|
+
"latestVersion"
|
|
20
|
+
]
|
|
17
21
|
|
|
18
22
|
description = """A client for Synapse, a collaborative compute space
|
|
19
|
-
that allows scientists to share and analyze data together.""".replace(
|
|
23
|
+
that allows scientists to share and analyze data together.""".replace(
|
|
24
|
+
"\n", " "
|
|
25
|
+
)
|
|
20
26
|
|
|
21
27
|
with open("README.md", "r") as fh:
|
|
22
28
|
long_description = fh.read()
|
|
23
29
|
|
|
24
30
|
# make sure not to overwrite existing .synapseConfig with our example one
|
|
25
|
-
data_files
|
|
26
|
-
[(os.path.expanduser(
|
|
27
|
-
if not os.path.exists(os.path.expanduser(
|
|
31
|
+
data_files = (
|
|
32
|
+
[(os.path.expanduser("~"), ["synapseclient/.synapseConfig"])]
|
|
33
|
+
if not os.path.exists(os.path.expanduser("~/.synapseConfig"))
|
|
28
34
|
else []
|
|
35
|
+
)
|
|
29
36
|
|
|
30
37
|
test_deps = [
|
|
31
38
|
"pytest>=5.0.0,<7.0",
|
|
@@ -35,9 +42,10 @@ test_deps = [
|
|
|
35
42
|
]
|
|
36
43
|
|
|
37
44
|
install_requires = [
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
45
|
+
"requests>=2.22.0,<3.0",
|
|
46
|
+
"keyring>=15,<23.5",
|
|
47
|
+
"deprecated>=1.2.4,<2.0",
|
|
48
|
+
"importlib-metadata<5.0",
|
|
41
49
|
]
|
|
42
50
|
|
|
43
51
|
# on Linux specify a cryptography dependency that will not
|
|
@@ -51,72 +59,67 @@ install_requires = [
|
|
|
51
59
|
# if a newer version of cryptography is already installed that is
|
|
52
60
|
# fine we don't want to trigger a downgrade, hence the conditional
|
|
53
61
|
# addition of the versioned dependency.
|
|
54
|
-
if platform.system() ==
|
|
62
|
+
if platform.system() == "Linux":
|
|
55
63
|
try:
|
|
56
64
|
import cryptography # noqa
|
|
65
|
+
|
|
57
66
|
# already installed, don't need to install (or downgrade)
|
|
58
67
|
except ImportError:
|
|
59
|
-
install_requires.append(
|
|
68
|
+
install_requires.append("cryptography<3.4")
|
|
60
69
|
|
|
61
70
|
setuptools.setup(
|
|
62
71
|
# basic
|
|
63
|
-
name=
|
|
72
|
+
name="synapseclient",
|
|
64
73
|
version=__version__,
|
|
65
74
|
packages=setuptools.find_packages(exclude=["tests", "tests.*"]),
|
|
66
|
-
|
|
67
75
|
# requirements
|
|
68
|
-
python_requires=
|
|
76
|
+
python_requires=">=3.7",
|
|
69
77
|
install_requires=install_requires,
|
|
70
78
|
extras_require={
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
':sys_platform=="linux"': [
|
|
79
|
+
"pandas": ["pandas>=0.25.0,<1.5"],
|
|
80
|
+
"pysftp": ["pysftp>=0.2.8,<0.3"],
|
|
81
|
+
"boto3": ["boto3>=1.7.0,<2.0"],
|
|
82
|
+
"docs": ["sphinx>=3.0,<4.0", "sphinx-argparse>=0.2,<0.3"],
|
|
83
|
+
"tests": test_deps,
|
|
84
|
+
':sys_platform=="linux"': ["keyrings.alt==3.1"],
|
|
77
85
|
},
|
|
78
|
-
|
|
79
86
|
# command line
|
|
80
|
-
entry_points={
|
|
81
|
-
'console_scripts': ['synapse = synapseclient.__main__:main']
|
|
82
|
-
},
|
|
83
|
-
|
|
87
|
+
entry_points={"console_scripts": ["synapse = synapseclient.__main__:main"]},
|
|
84
88
|
# data
|
|
85
|
-
package_data={
|
|
89
|
+
package_data={"synapseclient": ["synapsePythonClient", ".synapseConfig"]},
|
|
86
90
|
data_files=data_files,
|
|
87
91
|
zip_safe=False,
|
|
88
|
-
|
|
89
92
|
# test
|
|
90
93
|
tests_require=test_deps,
|
|
91
|
-
|
|
92
94
|
# metadata to display on PyPI
|
|
93
95
|
description=description,
|
|
94
96
|
long_description=long_description,
|
|
95
97
|
long_description_content_type="text/markdown",
|
|
96
|
-
url=
|
|
97
|
-
author=
|
|
98
|
-
author_email=
|
|
99
|
-
license=
|
|
98
|
+
url="https://www.synapse.org",
|
|
99
|
+
author="The Synapse Engineering Team",
|
|
100
|
+
author_email="platform@sagebase.org",
|
|
101
|
+
license="Apache",
|
|
100
102
|
project_urls={
|
|
101
103
|
"Documentation": "https://python-docs.synapse.org",
|
|
102
104
|
"Source Code": "https://github.com/Sage-Bionetworks/synapsePythonClient",
|
|
103
105
|
"Bug Tracker": "https://github.com/Sage-Bionetworks/synapsePythonClient/issues",
|
|
104
106
|
},
|
|
105
107
|
classifiers=[
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
108
|
+
"Development Status :: 5 - Production/Stable",
|
|
109
|
+
"Programming Language :: Python",
|
|
110
|
+
"Programming Language :: Python :: 3.7",
|
|
111
|
+
"Programming Language :: Python :: 3.8",
|
|
112
|
+
"Programming Language :: Python :: 3.9",
|
|
113
|
+
"Programming Language :: Python :: 3.10",
|
|
114
|
+
"Operating System :: MacOS",
|
|
115
|
+
"Operating System :: Microsoft :: Windows",
|
|
116
|
+
"Operating System :: Unix",
|
|
117
|
+
"Operating System :: POSIX :: Linux",
|
|
118
|
+
"Intended Audience :: Developers",
|
|
119
|
+
"Intended Audience :: Science/Research",
|
|
120
|
+
"License :: OSI Approved :: Apache Software License",
|
|
121
|
+
"Topic :: Software Development :: Libraries",
|
|
122
|
+
"Topic :: Scientific/Engineering",
|
|
123
|
+
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
|
124
|
+
],
|
|
122
125
|
)
|
|
@@ -21,6 +21,20 @@ the `web <https://www.synapse.org/>`_. The Python client can also be used from t
|
|
|
21
21
|
If you're just getting started with Synapse, have a look at the Getting Started guides for
|
|
22
22
|
`Synapse <https://docs.synapse.org/articles/getting_started.html>`_.
|
|
23
23
|
|
|
24
|
+
|
|
25
|
+
Next Major Release (3.0.0)
|
|
26
|
+
==========================
|
|
27
|
+
- Support only pandas `>=` 1.5
|
|
28
|
+
- Remove support for Python 3.7 due to its end of life.
|
|
29
|
+
- Remove support for login via passwords for best security practices.
|
|
30
|
+
- There will be major cosmetic changes to the cli such as
|
|
31
|
+
removing all camel case or non-standard single dash long command line interface (cli)
|
|
32
|
+
parameters.
|
|
33
|
+
Example: command line arguments like `-parent` will become
|
|
34
|
+
`--parent`. Commands that support camel case like `--parentId`
|
|
35
|
+
will be changed to `--parent-id`.
|
|
36
|
+
|
|
37
|
+
|
|
24
38
|
Installation
|
|
25
39
|
============
|
|
26
40
|
|
|
@@ -56,11 +70,6 @@ Next, either install the package in the site-packages directory ``python setup.p
|
|
|
56
70
|
python setup.py <install or develop>
|
|
57
71
|
|
|
58
72
|
|
|
59
|
-
Python 2 Support
|
|
60
|
-
----------------
|
|
61
|
-
|
|
62
|
-
synapseclient versions 2.0 and above require Python 3.6 or greater. Python 2.7 is no longer supported.
|
|
63
|
-
|
|
64
73
|
Connecting to Synapse
|
|
65
74
|
=====================
|
|
66
75
|
|
|
@@ -281,7 +290,7 @@ from .core.version_check import check_for_updates, release_notes
|
|
|
281
290
|
from .entity import Entity, Project, Folder, File, Link, DockerRepository
|
|
282
291
|
from .evaluation import Evaluation, Submission, SubmissionStatus
|
|
283
292
|
from .table import Schema, EntityViewSchema, Column, RowSet, Row, as_table_columns, Table, PartialRowset, \
|
|
284
|
-
EntityViewType, build_table, SubmissionViewSchema, MaterializedViewSchema
|
|
293
|
+
EntityViewType, build_table, SubmissionViewSchema, MaterializedViewSchema, Dataset
|
|
285
294
|
from .team import Team, UserProfile, UserGroupHeader, TeamMember
|
|
286
295
|
from .wiki import Wiki
|
|
287
296
|
|
|
@@ -297,7 +306,7 @@ __all__ = [
|
|
|
297
306
|
'Synapse', 'Activity', 'Entity', 'Project', 'Folder', 'File', 'Link', 'DockerRepository', 'Evaluation',
|
|
298
307
|
'Submission', 'SubmissionStatus', 'Schema', 'EntityViewSchema', 'Column', 'Row', 'RowSet', 'Table', 'PartialRowset',
|
|
299
308
|
'Team', 'UserProfile', 'UserGroupHeader', 'TeamMember', 'Wiki', 'Annotations', 'SubmissionViewSchema',
|
|
300
|
-
'MaterializedViewSchema',
|
|
309
|
+
'MaterializedViewSchema', 'Dataset',
|
|
301
310
|
# functions
|
|
302
311
|
'login', 'build_table', 'as_table_columns', 'check_for_updates', 'release_notes',
|
|
303
312
|
# enum
|
|
@@ -525,6 +525,12 @@ def submit(args, syn):
|
|
|
525
525
|
% (submission['id'], submission['entityId'], submission['name'], submission['evaluationId']))
|
|
526
526
|
|
|
527
527
|
|
|
528
|
+
def get_download_list(args, syn):
|
|
529
|
+
"""Download files from the Synapse download cart"""
|
|
530
|
+
manifest_path = syn.get_download_list(downloadLocation=args.downloadLocation)
|
|
531
|
+
syn.logger.info(f"Manifest file: {manifest_path}")
|
|
532
|
+
|
|
533
|
+
|
|
528
534
|
def login(args, syn):
|
|
529
535
|
"""Log in to Synapse, optionally caching credentials"""
|
|
530
536
|
login_with_prompt(syn, args.synapseUser, args.synapsePassword, rememberMe=args.rememberMe, forced=True)
|
|
@@ -711,8 +717,14 @@ def build_parser():
|
|
|
711
717
|
help='Send notifications via Synapse messaging (email) at specific intervals, '
|
|
712
718
|
'on errors and on completion.')
|
|
713
719
|
parser_sync.add_argument('--retries', metavar='INT', type=int, default=4)
|
|
714
|
-
parser_sync.add_argument(
|
|
715
|
-
|
|
720
|
+
parser_sync.add_argument(
|
|
721
|
+
'manifestFile', metavar='FILE', type=argparse.FileType("r"),
|
|
722
|
+
help=(
|
|
723
|
+
'A tsv file with file locations and metadata to be pushed to Synapse. '
|
|
724
|
+
'See https://python-docs.synapse.org/build/html/synapseutils.html#synapseutils.sync.syncToSynapse '
|
|
725
|
+
'for details on the format of a manifest.'
|
|
726
|
+
)
|
|
727
|
+
)
|
|
716
728
|
parser_sync.set_defaults(func=sync)
|
|
717
729
|
|
|
718
730
|
parser_store = subparsers.add_parser('store', # Python 3.2+ would support alias=['store']
|
|
@@ -834,6 +846,12 @@ def build_parser():
|
|
|
834
846
|
help='Do not copy the wiki pages')
|
|
835
847
|
parser_cp.set_defaults(func=copy)
|
|
836
848
|
|
|
849
|
+
parser_get_dl_list = subparsers.add_parser('get-download-list',
|
|
850
|
+
help='Download files from the Synapse download cart')
|
|
851
|
+
parser_get_dl_list.add_argument('--downloadLocation', metavar='path', type=str, default="./",
|
|
852
|
+
help='Directory to download file to [default: %(default)s].')
|
|
853
|
+
parser_get_dl_list.set_defaults(func=get_download_list)
|
|
854
|
+
|
|
837
855
|
parser_associate = subparsers.add_parser('associate',
|
|
838
856
|
help=(
|
|
839
857
|
'Associate local files with the files stored in Synapse so that calls'
|
|
@@ -1032,7 +1050,7 @@ See https://docs.synapse.org/rest/org/sagebionetworks/repo/web/controller/TableE
|
|
|
1032
1050
|
parser_login.add_argument('-u', '--username', dest='synapseUser',
|
|
1033
1051
|
help='Username used to connect to Synapse')
|
|
1034
1052
|
parser_login.add_argument('-p', '--password', dest='synapsePassword',
|
|
1035
|
-
help='Password or api key used to connect to Synapse')
|
|
1053
|
+
help='This will be deprecated. Password or api key used to connect to Synapse.')
|
|
1036
1054
|
parser_login.add_argument('--rememberMe', '--remember-me', dest='rememberMe', action='store_true', default=False,
|
|
1037
1055
|
help='Cache credentials for automatic authentication on future interactions with Synapse')
|
|
1038
1056
|
parser_login.set_defaults(func=login)
|
|
@@ -30,6 +30,7 @@ See also the `Synapse API documentation <https://docs.synapse.org/rest/>`_.
|
|
|
30
30
|
import collections
|
|
31
31
|
import collections.abc
|
|
32
32
|
import configparser
|
|
33
|
+
import csv
|
|
33
34
|
import deprecated
|
|
34
35
|
import errno
|
|
35
36
|
import functools
|
|
@@ -65,7 +66,8 @@ from .entity import Entity, File, Folder, Versionable,\
|
|
|
65
66
|
split_entity_namespaces, is_versionable, is_container, is_synapse_entity
|
|
66
67
|
from synapseclient.core.models.dict_object import DictObject
|
|
67
68
|
from .evaluation import Evaluation, Submission, SubmissionStatus
|
|
68
|
-
from .table import Schema, SchemaBase, Column, TableQueryResult, CsvFileTable,
|
|
69
|
+
from .table import Schema, SchemaBase, Column, TableQueryResult, CsvFileTable, \
|
|
70
|
+
EntityViewSchema, SubmissionViewSchema, Dataset
|
|
69
71
|
from .team import UserProfile, Team, TeamMember, UserGroupHeader
|
|
70
72
|
from .wiki import Wiki, WikiAttachment
|
|
71
73
|
from synapseclient.core import cache, exceptions, utils
|
|
@@ -335,9 +337,9 @@ class Synapse(object):
|
|
|
335
337
|
"""
|
|
336
338
|
Valid combinations of login() arguments:
|
|
337
339
|
|
|
338
|
-
- email/username and password
|
|
340
|
+
- email/username and password (**WILL BE DEPRECATED**)
|
|
339
341
|
|
|
340
|
-
- email/username and apiKey (Base64 encoded string)
|
|
342
|
+
- email/username and apiKey (Base64 encoded string) (**WILL BE DEPRECATED**)
|
|
341
343
|
|
|
342
344
|
- authToken
|
|
343
345
|
|
|
@@ -353,14 +355,14 @@ class Synapse(object):
|
|
|
353
355
|
#. cached credentials from previous `login()` where `rememberMe=True` was passed as a parameter
|
|
354
356
|
|
|
355
357
|
:param email: Synapse user name (or an email address associated with a Synapse account)
|
|
356
|
-
:param password: password
|
|
357
|
-
:param apiKey: Base64 encoded Synapse API key
|
|
358
|
+
:param password: **!!WILL BE DEPRECATED!!** password. Please use authToken (Synapse personal access token)
|
|
359
|
+
:param apiKey: **!!WILL BE DEPRECATED!!** Base64 encoded Synapse API key
|
|
358
360
|
:param sessionToken: **!!DEPRECATED FIELD!!** User's current session token. Using this field will ignore the
|
|
359
361
|
following fields: email, password, apiKey
|
|
360
362
|
:param rememberMe: Whether the authentication information should be cached in your operating system's
|
|
361
363
|
credential storage.
|
|
362
364
|
:param authToken: A bearer authorization token, e.g. a personal access token, can be used in lieu of a
|
|
363
|
-
|
|
365
|
+
password or apiKey
|
|
364
366
|
|
|
365
367
|
**GNOME Keyring** (recommended) or **KWallet** is recommended to be installed for credential storage on
|
|
366
368
|
**Linux** systems.
|
|
@@ -731,9 +733,12 @@ class Synapse(object):
|
|
|
731
733
|
def _check_entity_restrictions(self, bundle, entity, downloadFile):
|
|
732
734
|
restrictionInformation = bundle['restrictionInformation']
|
|
733
735
|
if restrictionInformation['hasUnmetAccessRequirement']:
|
|
734
|
-
warning_message = (
|
|
735
|
-
|
|
736
|
-
|
|
736
|
+
warning_message = (
|
|
737
|
+
'\nThis entity has access restrictions. Please visit the web page for this entity '
|
|
738
|
+
f'(syn.onweb(\"{id_of(entity)}\")). Look for the "Access" label and the lock icon underneath '
|
|
739
|
+
'the file name. Click "Request Access", and then review and fulfill the file '
|
|
740
|
+
'download requirement(s).\n'
|
|
741
|
+
)
|
|
737
742
|
if downloadFile and bundle.get('entityType') not in ('project', 'folder'):
|
|
738
743
|
raise SynapseUnmetAccessRestrictions(warning_message)
|
|
739
744
|
warnings.warn(warning_message)
|
|
@@ -991,7 +996,10 @@ class Synapse(object):
|
|
|
991
996
|
test_entity = syn.store(test_entity, activity=activity)
|
|
992
997
|
|
|
993
998
|
"""
|
|
994
|
-
|
|
999
|
+
# SYNPY-1031: activity must be Activity object or code will fail later
|
|
1000
|
+
if activity:
|
|
1001
|
+
if not isinstance(activity, synapseclient.Activity):
|
|
1002
|
+
raise ValueError("activity should be synapseclient.Activity object")
|
|
995
1003
|
# _before_store hook
|
|
996
1004
|
# give objects a chance to do something before being stored
|
|
997
1005
|
if hasattr(obj, '_before_synapse_store'):
|
|
@@ -1059,7 +1067,7 @@ class Synapse(object):
|
|
|
1059
1067
|
# modified, we want to upload the new version.
|
|
1060
1068
|
# If synapeStore is false then we must upload a ExternalFileHandle
|
|
1061
1069
|
needs_upload = not entity['synapseStore'] \
|
|
1062
|
-
|
|
1070
|
+
or not self.cache.contains(bundle['entity']['dataFileHandleId'], entity['path'])
|
|
1063
1071
|
elif entity.get('dataFileHandleId', None) is not None:
|
|
1064
1072
|
needs_upload = False
|
|
1065
1073
|
else:
|
|
@@ -1355,6 +1363,130 @@ class Synapse(object):
|
|
|
1355
1363
|
"""
|
|
1356
1364
|
return upload_file_handle(self, parent, path, synapseStore, md5, file_size, mimetype)
|
|
1357
1365
|
|
|
1366
|
+
############################################################
|
|
1367
|
+
# Download List #
|
|
1368
|
+
############################################################
|
|
1369
|
+
def clear_download_list(self):
|
|
1370
|
+
"""Clear all files from download list"""
|
|
1371
|
+
self.restDELETE("/download/list")
|
|
1372
|
+
|
|
1373
|
+
def remove_from_download_list(self, list_of_files: typing.List[typing.Dict]) -> int:
|
|
1374
|
+
"""Remove a batch of files from download list
|
|
1375
|
+
|
|
1376
|
+
:param: array of files in the format of a mapping
|
|
1377
|
+
{fileEntityId: synid, versionNumber: version}
|
|
1378
|
+
|
|
1379
|
+
:returns: Number of files removed from download list
|
|
1380
|
+
"""
|
|
1381
|
+
request_body = {"batchToRemove": list_of_files}
|
|
1382
|
+
num_files_removed = self.restPOST(
|
|
1383
|
+
"/download/list/remove",
|
|
1384
|
+
body=json.dumps(request_body)
|
|
1385
|
+
)
|
|
1386
|
+
return num_files_removed
|
|
1387
|
+
|
|
1388
|
+
def _generate_manifest_from_download_list(
|
|
1389
|
+
self, quoteCharacter: str = '"',
|
|
1390
|
+
escapeCharacter: str = "\\",
|
|
1391
|
+
lineEnd: str = os.linesep,
|
|
1392
|
+
separator: str = ",",
|
|
1393
|
+
header: bool = True
|
|
1394
|
+
):
|
|
1395
|
+
"""Creates a download list manifest generation request
|
|
1396
|
+
|
|
1397
|
+
:param quoteCharacter: The character to be used for quoted elements in the resulting file.
|
|
1398
|
+
Defaults to '"'.
|
|
1399
|
+
:param escapeCharacter: The escape character to be used for escaping a separator or quote in the resulting
|
|
1400
|
+
file. Defaults to "\".
|
|
1401
|
+
:param lineEnd: The line feed terminator to be used for the resulting file. Defaults to os.linesep.
|
|
1402
|
+
:param separator: The delimiter to be used for separating entries in the resulting file. Defaults to ",".
|
|
1403
|
+
:param header: Is the first line a header? Defaults to True.
|
|
1404
|
+
|
|
1405
|
+
:returns: Filehandle of download list manifest
|
|
1406
|
+
"""
|
|
1407
|
+
request_body = {
|
|
1408
|
+
"concreteType": "org.sagebionetworks.repo.model.download.DownloadListManifestRequest",
|
|
1409
|
+
"csvTableDescriptor": {
|
|
1410
|
+
"separator": separator,
|
|
1411
|
+
"quoteCharacter": quoteCharacter,
|
|
1412
|
+
"escapeCharacter": escapeCharacter,
|
|
1413
|
+
"lineEnd": lineEnd,
|
|
1414
|
+
"isFirstLineHeader": header
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
return self._waitForAsync(uri="/download/list/manifest/async", request=request_body)
|
|
1418
|
+
|
|
1419
|
+
def get_download_list_manifest(self):
|
|
1420
|
+
"""Get the path of the download list manifest file
|
|
1421
|
+
|
|
1422
|
+
:returns: path of download list manifest file
|
|
1423
|
+
"""
|
|
1424
|
+
manifest = self._generate_manifest_from_download_list()
|
|
1425
|
+
# Get file handle download link
|
|
1426
|
+
file_result = self._getFileHandleDownload(
|
|
1427
|
+
fileHandleId=manifest['resultFileHandleId'],
|
|
1428
|
+
objectId=manifest['resultFileHandleId'],
|
|
1429
|
+
objectType="FileEntity"
|
|
1430
|
+
)
|
|
1431
|
+
# Download the manifest
|
|
1432
|
+
downloaded_path = self._download_from_URL(
|
|
1433
|
+
url=file_result['preSignedURL'],
|
|
1434
|
+
destination="./",
|
|
1435
|
+
fileHandleId=file_result['fileHandleId'],
|
|
1436
|
+
expected_md5=file_result['fileHandle'].get('contentMd5')
|
|
1437
|
+
)
|
|
1438
|
+
return downloaded_path
|
|
1439
|
+
|
|
1440
|
+
def get_download_list(self, downloadLocation: str = None) -> str:
|
|
1441
|
+
"""Download all files from your Synapse download list
|
|
1442
|
+
|
|
1443
|
+
:param downloadLocation: Directory to download files to.
|
|
1444
|
+
|
|
1445
|
+
:returns: manifest file with file paths
|
|
1446
|
+
"""
|
|
1447
|
+
dl_list_path = self.get_download_list_manifest()
|
|
1448
|
+
downloaded_files = []
|
|
1449
|
+
new_manifest_path = f'manifest_{time.time_ns()}.csv'
|
|
1450
|
+
with open(dl_list_path) as manifest_f, \
|
|
1451
|
+
open(new_manifest_path, 'w') as write_obj:
|
|
1452
|
+
|
|
1453
|
+
reader = csv.DictReader(manifest_f)
|
|
1454
|
+
columns = reader.fieldnames
|
|
1455
|
+
columns.extend(["path", "error"])
|
|
1456
|
+
# Write the downloaded paths to a new manifest file
|
|
1457
|
+
writer = csv.DictWriter(write_obj, fieldnames=columns)
|
|
1458
|
+
writer.writeheader()
|
|
1459
|
+
|
|
1460
|
+
for row in reader:
|
|
1461
|
+
# You can add things to the download list that you don't have access to
|
|
1462
|
+
# So there must be a try catch here
|
|
1463
|
+
try:
|
|
1464
|
+
entity = self.get(row['ID'], downloadLocation=downloadLocation)
|
|
1465
|
+
# Must include version number because you can have multiple versions of a
|
|
1466
|
+
# file in the download list
|
|
1467
|
+
downloaded_files.append(
|
|
1468
|
+
{"fileEntityId": row['ID'], "versionNumber": row['versionNumber']}
|
|
1469
|
+
)
|
|
1470
|
+
row['path'] = entity.path
|
|
1471
|
+
row['error'] = ''
|
|
1472
|
+
except Exception:
|
|
1473
|
+
row['path'] = ''
|
|
1474
|
+
row['error'] = 'DOWNLOAD FAILED'
|
|
1475
|
+
self.logger.error("Unable to download file")
|
|
1476
|
+
writer.writerow(row)
|
|
1477
|
+
|
|
1478
|
+
# Don't want to clear all the download list because you can add things
|
|
1479
|
+
# to the download list after initiating this command.
|
|
1480
|
+
# Files that failed to download should not be removed from download list
|
|
1481
|
+
# Remove all files from download list after the entire download is complete.
|
|
1482
|
+
# This is because if download fails midway, we want to return the full manifest
|
|
1483
|
+
self.remove_from_download_list(list_of_files=downloaded_files)
|
|
1484
|
+
|
|
1485
|
+
# Always remove original manifest file
|
|
1486
|
+
os.remove(dl_list_path)
|
|
1487
|
+
|
|
1488
|
+
return new_manifest_path
|
|
1489
|
+
|
|
1358
1490
|
############################################################
|
|
1359
1491
|
# Get / Set Annotations #
|
|
1360
1492
|
############################################################
|
|
@@ -2963,9 +3095,9 @@ class Synapse(object):
|
|
|
2963
3095
|
entityBundleJSON['annotations'] = convert_old_annotation_json(annotations)
|
|
2964
3096
|
|
|
2965
3097
|
related = self._getWithEntityBundle(
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
3098
|
+
entityBundle=entityBundleJSON,
|
|
3099
|
+
entity=submission['entityId'],
|
|
3100
|
+
submission=submission_id, **kwargs)
|
|
2969
3101
|
submission.entity = related
|
|
2970
3102
|
submission.filePath = related.get('path', None)
|
|
2971
3103
|
|
|
@@ -3189,10 +3321,10 @@ class Synapse(object):
|
|
|
3189
3321
|
else:
|
|
3190
3322
|
ValueError("Can't get columns for a %s" % type(x))
|
|
3191
3323
|
|
|
3192
|
-
def create_snapshot_version(self, table: typing.Union[EntityViewSchema, Schema, str, SubmissionViewSchema],
|
|
3324
|
+
def create_snapshot_version(self, table: typing.Union[EntityViewSchema, Schema, str, SubmissionViewSchema, Dataset],
|
|
3193
3325
|
comment: str = None, label: str = None, activity: typing.Union[Activity, str] = None,
|
|
3194
3326
|
wait: bool = True) -> int:
|
|
3195
|
-
"""Create a new Table Version or
|
|
3327
|
+
"""Create a new Table Version, new View version, or new Dataset version.
|
|
3196
3328
|
|
|
3197
3329
|
:param table: The schema of the Table/View, or its ID.
|
|
3198
3330
|
:param comment: Optional snapshot comment.
|
|
@@ -3204,7 +3336,7 @@ class Synapse(object):
|
|
|
3204
3336
|
:return: the snapshot version number if wait=True, None if wait=False
|
|
3205
3337
|
"""
|
|
3206
3338
|
ent = self.get(id_of(table), downloadFile=False)
|
|
3207
|
-
if isinstance(ent, (EntityViewSchema, SubmissionViewSchema)):
|
|
3339
|
+
if isinstance(ent, (EntityViewSchema, SubmissionViewSchema, Dataset)):
|
|
3208
3340
|
result = self._async_table_update(
|
|
3209
3341
|
table,
|
|
3210
3342
|
create_snapshot=True,
|
|
@@ -3323,9 +3455,7 @@ class Synapse(object):
|
|
|
3323
3455
|
|
|
3324
3456
|
:param limit: specify the maximum number of rows to be returned, defaults to None
|
|
3325
3457
|
:param offset: don't return the first n rows, defaults to None
|
|
3326
|
-
:param isConsistent:
|
|
3327
|
-
without waiting for pending writes to complete.
|
|
3328
|
-
Only use this if you know what you're doing.
|
|
3458
|
+
:param isConsistent: (**DEPRECATED**)
|
|
3329
3459
|
|
|
3330
3460
|
For CSV files, there are several parameters to control the format of the resulting file:
|
|
3331
3461
|
|
|
@@ -3351,6 +3481,10 @@ class Synapse(object):
|
|
|
3351
3481
|
if resultsAs.lower() == "rowset":
|
|
3352
3482
|
return TableQueryResult(self, query, **kwargs)
|
|
3353
3483
|
elif resultsAs.lower() == "csv":
|
|
3484
|
+
# TODO: remove isConsistent because it has now been deprecated
|
|
3485
|
+
# from the backend
|
|
3486
|
+
if kwargs.get("isConsistent") is not None:
|
|
3487
|
+
kwargs.pop('isConsistent')
|
|
3354
3488
|
return CsvFileTable.from_table_query(self, query, **kwargs)
|
|
3355
3489
|
else:
|
|
3356
3490
|
raise ValueError("Unknown return type requested from tableQuery: " + str(resultsAs))
|
|
@@ -240,6 +240,12 @@ class Entity(collections.abc.MutableMapping):
|
|
|
240
240
|
if 'annotations' in properties and isinstance(properties['annotations'], collections.abc.Mapping):
|
|
241
241
|
annotations.update(properties['annotations'])
|
|
242
242
|
del properties['annotations']
|
|
243
|
+
|
|
244
|
+
# Re-map `items` to `datasetItems` to avoid namespace conflicts
|
|
245
|
+
# between Dataset schema and the items() builtin method.
|
|
246
|
+
if 'items' in properties:
|
|
247
|
+
properties['datasetItems'] = properties['items']
|
|
248
|
+
del properties['items']
|
|
243
249
|
self.__dict__['properties'].update(properties)
|
|
244
250
|
else:
|
|
245
251
|
raise SynapseMalformedEntityError(
|
|
@@ -731,7 +737,8 @@ ENTITY_TYPES = [
|
|
|
731
737
|
'org.sagebionetworks.repo.model.Folder',
|
|
732
738
|
'org.sagebionetworks.repo.model.Link',
|
|
733
739
|
'org.sagebionetworks.repo.model.Project',
|
|
734
|
-
'org.sagebionetworks.repo.model.table.TableEntity'
|
|
740
|
+
'org.sagebionetworks.repo.model.table.TableEntity',
|
|
741
|
+
'org.sagebionetworks.repo.model.table.Dataset'
|
|
735
742
|
]
|
|
736
743
|
|
|
737
744
|
|