synapseclient 4.4.0__tar.gz → 4.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. {synapseclient-4.4.0/synapseclient.egg-info → synapseclient-4.5.0}/PKG-INFO +5 -4
  2. {synapseclient-4.4.0 → synapseclient-4.5.0}/setup.cfg +4 -3
  3. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/client.py +24 -23
  4. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/credentials/cred_data.py +19 -2
  5. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/credentials/credential_provider.py +3 -1
  6. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/download/__init__.py +2 -0
  7. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/download/download_async.py +9 -2
  8. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/download/download_functions.py +84 -18
  9. synapseclient-4.5.0/synapseclient/core/dozer.py +44 -0
  10. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/retry.py +2 -0
  11. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/transfer_bar.py +1 -1
  12. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/synapsePythonClient +1 -1
  13. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/table.py +43 -4
  14. {synapseclient-4.4.0 → synapseclient-4.5.0/synapseclient.egg-info}/PKG-INFO +5 -4
  15. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient.egg-info/requires.txt +4 -3
  16. synapseclient-4.4.0/synapseclient/core/dozer.py +0 -28
  17. {synapseclient-4.4.0 → synapseclient-4.5.0}/MANIFEST.in +0 -0
  18. {synapseclient-4.4.0 → synapseclient-4.5.0}/README.md +0 -0
  19. {synapseclient-4.4.0 → synapseclient-4.5.0}/pyproject.toml +0 -0
  20. {synapseclient-4.4.0 → synapseclient-4.5.0}/setup.py +0 -0
  21. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/.synapseConfig +0 -0
  22. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/__init__.py +0 -0
  23. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/__main__.py +0 -0
  24. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/activity.py +0 -0
  25. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/annotations.py +0 -0
  26. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/api/__init__.py +0 -0
  27. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/api/annotations.py +0 -0
  28. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/api/configuration_services.py +0 -0
  29. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/api/entity_bundle_services_v2.py +0 -0
  30. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/api/entity_factory.py +0 -0
  31. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/api/entity_services.py +0 -0
  32. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/api/file_services.py +0 -0
  33. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/__init__.py +0 -0
  34. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/async_utils.py +0 -0
  35. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/cache.py +0 -0
  36. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/config.py +0 -0
  37. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/constants/__init__.py +0 -0
  38. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/constants/concrete_types.py +0 -0
  39. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/constants/config_file_constants.py +0 -0
  40. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/constants/limits.py +0 -0
  41. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/constants/method_flags.py +0 -0
  42. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/credentials/__init__.py +0 -0
  43. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/cumulative_transfer_progress.py +0 -0
  44. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/exceptions.py +0 -0
  45. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/lock.py +0 -0
  46. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/logging_setup.py +0 -0
  47. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/models/__init__.py +0 -0
  48. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/models/custom_json.py +0 -0
  49. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/models/dict_object.py +0 -0
  50. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/models/permission.py +0 -0
  51. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/multithread_download/__init__.py +0 -0
  52. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/multithread_download/download_threads.py +0 -0
  53. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/pool_provider.py +0 -0
  54. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/remote_file_storage_wrappers.py +0 -0
  55. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/sts_transfer.py +0 -0
  56. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/upload/__init__.py +0 -0
  57. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/upload/multipart_upload.py +0 -0
  58. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/upload/multipart_upload_async.py +0 -0
  59. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/upload/upload_functions.py +0 -0
  60. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/upload/upload_functions_async.py +0 -0
  61. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/upload/upload_utils.py +0 -0
  62. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/utils.py +0 -0
  63. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/core/version_check.py +0 -0
  64. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/entity.py +0 -0
  65. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/evaluation.py +0 -0
  66. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/__init__.py +0 -0
  67. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/activity.py +0 -0
  68. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/annotations.py +0 -0
  69. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/evaluation.py +0 -0
  70. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/file.py +0 -0
  71. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/folder.py +0 -0
  72. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/mixins/__init__.py +0 -0
  73. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/mixins/access_control.py +0 -0
  74. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/mixins/storable_container.py +0 -0
  75. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/project.py +0 -0
  76. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/__init__.py +0 -0
  77. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/access_control_protocol.py +0 -0
  78. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/activity_protocol.py +0 -0
  79. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/annotations_protocol.py +0 -0
  80. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/file_protocol.py +0 -0
  81. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/folder_protocol.py +0 -0
  82. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/project_protocol.py +0 -0
  83. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/storable_container_protocol.py +0 -0
  84. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/table_protocol.py +0 -0
  85. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/team_protocol.py +0 -0
  86. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/protocols/user_protocol.py +0 -0
  87. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/services/__init__.py +0 -0
  88. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/services/search.py +0 -0
  89. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/services/storable_entity.py +0 -0
  90. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/services/storable_entity_components.py +0 -0
  91. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/table.py +0 -0
  92. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/team.py +0 -0
  93. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/user.py +0 -0
  94. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/models/wiki.py +0 -0
  95. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/services/__init__.py +0 -0
  96. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/services/json_schema.py +0 -0
  97. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/team.py +0 -0
  98. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient/wiki.py +0 -0
  99. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient.egg-info/SOURCES.txt +0 -0
  100. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient.egg-info/dependency_links.txt +0 -0
  101. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient.egg-info/entry_points.txt +0 -0
  102. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient.egg-info/not-zip-safe +0 -0
  103. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseclient.egg-info/top_level.txt +0 -0
  104. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseutils/__init__.py +0 -0
  105. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseutils/copy_functions.py +0 -0
  106. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseutils/describe_functions.py +0 -0
  107. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseutils/migrate_functions.py +0 -0
  108. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseutils/monitor.py +0 -0
  109. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseutils/sync.py +0 -0
  110. {synapseclient-4.4.0 → synapseclient-4.5.0}/synapseutils/walk_functions.py +0 -0
  111. {synapseclient-4.4.0 → synapseclient-4.5.0}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: synapseclient
3
- Version: 4.4.0
3
+ Version: 4.5.0
4
4
  Summary: A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate.
5
5
  Home-page: https://www.synapse.org
6
6
  Author: The Synapse Engineering Team
@@ -32,9 +32,9 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
32
32
  Requires-Dist: requests<3.0,>=2.22.0
33
33
  Requires-Dist: urllib3<2,>=1.26.18
34
34
  Requires-Dist: deprecated<2.0,>=1.2.4
35
- Requires-Dist: opentelemetry-api~=1.21.0
36
- Requires-Dist: opentelemetry-sdk~=1.21.0
37
- Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.21.0
35
+ Requires-Dist: opentelemetry-api>=1.21.0
36
+ Requires-Dist: opentelemetry-sdk>=1.21.0
37
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.21.0
38
38
  Requires-Dist: nest-asyncio~=1.6.0
39
39
  Requires-Dist: asyncio-atexit~=1.0.1
40
40
  Requires-Dist: httpx~=0.27.0
@@ -67,6 +67,7 @@ Requires-Dist: pytest-cov~=4.1.0; extra == "tests"
67
67
  Requires-Dist: pandas<3.0,>=1.5; extra == "tests"
68
68
  Provides-Extra: pandas
69
69
  Requires-Dist: pandas<3.0,>=1.5; extra == "pandas"
70
+ Requires-Dist: numpy<2.0.0; extra == "pandas"
70
71
  Provides-Extra: pysftp
71
72
  Requires-Dist: pysftp<0.3,>=0.2.8; extra == "pysftp"
72
73
  Provides-Extra: boto3
@@ -41,9 +41,9 @@ install_requires =
41
41
  requests>=2.22.0,<3.0
42
42
  urllib3>=1.26.18,<2
43
43
  deprecated>=1.2.4,<2.0
44
- opentelemetry-api~=1.21.0
45
- opentelemetry-sdk~=1.21.0
46
- opentelemetry-exporter-otlp-proto-http~=1.21.0
44
+ opentelemetry-api>=1.21.0
45
+ opentelemetry-sdk>=1.21.0
46
+ opentelemetry-exporter-otlp-proto-http>=1.21.0
47
47
  nest-asyncio~=1.6.0
48
48
  asyncio-atexit~=1.0.1
49
49
  httpx~=0.27.0
@@ -89,6 +89,7 @@ tests =
89
89
  pandas>=1.5,<3.0
90
90
  pandas =
91
91
  pandas>=1.5,<3.0
92
+ numpy<2.0.0
92
93
  pysftp =
93
94
  pysftp>=0.2.8,<0.3
94
95
  boto3 =
@@ -277,6 +277,7 @@ class Synapse(object):
277
277
  requests_session_async_synapse: httpx.AsyncClient = None,
278
278
  requests_session_storage: httpx.Client = None,
279
279
  asyncio_event_loop: asyncio.AbstractEventLoop = None,
280
+ cache_client: bool = True,
280
281
  ) -> "Synapse":
281
282
  """
282
283
  Initialize Synapse object
@@ -300,6 +301,9 @@ class Synapse(object):
300
301
  asyncio_event_loop: The event loop that is going to be used while executing
301
302
  this code. This is optional and only used when you are manually
302
303
  specifying an async HTTPX client.
304
+ cache_client: Whether to cache the Synapse client object in the Synapse module. Defaults to True.
305
+ When set to True anywhere a `Synapse` object is optional you do not need to pass an
306
+ instance of `Synapse` to that function, method, or class.
303
307
 
304
308
  Raises:
305
309
  ValueError: Warn for non-boolean debug value.
@@ -376,6 +380,10 @@ class Synapse(object):
376
380
  if debug is None:
377
381
  debug = config_debug if config_debug is not None else DEBUG_DEFAULT
378
382
 
383
+ if not isinstance(debug, bool):
384
+ raise ValueError("debug must be set to a bool (either True or False)")
385
+ self.debug = debug
386
+
379
387
  self.cache = cache.Cache(cache_root_dir)
380
388
  self._sts_token_store = sts_transfer.StsTokenStore()
381
389
 
@@ -389,16 +397,12 @@ class Synapse(object):
389
397
  }
390
398
  self.credentials = None
391
399
 
392
- if not isinstance(debug, bool):
393
- raise ValueError("debug must be set to a bool (either True or False)")
394
- self.debug = debug
395
-
396
400
  self.silent = silent
397
401
  self._init_logger() # initializes self.logger
398
402
 
399
403
  self.skip_checks = skip_checks
400
404
 
401
- self.table_query_sleep = 2
405
+ self.table_query_sleep = 0.2 # Sleep for 200ms between table query retries
402
406
  self.table_query_backoff = 1.1
403
407
  self.table_query_max_sleep = 20
404
408
  self.table_query_timeout = 600 # in seconds
@@ -411,6 +415,8 @@ class Synapse(object):
411
415
  self._parallel_file_transfer_semaphore = {}
412
416
  self.use_boto_sts_transfers = transfer_config["use_boto_sts"]
413
417
  self._parts_transfered_counter = 0
418
+ if cache_client:
419
+ Synapse.set_client(synapse_client=self)
414
420
 
415
421
  def _get_requests_session_async_synapse(
416
422
  self, asyncio_event_loop: asyncio.AbstractEventLoop
@@ -690,10 +696,14 @@ class Synapse(object):
690
696
 
691
697
  # Update endpoints if we get redirected
692
698
  if not skip_checks:
693
- response = self._requests_session.get(
694
- endpoints[point],
695
- allow_redirects=False,
696
- headers=synapseclient.USER_AGENT,
699
+ response = with_retry(
700
+ lambda point=point: self._requests_session.get(
701
+ endpoints[point],
702
+ allow_redirects=False,
703
+ headers=synapseclient.USER_AGENT,
704
+ ),
705
+ verbose=self.debug,
706
+ **STANDARD_RETRY_PARAMS,
697
707
  )
698
708
  if response.status_code == 301:
699
709
  endpoints[point] = response.headers["location"]
@@ -708,7 +718,6 @@ class Synapse(object):
708
718
  email: str = None,
709
719
  silent: bool = False,
710
720
  authToken: str = None,
711
- cache_client: bool = True,
712
721
  ) -> None:
713
722
  """
714
723
  Valid combinations of login() arguments:
@@ -729,9 +738,6 @@ class Synapse(object):
729
738
  authToken: A bearer authorization token, e.g. a
730
739
  [personal access token](https://python-docs.synapse.org/tutorials/authentication/).
731
740
  silent: Defaults to False. Suppresses the "Welcome ...!" message.
732
- cache_client: Whether to cache the Synapse client object in the Synapse module. Defaults to True.
733
- When set to True anywhere a `Synapse` object is optional you do not need to pass an
734
- instance of `Synapse` to that function, method, or class.
735
741
 
736
742
  Example: Logging in
737
743
  Using an auth token:
@@ -770,17 +776,9 @@ class Synapse(object):
770
776
  raise SynapseNoCredentialsError("No credentials provided.")
771
777
 
772
778
  if not silent:
773
- profile = self.getUserProfile()
774
- display_name = (
775
- profile["displayName"]
776
- if "displayName" in profile
777
- else self.credentials.username
778
- )
779
+ display_name = self.credentials.displayname or self.credentials.username
779
780
  self.logger.info(f"Welcome, {display_name}!\n")
780
781
 
781
- if cache_client:
782
- Synapse.set_client(self)
783
-
784
782
  @deprecated(
785
783
  version="4.4.0",
786
784
  reason="To be removed in 5.0.0. "
@@ -4935,6 +4933,7 @@ class Synapse(object):
4935
4933
  destination=os.path.join(
4936
4934
  cache_dir, str(wiki.markdownFileHandleId) + ".md"
4937
4935
  ),
4936
+ synapse_client=self,
4938
4937
  ),
4939
4938
  syn=self,
4940
4939
  )
@@ -5046,6 +5045,7 @@ class Synapse(object):
5046
5045
  # Tables #
5047
5046
  ############################################################
5048
5047
 
5048
+ @tracer.start_as_current_span("Synapse::_waitForAsync")
5049
5049
  def _waitForAsync(self, uri, request, endpoint=None):
5050
5050
  if endpoint is None:
5051
5051
  endpoint = self.repoEndpoint
@@ -5077,7 +5077,7 @@ class Synapse(object):
5077
5077
  sleep = min(
5078
5078
  self.table_query_max_sleep, sleep * self.table_query_backoff
5079
5079
  )
5080
- doze(sleep)
5080
+ doze(sleep, trace_span_name="Synapse::doze_for_async")
5081
5081
  else:
5082
5082
  break
5083
5083
  else:
@@ -5603,6 +5603,7 @@ class Synapse(object):
5603
5603
  synapse_id=extract_synapse_id_from_query(query),
5604
5604
  entity_type="TableEntity",
5605
5605
  destination=os.path.join(download_dir, filename),
5606
+ synapse_client=self,
5606
5607
  ),
5607
5608
  syn=self,
5608
5609
  )
@@ -61,11 +61,14 @@ class SynapseAuthTokenCredentials(SynapseCredentials):
61
61
  # ValueError if the split string is not base64 encoded or if the decoded base64 is not json
62
62
  pass
63
63
 
64
- def __init__(self, token, username=None):
64
+ def __init__(
65
+ self, token: str, username: str = None, displayname: str = None
66
+ ) -> None:
65
67
  self._validate_token(token)
66
68
 
67
69
  self._token = token
68
70
  self.username = username
71
+ self.displayname = displayname
69
72
 
70
73
  @property
71
74
  def username(self) -> str:
@@ -76,6 +79,15 @@ class SynapseAuthTokenCredentials(SynapseCredentials):
76
79
  def username(self, username: str) -> None:
77
80
  self._username = username
78
81
 
82
+ @property
83
+ def displayname(self) -> str:
84
+ """The displayname associated with this token."""
85
+ return self._displayname
86
+
87
+ @displayname.setter
88
+ def displayname(self, displayname: str) -> None:
89
+ self._displayname = displayname
90
+
79
91
  @property
80
92
  def secret(self) -> str:
81
93
  """The bearer token."""
@@ -86,7 +98,12 @@ class SynapseAuthTokenCredentials(SynapseCredentials):
86
98
  return r
87
99
 
88
100
  def __repr__(self):
89
- return f"SynapseAuthTokenCredentials(username='{self.username}', token='{self.secret}')"
101
+ return (
102
+ f"SynapseAuthTokenCredentials("
103
+ f"username='{self.username}', "
104
+ f"displayname='{self.displayname}', "
105
+ f"token='{self.secret}')"
106
+ )
90
107
 
91
108
 
92
109
  # a class that just contains args passed form synapse client login
@@ -77,6 +77,7 @@ class SynapseCredentialsProvider(metaclass=abc.ABCMeta):
77
77
  profile = syn.restGET("/userProfile", auth=credentials)
78
78
  profile_username = profile.get("userName")
79
79
  profile_emails = profile.get("emails", [])
80
+ profile_displayname = profile.get("displayName")
80
81
 
81
82
  if username and (
82
83
  username != profile_username and username not in profile_emails
@@ -88,8 +89,9 @@ class SynapseCredentialsProvider(metaclass=abc.ABCMeta):
88
89
  "username/email and auth_token both provided but username does not "
89
90
  "match token profile"
90
91
  )
91
-
92
92
  credentials.username = profile_username
93
+ credentials.displayname = profile_displayname
94
+
93
95
  return credentials
94
96
 
95
97
  return None
@@ -6,6 +6,7 @@ from .download_async import (
6
6
  PresignedUrlInfo,
7
7
  PresignedUrlProvider,
8
8
  _MultithreadedDownloader,
9
+ _pre_signed_url_expiration_time,
9
10
  download_file,
10
11
  )
11
12
  from .download_functions import (
@@ -32,4 +33,5 @@ __all__ = [
32
33
  "PresignedUrlInfo",
33
34
  "PresignedUrlProvider",
34
35
  "_MultithreadedDownloader",
36
+ "_pre_signed_url_expiration_time",
35
37
  ]
@@ -274,7 +274,12 @@ def _get_file_size(
274
274
  if otel_context:
275
275
  context.attach(otel_context)
276
276
  with syn._requests_session_storage.stream("GET", url) as response:
277
- _raise_for_status_httpx(response=response, logger=syn.logger, verbose=debug)
277
+ _raise_for_status_httpx(
278
+ response=response,
279
+ logger=syn.logger,
280
+ verbose=debug,
281
+ read_response_content=False,
282
+ )
278
283
  return int(response.headers["Content-Length"])
279
284
 
280
285
 
@@ -314,7 +319,9 @@ class _MultithreadedDownloader:
314
319
  syn=self._syn, url=url_info.url, debug=self._download_request.debug
315
320
  )
316
321
  self._progress_bar = get_or_create_download_progress_bar(
317
- file_size=file_size, postfix=self._download_request.object_id
322
+ file_size=file_size,
323
+ postfix=self._download_request.object_id,
324
+ synapse_client=self._syn,
318
325
  )
319
326
  self._prep_file()
320
327
 
@@ -1,6 +1,7 @@
1
1
  """This module handles the various ways that a user can download a file to Synapse."""
2
2
 
3
3
  import asyncio
4
+ import datetime
4
5
  import errno
5
6
  import hashlib
6
7
  import os
@@ -14,7 +15,10 @@ from tqdm import tqdm
14
15
  from tqdm.contrib.logging import logging_redirect_tqdm
15
16
 
16
17
  from synapseclient.api.configuration_services import get_client_authenticated_s3_profile
17
- from synapseclient.api.file_services import get_file_handle_for_download_async
18
+ from synapseclient.api.file_services import (
19
+ get_file_handle_for_download,
20
+ get_file_handle_for_download_async,
21
+ )
18
22
  from synapseclient.core import exceptions, sts_transfer, utils
19
23
  from synapseclient.core.constants import concrete_types
20
24
  from synapseclient.core.constants.method_flags import (
@@ -25,6 +29,8 @@ from synapseclient.core.constants.method_flags import (
25
29
  from synapseclient.core.download import (
26
30
  SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE,
27
31
  DownloadRequest,
32
+ PresignedUrlProvider,
33
+ _pre_signed_url_expiration_time,
28
34
  download_file,
29
35
  )
30
36
  from synapseclient.core.exceptions import (
@@ -166,6 +172,7 @@ async def download_file_entity(
166
172
  synapse_id=object_id,
167
173
  entity_type=object_type,
168
174
  destination=download_path,
175
+ synapse_client=client,
169
176
  )
170
177
 
171
178
  if download_path is None or not os.path.exists(download_path):
@@ -274,6 +281,7 @@ async def download_file_entity_model(
274
281
  synapse_id=object_id,
275
282
  entity_type=object_type,
276
283
  destination=download_path,
284
+ synapse_client=client,
277
285
  )
278
286
 
279
287
  if download_path is None or not os.path.exists(download_path):
@@ -416,7 +424,7 @@ async def download_by_file_handle(
416
424
  )
417
425
 
418
426
  progress_bar = get_or_create_download_progress_bar(
419
- file_size=1, postfix=synapse_id
427
+ file_size=1, postfix=synapse_id, synapse_client=syn
420
428
  )
421
429
  loop = asyncio.get_running_loop()
422
430
  downloaded_path = await loop.run_in_executor(
@@ -440,7 +448,7 @@ async def download_by_file_handle(
440
448
  and concrete_type == concrete_types.S3_FILE_HANDLE
441
449
  ):
442
450
  progress_bar = get_or_create_download_progress_bar(
443
- file_size=1, postfix=synapse_id
451
+ file_size=1, postfix=synapse_id, synapse_client=syn
444
452
  )
445
453
 
446
454
  def download_fn(
@@ -496,13 +504,16 @@ async def download_by_file_handle(
496
504
  else:
497
505
  loop = asyncio.get_running_loop()
498
506
  progress_bar = get_or_create_download_progress_bar(
499
- file_size=1, postfix=synapse_id
507
+ file_size=1, postfix=synapse_id, synapse_client=syn
500
508
  )
509
+
501
510
  downloaded_path = await loop.run_in_executor(
502
511
  syn._get_thread_pool_executor(asyncio_event_loop=loop),
503
512
  lambda: download_from_url(
504
513
  url=file_handle_result["preSignedURL"],
505
514
  destination=destination,
515
+ entity_id=synapse_id,
516
+ file_handle_associate_type=entity_type,
506
517
  file_handle_id=file_handle["id"],
507
518
  expected_md5=file_handle.get("contentMd5"),
508
519
  progress_bar=progress_bar,
@@ -608,6 +619,8 @@ async def download_from_url_multi_threaded(
608
619
  def download_from_url(
609
620
  url: str,
610
621
  destination: str,
622
+ entity_id: Optional[str],
623
+ file_handle_associate_type: Optional[str],
611
624
  file_handle_id: Optional[str] = None,
612
625
  expected_md5: Optional[str] = None,
613
626
  progress_bar: Optional[tqdm] = None,
@@ -620,6 +633,11 @@ def download_from_url(
620
633
  Arguments:
621
634
  url: The source of download
622
635
  destination: The destination on local file system
636
+ entity_id: The id of the Synapse object that uses the FileHandle
637
+ e.g. "syn123"
638
+ file_handle_associate_type: The type of the Synapse object that uses the
639
+ FileHandle e.g. "FileEntity". Any of
640
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/file/FileHandleAssociateType.html>
623
641
  file_handle_id: Optional. If given, the file will be given a temporary name that includes the file
624
642
  handle id which allows resuming partial downloads of the same file from previous
625
643
  sessions
@@ -717,21 +735,69 @@ def download_from_url(
717
735
  else None
718
736
  )
719
737
 
720
- response = with_retry(
721
- lambda url=url, range_header=range_header, auth=auth: client._requests_session.get(
722
- url=url,
723
- headers=client._generate_headers(range_header),
724
- stream=True,
725
- allow_redirects=False,
726
- auth=auth,
727
- ),
728
- verbose=client.debug,
729
- **STANDARD_RETRY_PARAMS,
730
- )
731
738
  try:
739
+ url_has_expiration = "Expires" in urllib_urlparse.urlparse(url).query
740
+ url_is_expired = False
741
+ if url_has_expiration:
742
+ url_is_expired = datetime.datetime.now(
743
+ tz=datetime.timezone.utc
744
+ ) + PresignedUrlProvider._TIME_BUFFER >= _pre_signed_url_expiration_time(
745
+ url
746
+ )
747
+ if url_is_expired:
748
+ response = get_file_handle_for_download(
749
+ file_handle_id=file_handle_id,
750
+ synapse_id=entity_id,
751
+ entity_type=file_handle_associate_type,
752
+ synapse_client=client,
753
+ )
754
+ url = response["preSignedURL"]
755
+ response = with_retry(
756
+ lambda url=url, range_header=range_header, auth=auth: client._requests_session.get(
757
+ url=url,
758
+ headers=client._generate_headers(range_header),
759
+ stream=True,
760
+ allow_redirects=False,
761
+ auth=auth,
762
+ ),
763
+ verbose=client.debug,
764
+ **STANDARD_RETRY_PARAMS,
765
+ )
732
766
  exceptions._raise_for_status(response, verbose=client.debug)
733
767
  except SynapseHTTPError as err:
734
- if err.response.status_code == 404:
768
+ if err.response.status_code == 403:
769
+ url_has_expiration = (
770
+ "Expires" in urllib_urlparse.urlparse(url).query
771
+ )
772
+ url_is_expired = False
773
+ if url_has_expiration:
774
+ url_is_expired = datetime.datetime.now(
775
+ tz=datetime.timezone.utc
776
+ ) + PresignedUrlProvider._TIME_BUFFER >= _pre_signed_url_expiration_time(
777
+ url
778
+ )
779
+ if url_is_expired:
780
+ response = get_file_handle_for_download(
781
+ file_handle_id=file_handle_id,
782
+ synapse_id=entity_id,
783
+ entity_type=file_handle_associate_type,
784
+ synapse_client=client,
785
+ )
786
+ refreshed_url = response["preSignedURL"]
787
+ response = with_retry(
788
+ lambda url=refreshed_url, range_header=range_header, auth=auth: client._requests_session.get(
789
+ url=url,
790
+ headers=client._generate_headers(range_header),
791
+ stream=True,
792
+ allow_redirects=False,
793
+ auth=auth,
794
+ ),
795
+ verbose=client.debug,
796
+ **STANDARD_RETRY_PARAMS,
797
+ )
798
+ else:
799
+ raise
800
+ elif err.response.status_code == 404:
735
801
  raise SynapseError(f"Could not download the file at {url}") from err
736
802
  elif (
737
803
  err.response.status_code == 416
@@ -743,8 +809,8 @@ def download_from_url(
743
809
  # If it fails the user can retry with another download.
744
810
  shutil.move(temp_destination, destination)
745
811
  break
746
- raise
747
-
812
+ else:
813
+ raise
748
814
  # handle redirects
749
815
  if response.status_code in [301, 302, 303, 307, 308]:
750
816
  url = response.headers["location"]
@@ -0,0 +1,44 @@
1
+ """
2
+ Created on Sep 21, 2017
3
+
4
+ @author: bhoff
5
+
6
+ sleep while checking registered _listeners
7
+ """
8
+ import time
9
+
10
+ from opentelemetry import trace
11
+
12
+ tracer = trace.get_tracer("synapseclient")
13
+
14
+ _listeners = []
15
+
16
+
17
+ def add_listener(listener):
18
+ if not callable(listener):
19
+ raise ValueError("listener is not callable")
20
+ _listeners.append(listener)
21
+
22
+
23
+ def clear_listeners():
24
+ del _listeners[:]
25
+
26
+
27
+ def doze(
28
+ secs: float,
29
+ listener_check_interval_secs: float = 0.1,
30
+ trace_span_name: str = "doze",
31
+ ) -> None:
32
+ """Sleep for a given number of seconds while checking registered listeners.
33
+
34
+ Arguments:
35
+ secs: the number of seconds to sleep
36
+ listener_check_interval_secs: the interval at which to check the listeners
37
+ trace_span_name: the name of the trace span
38
+ """
39
+ with tracer.start_as_current_span(name=trace_span_name):
40
+ end_time = time.time() + secs
41
+ while time.time() < end_time:
42
+ for listener in _listeners:
43
+ listener()
44
+ time.sleep(listener_check_interval_secs)
@@ -69,6 +69,8 @@ RETRYABLE_CONNECTION_EXCEPTIONS = [
69
69
  "TimeoutException",
70
70
  "ConnectError",
71
71
  "ConnectTimeout",
72
+ # SSL Specific exceptions:
73
+ "SSLZeroReturnError",
72
74
  ]
73
75
 
74
76
  DEBUG_EXCEPTION = "calling %s resulted in an Exception"
@@ -85,7 +85,7 @@ def shared_download_progress_bar(
85
85
 
86
86
  syn = Synapse.get_client(synapse_client=synapse_client)
87
87
  with logging_redirect_tqdm(loggers=[syn.logger]):
88
- get_or_create_download_progress_bar(file_size=file_size)
88
+ get_or_create_download_progress_bar(file_size=file_size, synapse_client=syn)
89
89
  try:
90
90
  yield
91
91
  finally:
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "client": "synapsePythonClient",
3
- "latestVersion":"4.4.0",
3
+ "latestVersion":"4.5.0",
4
4
  "blacklist": [
5
5
  "0.0.0",
6
6
  "0.4.1",
@@ -407,7 +407,34 @@ def _csv_to_pandas_df(
407
407
  list_columns=None,
408
408
  rowIdAndVersionInIndex=True,
409
409
  dtype=None,
410
+ **kwargs,
410
411
  ):
412
+ """
413
+ Convert a csv file to a pandas dataframe
414
+
415
+ Arguments:
416
+ filepath: The path to the file.
417
+ separator: The separator for the file, Defaults to `DEFAULT_SEPARATOR`.
418
+ quote_char: The quote character for the file,
419
+ Defaults to `DEFAULT_QUOTE_CHARACTER`.
420
+ escape_char: The escape character for the file,
421
+ Defaults to `DEFAULT_ESCAPSE_CHAR`.
422
+ contain_headers: Whether the file contains headers,
423
+ Defaults to `True`.
424
+ lines_to_skip: The number of lines to skip at the beginning of the file,
425
+ Defaults to `0`.
426
+ date_columns: The names of the date columns in the file
427
+ list_columns: The names of the list columns in the file
428
+ rowIdAndVersionInIndex: Whether the file contains rowId and
429
+ version in the index, Defaults to `True`.
430
+ dtype: The data type for the file, Defaults to `None`.
431
+ **kwargs: Additional keyword arguments to pass to pandas.read_csv. See
432
+ https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
433
+ for complete list of supported arguments.
434
+
435
+ Returns:
436
+ A pandas dataframe
437
+ """
411
438
  test_import_pandas()
412
439
  import pandas as pd
413
440
 
@@ -426,6 +453,7 @@ def _csv_to_pandas_df(
426
453
  escapechar=escape_char,
427
454
  header=0 if contain_headers else None,
428
455
  skiprows=lines_to_skip,
456
+ **kwargs,
429
457
  )
430
458
  # parse date columns if exists
431
459
  if date_columns:
@@ -2445,14 +2473,23 @@ class CsvFileTable(TableAbstractBaseClass):
2445
2473
  self.etag = upload_to_table_result["etag"]
2446
2474
  return self
2447
2475
 
2448
- def asDataFrame(self, rowIdAndVersionInIndex=True, convert_to_datetime=False):
2476
+ def asDataFrame(
2477
+ self,
2478
+ rowIdAndVersionInIndex: bool = True,
2479
+ convert_to_datetime: bool = False,
2480
+ **kwargs,
2481
+ ):
2449
2482
  """Convert query result to a Pandas DataFrame.
2450
2483
 
2451
2484
  Arguments:
2452
- rowIdAndVersionInIndex: Make the dataframe index consist of the row_id and row_version
2453
- (and row_etag if it exists)
2485
+ rowIdAndVersionInIndex: Make the dataframe index consist of the
2486
+ row_id and row_version (and row_etag if it exists)
2454
2487
  convert_to_datetime: If set to True, will convert all Synapse DATE columns from UNIX timestamp
2455
2488
  integers into UTC datetime objects
2489
+ kwargs: Additional keyword arguments to pass to
2490
+ pandas.read_csv via _csv_to_pandas_df. See
2491
+ https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
2492
+ for complete list of supported arguments.
2456
2493
 
2457
2494
  Returns:
2458
2495
  A Pandas dataframe with results
@@ -2491,8 +2528,10 @@ class CsvFileTable(TableAbstractBaseClass):
2491
2528
  list_columns=list_columns,
2492
2529
  rowIdAndVersionInIndex=rowIdAndVersionInIndex,
2493
2530
  dtype=dtype,
2531
+ **kwargs,
2494
2532
  )
2495
- except pd.parser.CParserError:
2533
+
2534
+ except pd.errors.ParserError:
2496
2535
  return pd.DataFrame()
2497
2536
 
2498
2537
  def asRowSet(self):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: synapseclient
3
- Version: 4.4.0
3
+ Version: 4.5.0
4
4
  Summary: A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate.
5
5
  Home-page: https://www.synapse.org
6
6
  Author: The Synapse Engineering Team
@@ -32,9 +32,9 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
32
32
  Requires-Dist: requests<3.0,>=2.22.0
33
33
  Requires-Dist: urllib3<2,>=1.26.18
34
34
  Requires-Dist: deprecated<2.0,>=1.2.4
35
- Requires-Dist: opentelemetry-api~=1.21.0
36
- Requires-Dist: opentelemetry-sdk~=1.21.0
37
- Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.21.0
35
+ Requires-Dist: opentelemetry-api>=1.21.0
36
+ Requires-Dist: opentelemetry-sdk>=1.21.0
37
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.21.0
38
38
  Requires-Dist: nest-asyncio~=1.6.0
39
39
  Requires-Dist: asyncio-atexit~=1.0.1
40
40
  Requires-Dist: httpx~=0.27.0
@@ -67,6 +67,7 @@ Requires-Dist: pytest-cov~=4.1.0; extra == "tests"
67
67
  Requires-Dist: pandas<3.0,>=1.5; extra == "tests"
68
68
  Provides-Extra: pandas
69
69
  Requires-Dist: pandas<3.0,>=1.5; extra == "pandas"
70
+ Requires-Dist: numpy<2.0.0; extra == "pandas"
70
71
  Provides-Extra: pysftp
71
72
  Requires-Dist: pysftp<0.3,>=0.2.8; extra == "pysftp"
72
73
  Provides-Extra: boto3
@@ -1,9 +1,9 @@
1
1
  requests<3.0,>=2.22.0
2
2
  urllib3<2,>=1.26.18
3
3
  deprecated<2.0,>=1.2.4
4
- opentelemetry-api~=1.21.0
5
- opentelemetry-sdk~=1.21.0
6
- opentelemetry-exporter-otlp-proto-http~=1.21.0
4
+ opentelemetry-api>=1.21.0
5
+ opentelemetry-sdk>=1.21.0
6
+ opentelemetry-exporter-otlp-proto-http>=1.21.0
7
7
  nest-asyncio~=1.6.0
8
8
  asyncio-atexit~=1.0.1
9
9
  httpx~=0.27.0
@@ -39,6 +39,7 @@ markdown-include~=0.8.1
39
39
 
40
40
  [pandas]
41
41
  pandas<3.0,>=1.5
42
+ numpy<2.0.0
42
43
 
43
44
  [pysftp]
44
45
  pysftp<0.3,>=0.2.8
@@ -1,28 +0,0 @@
1
- """
2
- Created on Sep 21, 2017
3
-
4
- @author: bhoff
5
-
6
- sleep while checking registered _listeners
7
- """
8
- import time
9
-
10
- _listeners = []
11
-
12
-
13
- def add_listener(listener):
14
- if not callable(listener):
15
- raise ValueError("listener is not callable")
16
- _listeners.append(listener)
17
-
18
-
19
- def clear_listeners():
20
- del _listeners[:]
21
-
22
-
23
- def doze(secs, listener_check_interval_secs=0.1):
24
- end_time = time.time() + secs
25
- while time.time() < end_time:
26
- for listener in _listeners:
27
- listener()
28
- time.sleep(listener_check_interval_secs)
File without changes
File without changes
File without changes