synapseclient 4.6.0__tar.gz → 4.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. {synapseclient-4.6.0/synapseclient.egg-info → synapseclient-4.7.0}/PKG-INFO +3 -3
  2. {synapseclient-4.6.0 → synapseclient-4.7.0}/setup.cfg +1 -1
  3. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/api/__init__.py +15 -0
  4. synapseclient-4.7.0/synapseclient/api/agent_services.py +189 -0
  5. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/client.py +11 -14
  6. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/constants/concrete_types.py +3 -0
  7. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/credentials/credential_provider.py +5 -0
  8. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/download/download_async.py +24 -52
  9. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/download/download_functions.py +29 -9
  10. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/transfer_bar.py +30 -12
  11. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/__init__.py +10 -0
  12. synapseclient-4.7.0/synapseclient/models/agent.py +945 -0
  13. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/mixins/__init__.py +2 -0
  14. synapseclient-4.7.0/synapseclient/models/mixins/asynchronous_job.py +410 -0
  15. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/mixins/storable_container.py +138 -23
  16. synapseclient-4.7.0/synapseclient/models/protocols/agent_protocol.py +396 -0
  17. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/services/storable_entity_components.py +2 -1
  18. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/services/json_schema.py +96 -113
  19. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/synapsePythonClient +1 -1
  20. {synapseclient-4.6.0 → synapseclient-4.7.0/synapseclient.egg-info}/PKG-INFO +3 -3
  21. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient.egg-info/SOURCES.txt +4 -0
  22. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient.egg-info/requires.txt +1 -1
  23. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseutils/__init__.py +0 -2
  24. {synapseclient-4.6.0 → synapseclient-4.7.0}/MANIFEST.in +0 -0
  25. {synapseclient-4.6.0 → synapseclient-4.7.0}/README.md +0 -0
  26. {synapseclient-4.6.0 → synapseclient-4.7.0}/pyproject.toml +0 -0
  27. {synapseclient-4.6.0 → synapseclient-4.7.0}/setup.py +0 -0
  28. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/.synapseConfig +0 -0
  29. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/__init__.py +0 -0
  30. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/__main__.py +0 -0
  31. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/activity.py +0 -0
  32. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/annotations.py +0 -0
  33. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/api/annotations.py +0 -0
  34. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/api/configuration_services.py +0 -0
  35. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/api/entity_bundle_services_v2.py +0 -0
  36. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/api/entity_factory.py +0 -0
  37. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/api/entity_services.py +0 -0
  38. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/api/file_services.py +0 -0
  39. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/__init__.py +0 -0
  40. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/async_utils.py +0 -0
  41. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/cache.py +0 -0
  42. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/config.py +0 -0
  43. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/constants/__init__.py +0 -0
  44. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/constants/config_file_constants.py +0 -0
  45. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/constants/limits.py +0 -0
  46. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/constants/method_flags.py +0 -0
  47. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/credentials/__init__.py +0 -0
  48. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/credentials/cred_data.py +0 -0
  49. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/cumulative_transfer_progress.py +0 -0
  50. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/download/__init__.py +0 -0
  51. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/dozer.py +0 -0
  52. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/exceptions.py +0 -0
  53. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/lock.py +0 -0
  54. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/logging_setup.py +0 -0
  55. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/models/__init__.py +0 -0
  56. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/models/custom_json.py +0 -0
  57. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/models/dict_object.py +0 -0
  58. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/models/permission.py +0 -0
  59. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/multithread_download/__init__.py +0 -0
  60. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/multithread_download/download_threads.py +0 -0
  61. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/pool_provider.py +0 -0
  62. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/remote_file_storage_wrappers.py +0 -0
  63. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/retry.py +0 -0
  64. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/sts_transfer.py +0 -0
  65. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/upload/__init__.py +0 -0
  66. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/upload/multipart_upload.py +0 -0
  67. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/upload/multipart_upload_async.py +0 -0
  68. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/upload/upload_functions.py +0 -0
  69. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/upload/upload_functions_async.py +0 -0
  70. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/upload/upload_utils.py +0 -0
  71. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/utils.py +0 -0
  72. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/core/version_check.py +0 -0
  73. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/entity.py +0 -0
  74. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/evaluation.py +0 -0
  75. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/activity.py +0 -0
  76. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/annotations.py +0 -0
  77. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/evaluation.py +0 -0
  78. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/file.py +0 -0
  79. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/folder.py +0 -0
  80. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/mixins/access_control.py +0 -0
  81. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/project.py +0 -0
  82. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/__init__.py +0 -0
  83. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/access_control_protocol.py +0 -0
  84. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/activity_protocol.py +0 -0
  85. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/annotations_protocol.py +0 -0
  86. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/file_protocol.py +0 -0
  87. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/folder_protocol.py +0 -0
  88. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/project_protocol.py +0 -0
  89. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/storable_container_protocol.py +0 -0
  90. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/table_protocol.py +0 -0
  91. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/team_protocol.py +0 -0
  92. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/protocols/user_protocol.py +0 -0
  93. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/services/__init__.py +0 -0
  94. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/services/search.py +0 -0
  95. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/services/storable_entity.py +0 -0
  96. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/table.py +0 -0
  97. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/team.py +0 -0
  98. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/user.py +0 -0
  99. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/models/wiki.py +0 -0
  100. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/services/__init__.py +0 -0
  101. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/table.py +0 -0
  102. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/team.py +0 -0
  103. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient/wiki.py +0 -0
  104. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient.egg-info/dependency_links.txt +0 -0
  105. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient.egg-info/entry_points.txt +0 -0
  106. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient.egg-info/not-zip-safe +0 -0
  107. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseclient.egg-info/top_level.txt +0 -0
  108. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseutils/copy_functions.py +0 -0
  109. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseutils/describe_functions.py +0 -0
  110. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseutils/migrate_functions.py +0 -0
  111. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseutils/monitor.py +0 -0
  112. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseutils/sync.py +0 -0
  113. {synapseclient-4.6.0 → synapseclient-4.7.0}/synapseutils/walk_functions.py +0 -0
  114. {synapseclient-4.6.0 → synapseclient-4.7.0}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: synapseclient
3
- Version: 4.6.0
3
+ Version: 4.7.0
4
4
  Summary: A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate.
5
5
  Home-page: https://www.synapse.org
6
6
  Author: The Synapse Engineering Team
@@ -79,7 +79,7 @@ Provides-Extra: docs
79
79
  Requires-Dist: mkdocs>=1.5.3; extra == "docs"
80
80
  Requires-Dist: mkdocs-material>=9.4.14; extra == "docs"
81
81
  Requires-Dist: mkdocstrings>=0.24.0; extra == "docs"
82
- Requires-Dist: mkdocstrings-python>=1.7.5; extra == "docs"
82
+ Requires-Dist: mkdocstrings-python>=1.8.0; extra == "docs"
83
83
  Requires-Dist: termynal>=0.11.1; extra == "docs"
84
84
  Requires-Dist: mkdocs-open-in-new-tab~=1.0.3; extra == "docs"
85
85
  Requires-Dist: markdown-include~=0.8.1; extra == "docs"
@@ -101,7 +101,7 @@ docs =
101
101
  mkdocs>=1.5.3
102
102
  mkdocs-material>=9.4.14
103
103
  mkdocstrings>=0.24.0
104
- mkdocstrings-python>=1.7.5
104
+ mkdocstrings-python>=1.8.0
105
105
  termynal>=0.11.1
106
106
  mkdocs-open-in-new-tab~=1.0.3
107
107
  markdown-include~=0.8.1
@@ -1,4 +1,12 @@
1
1
  # These are all of the models that are used by the Synapse client.
2
+ from .agent_services import (
3
+ get_agent,
4
+ get_session,
5
+ get_trace,
6
+ register_agent,
7
+ start_session,
8
+ update_session,
9
+ )
2
10
  from .annotations import set_annotations, set_annotations_async
3
11
  from .configuration_services import (
4
12
  get_client_authenticated_s3_profile,
@@ -78,4 +86,11 @@ __all__ = [
78
86
  "get_transfer_config",
79
87
  # entity_factory
80
88
  "get_from_entity_factory",
89
+ # agent_services
90
+ "register_agent",
91
+ "get_agent",
92
+ "start_session",
93
+ "get_session",
94
+ "update_session",
95
+ "get_trace",
81
96
  ]
@@ -0,0 +1,189 @@
1
+ """This module is responsible for exposing the services defined at:
2
+ <https://rest-docs.synapse.org/rest/#org.sagebionetworks.repo.web.controller.AgentController>
3
+ """
4
+
5
+ import json
6
+ from typing import TYPE_CHECKING, Any, Dict, Optional
7
+
8
+ if TYPE_CHECKING:
9
+ from synapseclient import Synapse
10
+
11
+
12
+ async def register_agent(
13
+ cloud_agent_id: str,
14
+ cloud_alias_id: Optional[str] = None,
15
+ *,
16
+ synapse_client: Optional["Synapse"] = None,
17
+ ) -> Dict[str, Any]:
18
+ """
19
+ Registers an agent with Synapse OR gets existing agent registration.
20
+ Sends a request matching
21
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/AgentRegistrationRequest.html>
22
+
23
+ Arguments:
24
+ cloud_agent_id: The cloud provider ID of the agent to register.
25
+ cloud_alias_id: The cloud provider alias ID of the agent to register.
26
+ In the Synapse API, this defaults to 'TSTALIASID'.
27
+ synapse_client: If not passed in and caching was not disabled by
28
+ `Synapse.allow_client_caching(False)` this will use the last created
29
+ instance from the Synapse class constructor.
30
+
31
+ Returns:
32
+ The registered agent matching
33
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/AgentRegistration.html>
34
+ """
35
+ from synapseclient import Synapse
36
+
37
+ client = Synapse.get_client(synapse_client=synapse_client)
38
+
39
+ request = {"awsAgentId": cloud_agent_id}
40
+ if cloud_alias_id:
41
+ request["awsAliasId"] = cloud_alias_id
42
+ return await client.rest_put_async(
43
+ uri="/agent/registration", body=json.dumps(request)
44
+ )
45
+
46
+
47
+ async def get_agent(
48
+ registration_id: str,
49
+ *,
50
+ synapse_client: Optional["Synapse"] = None,
51
+ ) -> Dict[str, Any]:
52
+ """
53
+ Gets information about an existing agent registration.
54
+
55
+ Arguments:
56
+ registration_id: The ID of the agent registration to get.
57
+ synapse_client: If not passed in and caching was not disabled by
58
+ `Synapse.allow_client_caching(False)` this will use the last created
59
+ instance from the Synapse class constructor.
60
+
61
+ Returns:
62
+ The requested agent registration matching
63
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/AgentRegistration.html>
64
+ """
65
+ from synapseclient import Synapse
66
+
67
+ client = Synapse.get_client(synapse_client=synapse_client)
68
+ return await client.rest_get_async(uri=f"/agent/registration/{registration_id}")
69
+
70
+
71
+ async def start_session(
72
+ access_level: str,
73
+ agent_registration_id: str,
74
+ *,
75
+ synapse_client: Optional["Synapse"] = None,
76
+ ) -> Dict[str, Any]:
77
+ """
78
+ Starts a new chat session with an agent.
79
+ Sends a request matching
80
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/CreateAgentSessionRequest.html>
81
+
82
+ Arguments:
83
+ access_level: The access level of the agent.
84
+ agent_registration_id: The ID of the agent registration to start the session for.
85
+ synapse_client: If not passed in and caching was not disabled by
86
+ `Synapse.allow_client_caching(False)` this will use the last created
87
+ instance from the Synapse class constructor.
88
+ """
89
+ from synapseclient import Synapse
90
+
91
+ client = Synapse.get_client(synapse_client=synapse_client)
92
+
93
+ request = {
94
+ "agentAccessLevel": access_level,
95
+ "agentRegistrationId": agent_registration_id,
96
+ }
97
+ return await client.rest_post_async(uri="/agent/session", body=json.dumps(request))
98
+
99
+
100
+ async def get_session(
101
+ id: str,
102
+ *,
103
+ synapse_client: Optional["Synapse"] = None,
104
+ ) -> Dict[str, Any]:
105
+ """
106
+ Gets information about an existing chat session.
107
+
108
+ Arguments:
109
+ id: The ID of the session to get.
110
+ synapse_client: If not passed in and caching was not disabled by
111
+ `Synapse.allow_client_caching(False)` this will use the last created
112
+ instance from the Synapse class constructor.
113
+
114
+ Returns:
115
+ The requested session matching
116
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/AgentSession.html>
117
+ """
118
+ from synapseclient import Synapse
119
+
120
+ client = Synapse.get_client(synapse_client=synapse_client)
121
+ return await client.rest_get_async(uri=f"/agent/session/{id}")
122
+
123
+
124
+ async def update_session(
125
+ id: str,
126
+ access_level: str,
127
+ *,
128
+ synapse_client: Optional["Synapse"] = None,
129
+ ) -> Dict[str, Any]:
130
+ """
131
+ Updates the access level for a chat session.
132
+ Sends a request matching
133
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/UpdateAgentSessionRequest.html>
134
+
135
+ Arguments:
136
+ id: The ID of the session to update.
137
+ access_level: The access level of the agent.
138
+ synapse_client: If not passed in and caching was not disabled by
139
+ `Synapse.allow_client_caching(False)` this will use the last created
140
+ instance from the Synapse class constructor.
141
+ """
142
+ from synapseclient import Synapse
143
+
144
+ client = Synapse.get_client(synapse_client=synapse_client)
145
+
146
+ request = {
147
+ "sessionId": id,
148
+ "agentAccessLevel": access_level,
149
+ }
150
+ return await client.rest_put_async(
151
+ uri=f"/agent/session/{id}", body=json.dumps(request)
152
+ )
153
+
154
+
155
+ async def get_trace(
156
+ prompt_id: str,
157
+ *,
158
+ newer_than: Optional[int] = None,
159
+ synapse_client: Optional["Synapse"] = None,
160
+ ) -> Dict[str, Any]:
161
+ """
162
+ Gets the trace of a prompt.
163
+ Sends a request matching
164
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/TraceEventsRequest.html>
165
+
166
+ Arguments:
167
+ prompt_id: The token of the prompt to get the trace for.
168
+ newer_than: The timestamp to get trace results newer than. Defaults to None (all results).
169
+ Timestamps should be in milliseconds since the epoch per the API documentation.
170
+ https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/TraceEvent.html
171
+ synapse_client: If not passed in and caching was not disabled by
172
+ `Synapse.allow_client_caching(False)` this will use the last created
173
+ instance from the Synapse class constructor.
174
+
175
+ Returns:
176
+ The trace matching
177
+ <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/agent/TraceEventsResponse.html>
178
+ """
179
+ from synapseclient import Synapse
180
+
181
+ client = Synapse.get_client(synapse_client=synapse_client)
182
+
183
+ request = {
184
+ "jobId": prompt_id,
185
+ "newerThanTimestamp": newer_than,
186
+ }
187
+ return await client.rest_post_async(
188
+ uri=f"/agent/chat/trace/{prompt_id}", body=json.dumps(request)
189
+ )
@@ -6373,20 +6373,17 @@ class Synapse(object):
6373
6373
  Returns:
6374
6374
  JSON encoding of response
6375
6375
  """
6376
- try:
6377
- response = await self._rest_call_async(
6378
- "get",
6379
- uri,
6380
- None,
6381
- endpoint,
6382
- headers,
6383
- retry_policy,
6384
- requests_session_async_synapse,
6385
- **kwargs,
6386
- )
6387
- return self._return_rest_body(response)
6388
- except Exception:
6389
- self.logger.exception("Error in rest_get_async")
6376
+ response = await self._rest_call_async(
6377
+ "get",
6378
+ uri,
6379
+ None,
6380
+ endpoint,
6381
+ headers,
6382
+ retry_policy,
6383
+ requests_session_async_synapse,
6384
+ **kwargs,
6385
+ )
6386
+ return self._return_rest_body(response)
6390
6387
 
6391
6388
  async def rest_post_async(
6392
6389
  self,
@@ -68,3 +68,6 @@ MULTIPART_UPLOAD_COPY_REQUEST = (
68
68
  # Activity/Provenance
69
69
  USED_URL = "org.sagebionetworks.repo.model.provenance.UsedURL"
70
70
  USED_ENTITY = "org.sagebionetworks.repo.model.provenance.UsedEntity"
71
+
72
+ # Agent
73
+ AGENT_CHAT_REQUEST = "org.sagebionetworks.repo.model.agent.AgentChatRequest"
@@ -7,6 +7,8 @@ import abc
7
7
  import os
8
8
  from typing import TYPE_CHECKING, Dict, Tuple, Union
9
9
 
10
+ from opentelemetry import trace
11
+
10
12
  from synapseclient.api import get_config_authentication
11
13
  from synapseclient.core.credentials.cred_data import (
12
14
  SynapseAuthTokenCredentials,
@@ -92,6 +94,9 @@ class SynapseCredentialsProvider(metaclass=abc.ABCMeta):
92
94
  credentials.username = profile_username
93
95
  credentials.displayname = profile_displayname
94
96
  credentials.owner_id = profile.get("ownerId", None)
97
+ current_span = trace.get_current_span()
98
+ if current_span.is_recording():
99
+ current_span.set_attribute("user.id", credentials.owner_id)
95
100
 
96
101
  return credentials
97
102
 
@@ -16,10 +16,7 @@ from urllib.parse import parse_qs, urlparse
16
16
 
17
17
  import httpx
18
18
 
19
- from synapseclient.api.file_services import (
20
- get_file_handle_for_download,
21
- get_file_handle_for_download_async,
22
- )
19
+ from synapseclient.api.file_services import get_file_handle_for_download
23
20
  from synapseclient.core.exceptions import (
24
21
  SynapseDownloadAbortedException,
25
22
  _raise_for_status_httpx,
@@ -29,6 +26,7 @@ from synapseclient.core.retry import (
29
26
  RETRYABLE_CONNECTION_ERRORS,
30
27
  RETRYABLE_CONNECTION_EXCEPTIONS,
31
28
  with_retry_time_based,
29
+ with_retry_time_based_async,
32
30
  )
33
31
  from synapseclient.core.transfer_bar import get_or_create_download_progress_bar
34
32
 
@@ -110,24 +108,6 @@ class PresignedUrlProvider:
110
108
  # offset parameter used to buffer url expiration checks, time in seconds
111
109
  _TIME_BUFFER: datetime.timedelta = datetime.timedelta(seconds=5)
112
110
 
113
- async def get_info_async(self) -> PresignedUrlInfo:
114
- """
115
- Using async, returns the cached info if it's not expired, otherwise
116
- retrieves a new pre-signed url and returns that.
117
-
118
- Returns:
119
- Information about a retrieved presigned-url from either the cache or a
120
- new request
121
- """
122
- if not self._cached_info or (
123
- datetime.datetime.now(tz=datetime.timezone.utc)
124
- + PresignedUrlProvider._TIME_BUFFER
125
- >= self._cached_info.expiration_utc
126
- ):
127
- self._cached_info = await self._get_pre_signed_info_async()
128
-
129
- return self._cached_info
130
-
131
111
  def get_info(self) -> PresignedUrlInfo:
132
112
  """
133
113
  Using a thread lock, returns the cached info if it's not expired, otherwise
@@ -168,27 +148,6 @@ class PresignedUrlProvider:
168
148
  expiration_utc=_pre_signed_url_expiration_time(pre_signed_url),
169
149
  )
170
150
 
171
- async def _get_pre_signed_info_async(self) -> PresignedUrlInfo:
172
- """
173
- Make an HTTP request to get a pre-signed url to download a file.
174
-
175
- Returns:
176
- Information about a retrieved presigned-url from a new request.
177
- """
178
- response = await get_file_handle_for_download_async(
179
- file_handle_id=self.request.file_handle_id,
180
- synapse_id=self.request.object_id,
181
- entity_type=self.request.object_type,
182
- synapse_client=self.client,
183
- )
184
- file_name = response["fileHandle"]["fileName"]
185
- pre_signed_url = response["preSignedURL"]
186
- return PresignedUrlInfo(
187
- file_name=file_name,
188
- url=pre_signed_url,
189
- expiration_utc=_pre_signed_url_expiration_time(pre_signed_url),
190
- )
191
-
192
151
 
193
152
  def _generate_chunk_ranges(
194
153
  file_size: int,
@@ -232,40 +191,47 @@ def _pre_signed_url_expiration_time(url: str) -> datetime:
232
191
  return return_data
233
192
 
234
193
 
235
- async def _get_file_size_wrapper(syn: "Synapse", url: str, debug: bool) -> int:
194
+ async def _get_file_size_wrapper(
195
+ syn: "Synapse", url_provider: PresignedUrlProvider, debug: bool
196
+ ) -> int:
236
197
  """
237
198
  Gets the size of the file located at url
238
199
 
239
200
  Arguments:
240
201
  syn: The synapseclient
241
- url: The pre-signed url of the file
202
+ url_provider: A URL provider for the presigned urls
242
203
  debug: A boolean to specify if debug mode is on
243
204
 
244
205
  Returns:
245
206
  The size of the file in bytes
246
207
  """
208
+
247
209
  loop = asyncio.get_running_loop()
248
210
  return await loop.run_in_executor(
249
211
  syn._get_thread_pool_executor(asyncio_event_loop=loop),
250
212
  _get_file_size,
251
213
  syn,
252
- url,
214
+ url_provider,
253
215
  debug,
254
216
  )
255
217
 
256
218
 
257
- def _get_file_size(syn: "Synapse", url: str, debug: bool) -> int:
219
+ def _get_file_size(
220
+ syn: "Synapse", presigned_url_provider: PresignedUrlProvider, debug: bool
221
+ ) -> int:
258
222
  """
259
223
  Gets the size of the file located at url
260
224
 
261
225
  Arguments:
262
- url: The pre-signed url of the file
226
+ url_provider: A URL provider for the presigned urls
263
227
  debug: A boolean to specify if debug mode is on
264
228
 
265
229
  Returns:
266
230
  The size of the file in bytes
267
231
  """
268
- with syn._requests_session_storage.stream("GET", url) as response:
232
+ with syn._requests_session_storage.stream(
233
+ method="GET", url=presigned_url_provider.get_info().url
234
+ ) as response:
269
235
  _raise_for_status_httpx(
270
236
  response=response,
271
237
  logger=syn.logger,
@@ -306,9 +272,15 @@ class _MultithreadedDownloader:
306
272
  """
307
273
  url_provider = PresignedUrlProvider(self._syn, request=self._download_request)
308
274
 
309
- url_info = await url_provider.get_info_async()
310
- file_size = await _get_file_size_wrapper(
311
- syn=self._syn, url=url_info.url, debug=self._download_request.debug
275
+ file_size = await with_retry_time_based_async(
276
+ function=lambda: _get_file_size_wrapper(
277
+ syn=self._syn,
278
+ url_provider=url_provider,
279
+ debug=self._download_request.debug,
280
+ ),
281
+ retry_status_codes=[403],
282
+ retry_max_wait_before_failure=30,
283
+ read_response_content=False,
312
284
  )
313
285
  self._progress_bar = get_or_create_download_progress_bar(
314
286
  file_size=file_size,
@@ -146,6 +146,7 @@ async def download_file_entity(
146
146
  if_collision=if_collision,
147
147
  synapse_cache_location=synapse_cache_location,
148
148
  cached_file_path=cached_file_path,
149
+ entity_id=getattr(entity, "id", None),
149
150
  synapse_client=client,
150
151
  )
151
152
  if download_path is None:
@@ -157,9 +158,15 @@ async def download_file_entity(
157
158
  if not os.path.exists(download_location):
158
159
  os.makedirs(download_location)
159
160
  client.logger.info(
160
- f"Copying existing file from {cached_file_path} to {download_path}"
161
+ f"[{getattr(entity, 'id', None)}:{file_name}]: Copying existing "
162
+ f"file from {cached_file_path} to {download_path}"
161
163
  )
162
164
  shutil.copy(cached_file_path, download_path)
165
+ else:
166
+ client.logger.info(
167
+ f"[{getattr(entity, 'id', None)}:{file_name}]: Found existing file "
168
+ f"at {download_path}, skipping download."
169
+ )
163
170
 
164
171
  else: # download the file from URL (could be a local file)
165
172
  object_type = "FileEntity" if submission is None else "SubmissionAttachment"
@@ -257,6 +264,7 @@ async def download_file_entity_model(
257
264
  if_collision=if_collision,
258
265
  synapse_cache_location=synapse_cache_location,
259
266
  cached_file_path=cached_file_path,
267
+ entity_id=file.id,
260
268
  synapse_client=client,
261
269
  )
262
270
  if download_path is None:
@@ -268,9 +276,13 @@ async def download_file_entity_model(
268
276
  if not os.path.exists(download_location):
269
277
  os.makedirs(download_location)
270
278
  client.logger.info(
271
- f"Copying existing file from {cached_file_path} to {download_path}"
279
+ f"[{file.id}:{file_name}]: Copying existing file from {cached_file_path} to {download_path}"
272
280
  )
273
281
  shutil.copy(cached_file_path, download_path)
282
+ else:
283
+ client.logger.info(
284
+ f"[{file.id}:{file_name}]: Found existing file at {download_path}, skipping download."
285
+ )
274
286
 
275
287
  else: # download the file from URL (could be a local file)
276
288
  object_type = "FileEntity" if submission is None else "SubmissionAttachment"
@@ -526,7 +538,7 @@ async def download_by_file_handle(
526
538
  ),
527
539
  )
528
540
 
529
- syn.logger.info(f"Downloaded {synapse_id} to {downloaded_path}")
541
+ syn.logger.info(f"[{synapse_id}]: Downloaded to {downloaded_path}")
530
542
  syn.cache.add(
531
543
  file_handle["id"], downloaded_path, file_handle.get("contentMd5", None)
532
544
  )
@@ -541,7 +553,8 @@ async def download_by_file_handle(
541
553
  exc_info = sys.exc_info()
542
554
  ex.progress = 0 if not hasattr(ex, "progress") else ex.progress
543
555
  syn.logger.debug(
544
- f"\nRetrying download on error: [{exc_info[0]}] after progressing {ex.progress} bytes",
556
+ f"\n[{synapse_id}]: Retrying "
557
+ f"download on error: [{exc_info[0]}] after progressing {ex.progress} bytes",
545
558
  exc_info=True,
546
559
  ) # this will include stack trace
547
560
  if ex.progress == 0: # No progress was made reduce remaining retries.
@@ -669,7 +682,7 @@ def download_from_url(
669
682
  actual_md5 = None
670
683
  redirect_count = 0
671
684
  delete_on_md5_mismatch = True
672
- client.logger.debug(f"Downloading from {url} to {destination}")
685
+ client.logger.debug(f"[{entity_id}]: Downloading from {url} to {destination}")
673
686
  while redirect_count < REDIRECT_LIMIT:
674
687
  redirect_count += 1
675
688
  scheme = urllib_urlparse.urlparse(url).scheme
@@ -854,7 +867,8 @@ def download_from_url(
854
867
  )
855
868
  increment_progress_bar(n=transferred, progress_bar=progress_bar)
856
869
  client.logger.debug(
857
- f"Resuming partial download to {temp_destination}. "
870
+ f"[{entity_id}]: Resuming "
871
+ f"partial download to {temp_destination}. "
858
872
  f"{previously_transferred}/{to_be_transferred} bytes already "
859
873
  "transferred."
860
874
  )
@@ -894,7 +908,8 @@ def download_from_url(
894
908
  # verify that the file was completely downloaded and retry if it is not complete
895
909
  if to_be_transferred > 0 and transferred < to_be_transferred:
896
910
  client.logger.warning(
897
- "\nRetrying download because the connection ended early.\n"
911
+ f"\n[{entity_id}]: "
912
+ "Retrying download because the connection ended early.\n"
898
913
  )
899
914
  continue
900
915
 
@@ -903,7 +918,9 @@ def download_from_url(
903
918
  shutil.move(temp_destination, destination)
904
919
  break
905
920
  else:
906
- client.logger.error(f"Unable to download URLs of type {scheme}")
921
+ client.logger.error(
922
+ f"[{entity_id}]: Unable to download URLs of type {scheme}"
923
+ )
907
924
  return None
908
925
 
909
926
  else: # didn't break out of loop
@@ -949,6 +966,7 @@ def resolve_download_path_collisions(
949
966
  if_collision: str,
950
967
  synapse_cache_location: str,
951
968
  cached_file_path: str,
969
+ entity_id: str,
952
970
  *,
953
971
  synapse_client: Optional["Synapse"] = None,
954
972
  ) -> Union[str, None]:
@@ -964,6 +982,7 @@ def resolve_download_path_collisions(
964
982
  May be "overwrite.local", "keep.local", or "keep.both".
965
983
  synapse_cache_location: The location in .synapseCache where the file would be
966
984
  corresponding to its FileHandleId.
985
+ entity_id: The entity id
967
986
  cached_file_path: The file path of the cached copy
968
987
 
969
988
  Raises:
@@ -1000,7 +1019,8 @@ def resolve_download_path_collisions(
1000
1019
  pass # Let the download proceed and overwrite the local file.
1001
1020
  elif if_collision == COLLISION_KEEP_LOCAL:
1002
1021
  client.logger.info(
1003
- f"Found existing file at {download_path}, skipping download."
1022
+ f"[{entity_id}]: Found existing "
1023
+ f"file at {download_path}, skipping download."
1004
1024
  )
1005
1025
 
1006
1026
  # Don't want to overwrite the local file.
@@ -66,13 +66,17 @@ def increment_progress_bar(n: int, progress_bar: Union[tqdm, None]) -> None:
66
66
 
67
67
  @contextmanager
68
68
  def shared_download_progress_bar(
69
- file_size: int, *, synapse_client: Optional["Synapse"] = None
69
+ file_size: int,
70
+ custom_message: str = None,
71
+ *,
72
+ synapse_client: Optional["Synapse"] = None,
70
73
  ):
71
74
  """An outside process that will eventually trigger a download through this module
72
75
  can configure a shared Progress Bar by running its code within this context manager.
73
76
 
74
77
  Arguments:
75
78
  file_size: The size of the file being downloaded.
79
+ custom_message: A custom message to display on the progress bar instead of default.
76
80
  synapse_client: If not passed in and caching was not disabled by
77
81
  `Synapse.allow_client_caching(False)` this will use the last created
78
82
  instance from the Synapse class constructor.
@@ -86,22 +90,28 @@ def shared_download_progress_bar(
86
90
 
87
91
  syn = Synapse.get_client(synapse_client=synapse_client)
88
92
  with logging_redirect_tqdm(loggers=[syn.logger]):
89
- get_or_create_download_progress_bar(file_size=file_size, synapse_client=syn)
93
+ get_or_create_download_progress_bar(
94
+ file_size=file_size, custom_message=custom_message, synapse_client=syn
95
+ )
90
96
  try:
91
97
  yield
92
98
  finally:
93
99
  _thread_local.progress_bar_download_context_managed = False
94
- if _thread_local.progress_bar_download:
95
- _thread_local.progress_bar_download.close()
96
- _thread_local.progress_bar_download.refresh()
97
- del _thread_local.progress_bar_download
100
+ close_download_progress_bar()
98
101
 
99
102
 
100
- def close_download_progress_bar() -> None:
101
- """Handle closing the download progress bar if it is not context managed."""
102
- if not _is_context_managed_download_bar():
103
+ def close_download_progress_bar(force_close: bool = False) -> None:
104
+ """Handle closing the download progress bar if it is not context managed. This will
105
+ also only close the progress bar if there are no other downloads sharing it."""
106
+ if force_close or not _is_context_managed_download_bar():
103
107
  progress_bar: tqdm = getattr(_thread_local, "progress_bar_download", None)
104
- if progress_bar is not None:
108
+ transfer_count: int = getattr(_thread_local, "transfer_count", 0)
109
+ transfer_count -= 1
110
+ if transfer_count < 0:
111
+ transfer_count = 0
112
+
113
+ _thread_local.transfer_count = transfer_count
114
+ if progress_bar is not None and not transfer_count:
105
115
  progress_bar.close()
106
116
  progress_bar.refresh()
107
117
  del _thread_local.progress_bar_download
@@ -113,7 +123,11 @@ def _is_context_managed_download_bar() -> bool:
113
123
 
114
124
 
115
125
  def get_or_create_download_progress_bar(
116
- file_size: int, postfix: str = None, *, synapse_client: Optional["Synapse"] = None
126
+ file_size: int,
127
+ postfix: str = None,
128
+ custom_message: str = None,
129
+ *,
130
+ synapse_client: Optional["Synapse"] = None,
117
131
  ) -> Union[tqdm, None]:
118
132
  """Return the existing progress bar if it exists, otherwise create a new one.
119
133
 
@@ -132,11 +146,15 @@ def get_or_create_download_progress_bar(
132
146
  if syn.silent:
133
147
  return None
134
148
 
149
+ transfer_count: int = getattr(_thread_local, "transfer_count", 0)
150
+ transfer_count += 1
151
+ _thread_local.transfer_count = transfer_count
152
+
135
153
  progress_bar: tqdm = getattr(_thread_local, "progress_bar_download", None)
136
154
  if progress_bar is None:
137
155
  progress_bar = tqdm(
138
156
  total=file_size,
139
- desc="Downloading files",
157
+ desc=custom_message or "Downloading files",
140
158
  unit="B",
141
159
  unit_scale=True,
142
160
  smoothing=0,