synapseclient 4.4.1__tar.gz → 4.5.0__tar.gz

{synapseclient-4.4.1/synapseclient.egg-info → synapseclient-4.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapseclient
-Version: 4.4.1
+Version: 4.5.0
 Summary: A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate.
 Home-page: https://www.synapse.org
 Author: The Synapse Engineering Team
@@ -32,9 +32,9 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Requires-Dist: requests<3.0,>=2.22.0
 Requires-Dist: urllib3<2,>=1.26.18
 Requires-Dist: deprecated<2.0,>=1.2.4
-Requires-Dist: opentelemetry-api~=1.21.0
-Requires-Dist: opentelemetry-sdk~=1.21.0
-Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.21.0
+Requires-Dist: opentelemetry-api>=1.21.0
+Requires-Dist: opentelemetry-sdk>=1.21.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.21.0
 Requires-Dist: nest-asyncio~=1.6.0
 Requires-Dist: asyncio-atexit~=1.0.1
 Requires-Dist: httpx~=0.27.0
@@ -67,6 +67,7 @@ Requires-Dist: pytest-cov~=4.1.0; extra == "tests"
 Requires-Dist: pandas<3.0,>=1.5; extra == "tests"
 Provides-Extra: pandas
 Requires-Dist: pandas<3.0,>=1.5; extra == "pandas"
+Requires-Dist: numpy<2.0.0; extra == "pandas"
 Provides-Extra: pysftp
 Requires-Dist: pysftp<0.3,>=0.2.8; extra == "pysftp"
 Provides-Extra: boto3

{synapseclient-4.4.1 → synapseclient-4.5.0}/setup.cfg

@@ -41,9 +41,9 @@ install_requires =
 	requests>=2.22.0,<3.0
 	urllib3>=1.26.18,<2
 	deprecated>=1.2.4,<2.0
-	opentelemetry-api~=1.21.0
-	opentelemetry-sdk~=1.21.0
-	opentelemetry-exporter-otlp-proto-http~=1.21.0
+	opentelemetry-api>=1.21.0
+	opentelemetry-sdk>=1.21.0
+	opentelemetry-exporter-otlp-proto-http>=1.21.0
 	nest-asyncio~=1.6.0
 	asyncio-atexit~=1.0.1
 	httpx~=0.27.0
@@ -89,6 +89,7 @@ tests =
 	pandas>=1.5,<3.0
 pandas = 
 	pandas>=1.5,<3.0
+	numpy<2.0.0
 pysftp = 
 	pysftp>=0.2.8,<0.3
 boto3 = 

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient/client.py

@@ -380,6 +380,10 @@ class Synapse(object):
         if debug is None:
             debug = config_debug if config_debug is not None else DEBUG_DEFAULT
 
+        if not isinstance(debug, bool):
+            raise ValueError("debug must be set to a bool (either True or False)")
+        self.debug = debug
+
         self.cache = cache.Cache(cache_root_dir)
         self._sts_token_store = sts_transfer.StsTokenStore()
 
@@ -393,16 +397,12 @@ class Synapse(object):
         }
         self.credentials = None
 
-        if not isinstance(debug, bool):
-            raise ValueError("debug must be set to a bool (either True or False)")
-        self.debug = debug
-
         self.silent = silent
         self._init_logger()  # initializes self.logger
 
         self.skip_checks = skip_checks
 
-        self.table_query_sleep = 2
+        self.table_query_sleep = 0.2  # Sleep for 200ms between table query retries
         self.table_query_backoff = 1.1
         self.table_query_max_sleep = 20
         self.table_query_timeout = 600  # in seconds
@@ -696,10 +696,14 @@ class Synapse(object):
 
             # Update endpoints if we get redirected
             if not skip_checks:
-                response = self._requests_session.get(
-                    endpoints[point],
-                    allow_redirects=False,
-                    headers=synapseclient.USER_AGENT,
+                response = with_retry(
+                    lambda point=point: self._requests_session.get(
+                        endpoints[point],
+                        allow_redirects=False,
+                        headers=synapseclient.USER_AGENT,
+                    ),
+                    verbose=self.debug,
+                    **STANDARD_RETRY_PARAMS,
                 )
                 if response.status_code == 301:
                     endpoints[point] = response.headers["location"]
@@ -5041,6 +5045,7 @@ class Synapse(object):
     #                      Tables                              #
     ############################################################
 
+    @tracer.start_as_current_span("Synapse::_waitForAsync")
     def _waitForAsync(self, uri, request, endpoint=None):
         if endpoint is None:
             endpoint = self.repoEndpoint
@@ -5072,7 +5077,7 @@ class Synapse(object):
                 sleep = min(
                     self.table_query_max_sleep, sleep * self.table_query_backoff
                 )
-                doze(sleep)
+                doze(sleep, trace_span_name="Synapse::doze_for_async")
             else:
                 break
         else:

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient/core/download/__init__.py

@@ -6,6 +6,7 @@ from .download_async import (
     PresignedUrlInfo,
     PresignedUrlProvider,
     _MultithreadedDownloader,
+    _pre_signed_url_expiration_time,
     download_file,
 )
 from .download_functions import (
@@ -32,4 +33,5 @@ __all__ = [
     "PresignedUrlInfo",
     "PresignedUrlProvider",
     "_MultithreadedDownloader",
+    "_pre_signed_url_expiration_time",
 ]

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient/core/download/download_async.py

@@ -274,7 +274,12 @@ def _get_file_size(
     if otel_context:
         context.attach(otel_context)
     with syn._requests_session_storage.stream("GET", url) as response:
-        _raise_for_status_httpx(response=response, logger=syn.logger, verbose=debug)
+        _raise_for_status_httpx(
+            response=response,
+            logger=syn.logger,
+            verbose=debug,
+            read_response_content=False,
+        )
         return int(response.headers["Content-Length"])
 
 

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient/core/download/download_functions.py

@@ -1,6 +1,7 @@
 """This module handles the various ways that a user can download a file to Synapse."""
 
 import asyncio
+import datetime
 import errno
 import hashlib
 import os
@@ -14,7 +15,10 @@ from tqdm import tqdm
 from tqdm.contrib.logging import logging_redirect_tqdm
 
 from synapseclient.api.configuration_services import get_client_authenticated_s3_profile
-from synapseclient.api.file_services import get_file_handle_for_download_async
+from synapseclient.api.file_services import (
+    get_file_handle_for_download,
+    get_file_handle_for_download_async,
+)
 from synapseclient.core import exceptions, sts_transfer, utils
 from synapseclient.core.constants import concrete_types
 from synapseclient.core.constants.method_flags import (
@@ -25,6 +29,8 @@ from synapseclient.core.constants.method_flags import (
 from synapseclient.core.download import (
     SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE,
     DownloadRequest,
+    PresignedUrlProvider,
+    _pre_signed_url_expiration_time,
     download_file,
 )
 from synapseclient.core.exceptions import (
@@ -500,11 +506,14 @@ async def download_by_file_handle(
                 progress_bar = get_or_create_download_progress_bar(
                     file_size=1, postfix=synapse_id, synapse_client=syn
                 )
+
                 downloaded_path = await loop.run_in_executor(
                     syn._get_thread_pool_executor(asyncio_event_loop=loop),
                     lambda: download_from_url(
                         url=file_handle_result["preSignedURL"],
                         destination=destination,
+                        entity_id=synapse_id,
+                        file_handle_associate_type=entity_type,
                         file_handle_id=file_handle["id"],
                         expected_md5=file_handle.get("contentMd5"),
                         progress_bar=progress_bar,
@@ -610,6 +619,8 @@ async def download_from_url_multi_threaded(
 def download_from_url(
     url: str,
     destination: str,
+    entity_id: Optional[str],
+    file_handle_associate_type: Optional[str],
     file_handle_id: Optional[str] = None,
     expected_md5: Optional[str] = None,
     progress_bar: Optional[tqdm] = None,
@@ -622,6 +633,11 @@ def download_from_url(
     Arguments:
         url:           The source of download
         destination:   The destination on local file system
+        entity_id:      The id of the Synapse object that uses the FileHandle
+            e.g. "syn123"
+        file_handle_associate_type:    The type of the Synapse object that uses the
+            FileHandle e.g. "FileEntity". Any of
+            <https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/file/FileHandleAssociateType.html>
         file_handle_id:  Optional. If given, the file will be given a temporary name that includes the file
                                 handle id which allows resuming partial downloads of the same file from previous
                                 sessions
@@ -719,21 +735,69 @@ def download_from_url(
                 else None
             )
 
-            response = with_retry(
-                lambda url=url, range_header=range_header, auth=auth: client._requests_session.get(
-                    url=url,
-                    headers=client._generate_headers(range_header),
-                    stream=True,
-                    allow_redirects=False,
-                    auth=auth,
-                ),
-                verbose=client.debug,
-                **STANDARD_RETRY_PARAMS,
-            )
             try:
+                url_has_expiration = "Expires" in urllib_urlparse.urlparse(url).query
+                url_is_expired = False
+                if url_has_expiration:
+                    url_is_expired = datetime.datetime.now(
+                        tz=datetime.timezone.utc
+                    ) + PresignedUrlProvider._TIME_BUFFER >= _pre_signed_url_expiration_time(
+                        url
+                    )
+                if url_is_expired:
+                    response = get_file_handle_for_download(
+                        file_handle_id=file_handle_id,
+                        synapse_id=entity_id,
+                        entity_type=file_handle_associate_type,
+                        synapse_client=client,
+                    )
+                    url = response["preSignedURL"]
+                response = with_retry(
+                    lambda url=url, range_header=range_header, auth=auth: client._requests_session.get(
+                        url=url,
+                        headers=client._generate_headers(range_header),
+                        stream=True,
+                        allow_redirects=False,
+                        auth=auth,
+                    ),
+                    verbose=client.debug,
+                    **STANDARD_RETRY_PARAMS,
+                )
                 exceptions._raise_for_status(response, verbose=client.debug)
             except SynapseHTTPError as err:
-                if err.response.status_code == 404:
+                if err.response.status_code == 403:
+                    url_has_expiration = (
+                        "Expires" in urllib_urlparse.urlparse(url).query
+                    )
+                    url_is_expired = False
+                    if url_has_expiration:
+                        url_is_expired = datetime.datetime.now(
+                            tz=datetime.timezone.utc
+                        ) + PresignedUrlProvider._TIME_BUFFER >= _pre_signed_url_expiration_time(
+                            url
+                        )
+                    if url_is_expired:
+                        response = get_file_handle_for_download(
+                            file_handle_id=file_handle_id,
+                            synapse_id=entity_id,
+                            entity_type=file_handle_associate_type,
+                            synapse_client=client,
+                        )
+                        refreshed_url = response["preSignedURL"]
+                        response = with_retry(
+                            lambda url=refreshed_url, range_header=range_header, auth=auth: client._requests_session.get(
+                                url=url,
+                                headers=client._generate_headers(range_header),
+                                stream=True,
+                                allow_redirects=False,
+                                auth=auth,
+                            ),
+                            verbose=client.debug,
+                            **STANDARD_RETRY_PARAMS,
+                        )
+                    else:
+                        raise
+                elif err.response.status_code == 404:
                     raise SynapseError(f"Could not download the file at {url}") from err
                 elif (
                     err.response.status_code == 416
@@ -745,8 +809,8 @@ def download_from_url(
                     # If it fails the user can retry with another download.
                     shutil.move(temp_destination, destination)
                     break
-                raise
-
+                else:
+                    raise
             # handle redirects
             if response.status_code in [301, 302, 303, 307, 308]:
                 url = response.headers["location"]

synapseclient-4.5.0/synapseclient/core/dozer.py

@@ -0,0 +1,44 @@
+"""
+Created on Sep 21, 2017
+
+@author: bhoff
+
+sleep while checking registered _listeners
+"""
+import time
+
+from opentelemetry import trace
+
+tracer = trace.get_tracer("synapseclient")
+
+_listeners = []
+
+
+def add_listener(listener):
+    if not callable(listener):
+        raise ValueError("listener is not callable")
+    _listeners.append(listener)
+
+
+def clear_listeners():
+    del _listeners[:]
+
+
+def doze(
+    secs: float,
+    listener_check_interval_secs: float = 0.1,
+    trace_span_name: str = "doze",
+) -> None:
+    """Sleep for a given number of seconds while checking registered listeners.
+
+    Arguments:
+        secs: the number of seconds to sleep
+        listener_check_interval_secs: the interval at which to check the listeners
+        trace_span_name: the name of the trace span
+    """
+    with tracer.start_as_current_span(name=trace_span_name):
+        end_time = time.time() + secs
+        while time.time() < end_time:
+            for listener in _listeners:
+                listener()
+            time.sleep(listener_check_interval_secs)

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient/core/retry.py

@@ -69,6 +69,8 @@ RETRYABLE_CONNECTION_EXCEPTIONS = [
     "TimeoutException",
     "ConnectError",
     "ConnectTimeout",
+    # SSL Specific exceptions:
+    "SSLZeroReturnError",
 ]
 
 DEBUG_EXCEPTION = "calling %s resulted in an Exception"

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient/synapsePythonClient

@@ -1,6 +1,6 @@
 {
   "client": "synapsePythonClient",
-  "latestVersion":"4.4.1",
+  "latestVersion":"4.5.0",
   "blacklist": [
     "0.0.0",
     "0.4.1",

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient/table.py

@@ -407,7 +407,34 @@ def _csv_to_pandas_df(
     list_columns=None,
     rowIdAndVersionInIndex=True,
     dtype=None,
+    **kwargs,
 ):
+    """
+    Convert a csv file to a pandas dataframe
+
+    Arguments:
+        filepath: The path to the file.
+        separator: The separator for the file, Defaults to `DEFAULT_SEPARATOR`.
+        quote_char: The quote character for the file,
+          Defaults to `DEFAULT_QUOTE_CHARACTER`.
+        escape_char: The escape character for the file,
+                    Defaults to `DEFAULT_ESCAPSE_CHAR`.
+                    contain_headers: Whether the file contains headers,
+                    Defaults to `True`.
+        lines_to_skip: The number of lines to skip at the beginning of the file,
+                        Defaults to `0`.
+        date_columns: The names of the date columns in the file
+        list_columns: The names of the list columns in the file
+        rowIdAndVersionInIndex: Whether the file contains rowId and
+                                version in the index, Defaults to `True`.
+        dtype: The data type for the file, Defaults to `None`.
+        **kwargs: Additional keyword arguments to pass to pandas.read_csv. See
+                    https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+                    for complete list of supported arguments.
+
+    Returns:
+        A pandas dataframe
+    """
     test_import_pandas()
     import pandas as pd
 
@@ -426,6 +453,7 @@ def _csv_to_pandas_df(
         escapechar=escape_char,
         header=0 if contain_headers else None,
         skiprows=lines_to_skip,
+        **kwargs,
     )
     # parse date columns if exists
     if date_columns:
@@ -2445,14 +2473,23 @@ class CsvFileTable(TableAbstractBaseClass):
             self.etag = upload_to_table_result["etag"]
         return self
 
-    def asDataFrame(self, rowIdAndVersionInIndex=True, convert_to_datetime=False):
+    def asDataFrame(
+        self,
+        rowIdAndVersionInIndex: bool = True,
+        convert_to_datetime: bool = False,
+        **kwargs,
+    ):
         """Convert query result to a Pandas DataFrame.
 
         Arguments:
-            rowIdAndVersionInIndex: Make the dataframe index consist of the row_id and row_version
-                                    (and row_etag if it exists)
+            rowIdAndVersionInIndex: Make the dataframe index consist of the
+                                    row_id and row_version (and row_etag if it exists)
             convert_to_datetime:    If set to True, will convert all Synapse DATE columns from UNIX timestamp
                                     integers into UTC datetime objects
+            kwargs:                 Additional keyword arguments to pass to
+                                    pandas.read_csv via _csv_to_pandas_df. See
+                                    https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+                                    for complete list of supported arguments.
 
         Returns:
             A Pandas dataframe with results
@@ -2491,8 +2528,10 @@ class CsvFileTable(TableAbstractBaseClass):
                 list_columns=list_columns,
                 rowIdAndVersionInIndex=rowIdAndVersionInIndex,
                 dtype=dtype,
+                **kwargs,
             )
-        except pd.parser.CParserError:
+
+        except pd.errors.ParserError:
             return pd.DataFrame()
 
     def asRowSet(self):

{synapseclient-4.4.1 → synapseclient-4.5.0/synapseclient.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapseclient
-Version: 4.4.1
+Version: 4.5.0
 Summary: A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate.
 Home-page: https://www.synapse.org
 Author: The Synapse Engineering Team
@@ -32,9 +32,9 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Requires-Dist: requests<3.0,>=2.22.0
 Requires-Dist: urllib3<2,>=1.26.18
 Requires-Dist: deprecated<2.0,>=1.2.4
-Requires-Dist: opentelemetry-api~=1.21.0
-Requires-Dist: opentelemetry-sdk~=1.21.0
-Requires-Dist: opentelemetry-exporter-otlp-proto-http~=1.21.0
+Requires-Dist: opentelemetry-api>=1.21.0
+Requires-Dist: opentelemetry-sdk>=1.21.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.21.0
 Requires-Dist: nest-asyncio~=1.6.0
 Requires-Dist: asyncio-atexit~=1.0.1
 Requires-Dist: httpx~=0.27.0
@@ -67,6 +67,7 @@ Requires-Dist: pytest-cov~=4.1.0; extra == "tests"
 Requires-Dist: pandas<3.0,>=1.5; extra == "tests"
 Provides-Extra: pandas
 Requires-Dist: pandas<3.0,>=1.5; extra == "pandas"
+Requires-Dist: numpy<2.0.0; extra == "pandas"
 Provides-Extra: pysftp
 Requires-Dist: pysftp<0.3,>=0.2.8; extra == "pysftp"
 Provides-Extra: boto3

{synapseclient-4.4.1 → synapseclient-4.5.0}/synapseclient.egg-info/requires.txt

@@ -1,9 +1,9 @@
 requests<3.0,>=2.22.0
 urllib3<2,>=1.26.18
 deprecated<2.0,>=1.2.4
-opentelemetry-api~=1.21.0
-opentelemetry-sdk~=1.21.0
-opentelemetry-exporter-otlp-proto-http~=1.21.0
+opentelemetry-api>=1.21.0
+opentelemetry-sdk>=1.21.0
+opentelemetry-exporter-otlp-proto-http>=1.21.0
 nest-asyncio~=1.6.0
 asyncio-atexit~=1.0.1
 httpx~=0.27.0
@@ -39,6 +39,7 @@ markdown-include~=0.8.1
 
 [pandas]
 pandas<3.0,>=1.5
+numpy<2.0.0
 
 [pysftp]
 pysftp<0.3,>=0.2.8

synapseclient-4.4.1/synapseclient/core/dozer.py

@@ -1,28 +0,0 @@
-"""
-Created on Sep 21, 2017
-
-@author: bhoff
-
-sleep while checking registered _listeners
-"""
-import time
-
-_listeners = []
-
-
-def add_listener(listener):
-    if not callable(listener):
-        raise ValueError("listener is not callable")
-    _listeners.append(listener)
-
-
-def clear_listeners():
-    del _listeners[:]
-
-
-def doze(secs, listener_check_interval_secs=0.1):
-    end_time = time.time() + secs
-    while time.time() < end_time:
-        for listener in _listeners:
-            listener()
-        time.sleep(listener_check_interval_secs)