synapseclient 4.3.0__tar.gz → 4.3.1__tar.gz

{synapseclient-4.3.0/synapseclient.egg-info → synapseclient-4.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapseclient
-Version: 4.3.0
+Version: 4.3.1
 Summary: A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate.
 Home-page: https://www.synapse.org
 Author: The Synapse Engineering Team
@@ -39,7 +39,6 @@ Requires-Dist: nest-asyncio~=1.6.0
 Requires-Dist: asyncio-atexit~=1.0.1
 Requires-Dist: httpx~=0.27.0
 Requires-Dist: tqdm<5.0,>=4.66.2
-Requires-Dist: loky~=3.0.0
 Requires-Dist: async-lru~=2.0.4
 Requires-Dist: psutil~=5.9.8
 Provides-Extra: dev

{synapseclient-4.3.0 → synapseclient-4.3.1}/setup.cfg

@@ -48,7 +48,6 @@ install_requires =
 	asyncio-atexit~=1.0.1
 	httpx~=0.27.0
 	tqdm>=4.66.2,<5.0
-	loky~=3.0.0
 	async-lru~=2.0.4
 	psutil~=5.9.8
 tests_require = 

{synapseclient-4.3.0 → synapseclient-4.3.1}/synapseclient/client.py

@@ -34,7 +34,6 @@ import httpx
 from deprecated import deprecated
 
 from concurrent.futures import ThreadPoolExecutor
-from loky import get_reusable_executor
 
 import synapseclient
 from .annotations import (
@@ -399,7 +398,6 @@ class Synapse(object):
         self._thread_executor = {}
         self._process_executor = {}
         self._parallel_file_transfer_semaphore = {}
-        self._md5_semaphore = {}
         self.use_boto_sts_transfers = transfer_config["use_boto_sts"]
 
     def _get_requests_session_async_synapse(
@@ -510,55 +508,6 @@ class Synapse(object):
         asyncio_atexit.register(close_pool)
         return self._thread_executor[asyncio_event_loop]
 
-    def _get_process_pool_executor(self, asyncio_event_loop: asyncio.AbstractEventLoop):
-        """
-        Retrieve the process pool executor for the Synapse client. Or create a new one
-        if it does not exist. This executor is used for parallel processing of data.
-
-        This is expected to be called from within an AsyncIO loop.
-
-        Note: Within Windows a ProcessPoolExecutor requires that the initial entry point
-        into the code be within a `if __name__ == "__main__":` block. This is not
-        possible within the current codebase as it would require everyone using this
-        library to have this as their entry point. As a result, the ProcessPoolExecutor
-        will not work within Windows.
-
-        To get around this Windows limitation this is using this package:
-        https://github.com/joblib/loky
-        """
-        if (
-            hasattr(self, "_process_executor")
-            and asyncio_event_loop in self._process_executor
-            and self._process_executor[asyncio_event_loop] is not None
-        ):
-            return self._process_executor[asyncio_event_loop]
-
-        self._process_executor.update({asyncio_event_loop: get_reusable_executor(1)})
-
-        return self._process_executor[asyncio_event_loop]
-
-    def _get_md5_semaphore(
-        self, asyncio_event_loop: asyncio.AbstractEventLoop
-    ) -> asyncio.Semaphore:
-        """
-        Retrieve the semaphore for the Synapse client. Or create a new one if it does not
-        exist. This semaphore is used to ensure that only one process is calculating the
-        MD5 hash at a time. This is to prevent the custom process pool executor from
-        thrashing or handling the waiting. We should let asyncio handle the waiting.
-
-        This is expected to be called from within an AsyncIO loop.
-        """
-        if (
-            hasattr(self, "_md5_semaphore")
-            and asyncio_event_loop in self._md5_semaphore
-            and self._md5_semaphore[asyncio_event_loop] is not None
-        ):
-            return self._md5_semaphore[asyncio_event_loop]
-
-        self._md5_semaphore.update({asyncio_event_loop: asyncio.Semaphore(1)})
-
-        return self._md5_semaphore[asyncio_event_loop]
-
     def _get_parallel_file_transfer_semaphore(
         self, asyncio_event_loop: asyncio.AbstractEventLoop
     ) -> asyncio.Semaphore:
@@ -3534,7 +3483,7 @@ class Synapse(object):
                     else:
                         mode = "wb"
                         previouslyTransferred = 0
-                        sig = hashlib.new("md5", usedforsecurity=False)
+                        sig = hashlib.new("md5", usedforsecurity=False)  # nosec
 
                     try:
                         with open(temp_destination, mode) as fd:

{synapseclient-4.3.0 → synapseclient-4.3.1}/synapseclient/core/upload/multipart_upload_async.py

@@ -114,7 +114,7 @@ from synapseclient.core.upload.upload_utils import (
 )
 from synapseclient.core.utils import MB
 from synapseclient.core.utils import md5_fn as md5_fn_util
-from synapseclient.core.utils import md5_for_file_multiprocessing
+from synapseclient.core.utils import md5_for_file_hex
 
 if TYPE_CHECKING:
     from synapseclient import Synapse
@@ -669,17 +669,7 @@ async def multipart_upload_file_async(
         mime_type, _ = mimetypes.guess_type(file_path, strict=False)
         content_type = mime_type or "application/octet-stream"
 
-    md5_hex = md5 or (
-        await md5_for_file_multiprocessing(
-            filename=file_path,
-            process_pool_executor=syn._get_process_pool_executor(
-                asyncio_event_loop=asyncio.get_running_loop()
-            ),
-            md5_semaphore=syn._get_md5_semaphore(
-                asyncio_event_loop=asyncio.get_running_loop()
-            ),
-        )
-    )
+    md5_hex = md5 or md5_for_file_hex(filename=file_path)
 
     part_size = get_part_size(
         part_size or DEFAULT_PART_SIZE,

{synapseclient-4.3.0 → synapseclient-4.3.1}/synapseclient/core/upload/upload_functions_async.py

@@ -77,15 +77,7 @@ async def upload_file_handle(
     expanded_upload_path = os.path.expandvars(os.path.expanduser(path))
 
     if md5 is None and os.path.isfile(expanded_upload_path):
-        md5 = await utils.md5_for_file_multiprocessing(
-            filename=expanded_upload_path,
-            process_pool_executor=syn._get_process_pool_executor(
-                asyncio_event_loop=asyncio.get_running_loop()
-            ),
-            md5_semaphore=syn._get_md5_semaphore(
-                asyncio_event_loop=asyncio.get_running_loop()
-            ),
-        )
+        md5 = utils.md5_for_file_hex(filename=expanded_upload_path)
 
     entity_parent_id = id_of(parent_entity_id)
 
@@ -203,15 +195,7 @@ async def create_external_file_handle(
         parsed_url = urllib_parse.urlparse(url)
         parsed_path = file_url_to_path(url)
         if parsed_url.scheme == "file" and os.path.isfile(parsed_path):
-            actual_md5 = await utils.md5_for_file_multiprocessing(
-                filename=parsed_path,
-                process_pool_executor=syn._get_process_pool_executor(
-                    asyncio_event_loop=asyncio.get_running_loop()
-                ),
-                md5_semaphore=syn._get_md5_semaphore(
-                    asyncio_event_loop=asyncio.get_running_loop()
-                ),
-            )
+            actual_md5 = utils.md5_for_file_hex(filename=parsed_path)
             if md5 is not None and md5 != actual_md5:
                 raise SynapseMd5MismatchError(
                     f"The specified md5 [{md5}] does not match the calculated md5 "
@@ -255,15 +239,7 @@ async def upload_external_file_handle_sftp(
         storage_str=storage_str,
     )
 
-    file_md5 = md5 or await utils.md5_for_file_multiprocessing(
-        filename=file_path,
-        process_pool_executor=syn._get_process_pool_executor(
-            asyncio_event_loop=asyncio.get_running_loop()
-        ),
-        md5_semaphore=syn._get_md5_semaphore(
-            asyncio_event_loop=asyncio.get_running_loop()
-        ),
-    )
+    file_md5 = md5 or utils.md5_for_file_hex(filename=file_path)
     file_handle = await post_external_filehandle(
         external_url=uploaded_url,
         mimetype=mimetype,

{synapseclient-4.3.0 → synapseclient-4.3.1}/synapseclient/core/utils.py

@@ -2,7 +2,6 @@
 Utility functions useful in the implementation and testing of the Synapse client.
 """
 
-import asyncio
 import base64
 import cgi
 import collections.abc
@@ -68,7 +67,7 @@ def md5_for_file(
         The MD5 Checksum
     """
     loop_iteration = 0
-    md5 = hashlib.new("md5", usedforsecurity=False)
+    md5 = hashlib.new("md5", usedforsecurity=False)  # nosec
     with open(filename, "rb") as f:
         while True:
             loop_iteration += 1
@@ -106,37 +105,6 @@ def md5_for_file_hex(
     return md5_for_file(filename, block_size, callback).hexdigest()
 
 
-async def md5_for_file_multiprocessing(
-    filename: str,
-    process_pool_executor,
-    md5_semaphore: asyncio.Semaphore,
-    block_size: int = 2 * MB,
-) -> str:
-    """
-    Calculates the MD5 of the given file.
-    See source <http://stackoverflow.com/questions/1131220/get-md5-hash-of-a-files-without-open-it-in-python>.
-
-    Arguments:
-        filename: The file to read in
-        process_pool_executor: The process pool executor to use for the calculation.
-        md5_semaphore: The semaphore to use for waiting to calculate.
-        block_size: How much of the file to read in at once (bytes).
-                    Defaults to 2 MB.
-
-    Returns:
-        The MD5 Checksum
-    """
-    async with md5_semaphore:
-        with tracer.start_as_current_span("Utils::md5_for_file_multiprocessing"):
-            future = process_pool_executor.submit(
-                md5_for_file_hex, filename, block_size
-            )
-            while not future.done():
-                await asyncio.sleep(0)
-            result = future.result()
-            return result
-
-
 @tracer.start_as_current_span("Utils::md5_fn")
 def md5_fn(part, _) -> str:
     """Calculate the MD5 of a file-like object.
@@ -147,7 +115,7 @@ def md5_fn(part, _) -> str:
     Returns:
         The MD5 Checksum
     """
-    md5 = hashlib.new("md5", usedforsecurity=False)
+    md5 = hashlib.new("md5", usedforsecurity=False)  # nosec
     md5.update(part)
     return md5.hexdigest()
 

{synapseclient-4.3.0 → synapseclient-4.3.1}/synapseclient/models/file.py

@@ -635,15 +635,7 @@ class File(FileSynchronousProtocol, AccessControllable):
         """Load the MD5 of the file if it's a local file and we have not already loaded
         it."""
         if not self.content_md5 and self.path and os.path.isfile(self.path):
-            self.content_md5 = await utils.md5_for_file_multiprocessing(
-                filename=self.path,
-                process_pool_executor=syn._get_process_pool_executor(
-                    asyncio_event_loop=asyncio.get_running_loop()
-                ),
-                md5_semaphore=syn._get_md5_semaphore(
-                    asyncio_event_loop=asyncio.get_running_loop()
-                ),
-            )
+            self.content_md5 = utils.md5_for_file_hex(filename=self.path)
 
     async def _find_existing_file(
         self, synapse_client: Optional[Synapse] = None

{synapseclient-4.3.0 → synapseclient-4.3.1}/synapseclient/synapsePythonClient

@@ -1,6 +1,6 @@
 {
   "client": "synapsePythonClient",
-  "latestVersion":"4.3.0",
+  "latestVersion":"4.3.1",
   "blacklist": [
     "0.0.0",
     "0.4.1",

{synapseclient-4.3.0 → synapseclient-4.3.1/synapseclient.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapseclient
-Version: 4.3.0
+Version: 4.3.1
 Summary: A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate.
 Home-page: https://www.synapse.org
 Author: The Synapse Engineering Team
@@ -39,7 +39,6 @@ Requires-Dist: nest-asyncio~=1.6.0
 Requires-Dist: asyncio-atexit~=1.0.1
 Requires-Dist: httpx~=0.27.0
 Requires-Dist: tqdm<5.0,>=4.66.2
-Requires-Dist: loky~=3.0.0
 Requires-Dist: async-lru~=2.0.4
 Requires-Dist: psutil~=5.9.8
 Provides-Extra: dev

{synapseclient-4.3.0 → synapseclient-4.3.1}/synapseclient.egg-info/requires.txt

@@ -8,7 +8,6 @@ nest-asyncio~=1.6.0
 asyncio-atexit~=1.0.1
 httpx~=0.27.0
 tqdm<5.0,>=4.66.2
-loky~=3.0.0
 async-lru~=2.0.4
 psutil~=5.9.8