specklia 1.9.16__py3-none-any.whl → 1.9.17__py3-none-any.whl

specklia/chunked_transfer.py

@@ -25,16 +25,21 @@ from http import HTTPStatus
 from io import BytesIO
 from logging import Logger
 import struct
+import time
 from typing import List, Tuple, Union
 
-from geopandas import GeoDataFrame, read_feather as read_geofeather
-from pandas import DataFrame, read_feather
+from geopandas import GeoDataFrame
+from geopandas import read_feather as read_geofeather
+from pandas import DataFrame
+from pandas import read_feather
 import requests
 
 CHUNK_DB_NAME = "data_transfer_chunks"
 CHUNK_METADATA_COLLECTION_NAME = "chunk_metadata"
 MAX_CHUNK_AGE_SECONDS = 3600
 MAX_CHUNK_SIZE_BYTES = 5 * 1024 ** 2  # must be small enough to fit into an HTTP GET Request
+CHUNK_DOWNLOAD_RETRIES = 10
+CHUNK_DOWNLOAD_TIMEOUT_S = 10
 
 
 class ChunkSetStatus(Enum):
@@ -87,7 +92,7 @@ def upload_chunks(api_address: str, chunks: List[Tuple[int, bytes]], logger: Log
     return chunk_set_uuid
 
 
-def download_chunks(api_address: str, chunk_set_uuid: str) -> List[Tuple[int, bytes]]:
+def download_chunks(api_address: str, chunk_set_uuid: str, logger: Logger) -> List[Tuple[int, bytes]]:
     """
     Download data chunks.
 
@@ -100,24 +105,52 @@ def download_chunks(api_address: str, chunk_set_uuid: str) -> List[Tuple[int, by
         The full URL of the API, including port but not including endpoint, e.g. "http://127.0.0.1:9999"
     chunk_set_uuid : str
         The uuid of the chunk set to download.
+    logger : Logger
+        A logger with which to log the download.
 
     Returns
     -------
     chunks : List[Tuple[int, bytes]]
         A list of tuples containing the ordinal number of the chunk and each chunk
+
+    Raises
+    ------
+    RuntimeError
+        If the download fails after a number of retries.
     """
     # fetch the data
     data_chunks = []
     finished = False
+
     while not finished:
-        this_chunk_response = requests.get(api_address + f"/chunk/download/{chunk_set_uuid}")
-        if this_chunk_response.status_code == HTTPStatus.NO_CONTENT:
-            finished = True
-        else:
-            data_chunks.append((
-                struct.unpack('i', this_chunk_response.content[:4])[0],
-                this_chunk_response.content[4:]))
+        retries = 0
+        success = False
 
+        while retries < CHUNK_DOWNLOAD_RETRIES and not success:
+            try:
+                this_chunk_response = requests.get(
+                    f"{api_address}/chunk/download/{chunk_set_uuid}",
+                    timeout=CHUNK_DOWNLOAD_TIMEOUT_S
+                )
+                if this_chunk_response.status_code == HTTPStatus.NO_CONTENT:
+                    finished = True
+                else:
+                    data_chunks.append((
+                        struct.unpack('i', this_chunk_response.content[:4])[0],
+                        this_chunk_response.content[4:]))
+                success = True
+
+            except (requests.Timeout, requests.ConnectionError) as e:
+                retries += 1
+                logger.warning(
+                    "Request failed with %s. Retrying (%s/%s)...", e, retries, CHUNK_DOWNLOAD_RETRIES)
+                time.sleep(1)  # Small backoff before retrying
+
+        if not success:
+            error_message = (
+                f"Failed to download from chunk set {chunk_set_uuid} after {CHUNK_DOWNLOAD_TIMEOUT_S} attempts.")
+            logger.error(error_message)
+            raise RuntimeError(error_message)
     return data_chunks
 
 

specklia/client.py

@@ -232,7 +232,7 @@ class Specklia:
             gdf = chunked_transfer.deserialise_dataframe(
                 chunked_transfer.merge_from_chunks(
                     chunked_transfer.download_chunks(
-                        self.server_url, response_dict['chunk_set_uuid'])))
+                        self.server_url, response_dict['chunk_set_uuid'], _log)))
         else:
             gdf = gpd.GeoDataFrame()
 

{specklia-1.9.16.dist-info → specklia-1.9.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: specklia
-Version: 1.9.16
+Version: 1.9.17
 Summary: Python client for Specklia, a geospatial point cloud database by Earthwave.
 Home-page: https://specklia.earthwave.co.uk/
 Author: Earthwave Ltd

specklia-1.9.17.dist-info/RECORD

@@ -0,0 +1,9 @@
+specklia/__init__.py,sha256=ePVHqq642NocoE8tS0cNTd0B5wJdUB7r3y815oQXD6A,51
+specklia/chunked_transfer.py,sha256=Mw0wLaOYzNvVTRE421RdHnsnaDbEpNYgQNGV_kcjuVs,8327
+specklia/client.py,sha256=P4XXzC24OuLK5S6joc7d0zt9Q39-IveAK87EUCF7n3M,41858
+specklia/utilities.py,sha256=0_pgTbcq2RgQhys0-CZ6h5YZJg9ZMPhD_ibGPggFUpE,5018
+specklia-1.9.17.dist-info/LICENCE,sha256=kjWTA-TtT_rJtsWuAgWvesvu01BytVXgt_uCbeQgjOg,1061
+specklia-1.9.17.dist-info/METADATA,sha256=AUV3Mor8hxiA330rCnyWHMhpgf5uq7MKoEULcDpw2xE,3082
+specklia-1.9.17.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+specklia-1.9.17.dist-info/top_level.txt,sha256=XgU53UpAJbqEni5EjJaPdQPYuNx16Geg2I5A9lo1BQw,9
+specklia-1.9.17.dist-info/RECORD,,

specklia-1.9.16.dist-info/RECORD

@@ -1,9 +0,0 @@
-specklia/__init__.py,sha256=ePVHqq642NocoE8tS0cNTd0B5wJdUB7r3y815oQXD6A,51
-specklia/chunked_transfer.py,sha256=hO7luSNjznsH-8s585PFNks1agn3cj6v_Sxg_nLVdM8,7179
-specklia/client.py,sha256=6grKdJwtRUsj5u3I91rbJFVXvLONS-CoPNy9BUr6LVw,41852
-specklia/utilities.py,sha256=0_pgTbcq2RgQhys0-CZ6h5YZJg9ZMPhD_ibGPggFUpE,5018
-specklia-1.9.16.dist-info/LICENCE,sha256=kjWTA-TtT_rJtsWuAgWvesvu01BytVXgt_uCbeQgjOg,1061
-specklia-1.9.16.dist-info/METADATA,sha256=rNtsnqyOI9BLCe8LMLBKZj1wF9ZyG4R_9qWg1tFAvcs,3082
-specklia-1.9.16.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-specklia-1.9.16.dist-info/top_level.txt,sha256=XgU53UpAJbqEni5EjJaPdQPYuNx16Geg2I5A9lo1BQw,9
-specklia-1.9.16.dist-info/RECORD,,