specklia 1.9.16__tar.gz → 1.9.17__tar.gz

{specklia-1.9.16 → specklia-1.9.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: specklia
-Version: 1.9.16
+Version: 1.9.17
 Summary: Python client for Specklia, a geospatial point cloud database by Earthwave.
 Home-page: https://specklia.earthwave.co.uk/
 Author: Earthwave Ltd

{specklia-1.9.16 → specklia-1.9.17}/specklia/chunked_transfer.py

@@ -25,16 +25,21 @@ from http import HTTPStatus
 from io import BytesIO
 from logging import Logger
 import struct
+import time
 from typing import List, Tuple, Union
 
-from geopandas import GeoDataFrame, read_feather as read_geofeather
-from pandas import DataFrame, read_feather
+from geopandas import GeoDataFrame
+from geopandas import read_feather as read_geofeather
+from pandas import DataFrame
+from pandas import read_feather
 import requests
 
 CHUNK_DB_NAME = "data_transfer_chunks"
 CHUNK_METADATA_COLLECTION_NAME = "chunk_metadata"
 MAX_CHUNK_AGE_SECONDS = 3600
 MAX_CHUNK_SIZE_BYTES = 5 * 1024 ** 2  # must be small enough to fit into an HTTP GET Request
+CHUNK_DOWNLOAD_RETRIES = 10
+CHUNK_DOWNLOAD_TIMEOUT_S = 10
 
 
 class ChunkSetStatus(Enum):
@@ -87,7 +92,7 @@ def upload_chunks(api_address: str, chunks: List[Tuple[int, bytes]], logger: Log
     return chunk_set_uuid
 
 
-def download_chunks(api_address: str, chunk_set_uuid: str) -> List[Tuple[int, bytes]]:
+def download_chunks(api_address: str, chunk_set_uuid: str, logger: Logger) -> List[Tuple[int, bytes]]:
     """
     Download data chunks.
 
@@ -100,24 +105,52 @@ def download_chunks(api_address: str, chunk_set_uuid: str) -> List[Tuple[int, by
         The full URL of the API, including port but not including endpoint, e.g. "http://127.0.0.1:9999"
     chunk_set_uuid : str
         The uuid of the chunk set to download.
+    logger : Logger
+        A logger with which to log the download.
 
     Returns
     -------
     chunks : List[Tuple[int, bytes]]
         A list of tuples containing the ordinal number of the chunk and each chunk
+
+    Raises
+    ------
+    RuntimeError
+        If the download fails after a number of retries.
     """
     # fetch the data
     data_chunks = []
     finished = False
+
     while not finished:
-        this_chunk_response = requests.get(api_address + f"/chunk/download/{chunk_set_uuid}")
-        if this_chunk_response.status_code == HTTPStatus.NO_CONTENT:
-            finished = True
-        else:
-            data_chunks.append((
-                struct.unpack('i', this_chunk_response.content[:4])[0],
-                this_chunk_response.content[4:]))
+        retries = 0
+        success = False
 
+        while retries < CHUNK_DOWNLOAD_RETRIES and not success:
+            try:
+                this_chunk_response = requests.get(
+                    f"{api_address}/chunk/download/{chunk_set_uuid}",
+                    timeout=CHUNK_DOWNLOAD_TIMEOUT_S
+                )
+                if this_chunk_response.status_code == HTTPStatus.NO_CONTENT:
+                    finished = True
+                else:
+                    data_chunks.append((
+                        struct.unpack('i', this_chunk_response.content[:4])[0],
+                        this_chunk_response.content[4:]))
+                success = True
+
+            except (requests.Timeout, requests.ConnectionError) as e:
+                retries += 1
+                logger.warning(
+                    "Request failed with %s. Retrying (%s/%s)...", e, retries, CHUNK_DOWNLOAD_RETRIES)
+                time.sleep(1)  # Small backoff before retrying
+
+        if not success:
+            error_message = (
+                f"Failed to download from chunk set {chunk_set_uuid} after {CHUNK_DOWNLOAD_TIMEOUT_S} attempts.")
+            logger.error(error_message)
+            raise RuntimeError(error_message)
     return data_chunks
 
 

{specklia-1.9.16 → specklia-1.9.17}/specklia/client.py

@@ -232,7 +232,7 @@ class Specklia:
             gdf = chunked_transfer.deserialise_dataframe(
                 chunked_transfer.merge_from_chunks(
                     chunked_transfer.download_chunks(
-                        self.server_url, response_dict['chunk_set_uuid'])))
+                        self.server_url, response_dict['chunk_set_uuid'], _log)))
         else:
             gdf = gpd.GeoDataFrame()
 

{specklia-1.9.16 → specklia-1.9.17}/specklia.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: specklia
-Version: 1.9.16
+Version: 1.9.17
 Summary: Python client for Specklia, a geospatial point cloud database by Earthwave.
 Home-page: https://specklia.earthwave.co.uk/
 Author: Earthwave Ltd

{specklia-1.9.16 → specklia-1.9.17}/tests/test_chunked_transfer.py

@@ -40,7 +40,8 @@ def test_download_chunks():
             MagicMock(name="mock_response_2", status_code=HTTPStatus.OK, content=struct.pack('i', 2) + b'wobble'),
             MagicMock(name="mock_response_3", status_code=HTTPStatus.NO_CONTENT, content=b'')]
 
-        assert chunked_transfer.download_chunks(api_address='wibble', chunk_set_uuid='rawr') == [
+        assert chunked_transfer.download_chunks(
+            api_address='wibble', chunk_set_uuid='rawr', logger=MagicMock(name="mock_logger")) == [
             (1, b'wibble'), (2, b'wobble')]