specklia 1.9.38__py3-none-any.whl → 1.9.39__py3-none-any.whl

specklia/chunked_transfer.py

@@ -41,6 +41,8 @@ MAX_CHUNK_SIZE_BYTES = 5 * 1024 ** 2  # must be small enough to fit into an HTTP
 CHUNK_DOWNLOAD_RETRIES = 10
 CHUNK_DOWNLOAD_TIMEOUT_S = 10
 
+log = Logger(__name__)
+
 
 class ChunkSetStatus(Enum):
     """
@@ -53,7 +55,7 @@ class ChunkSetStatus(Enum):
     EMPTYING = 1
 
 
-def upload_chunks(api_address: str, chunks: List[Tuple[int, bytes]], logger: Logger) -> str:
+def upload_chunks(api_address: str, chunks: List[Tuple[int, bytes]]) -> str:
     """
     Upload data chunks.
 
@@ -66,8 +68,6 @@ def upload_chunks(api_address: str, chunks: List[Tuple[int, bytes]], logger: Log
         The full URL of the API, including port but not including endpoint, e.g. "http://127.0.0.1:9999"
     chunks : List[Tuple[int, bytes]]
         A list of tuples containing the ordinal number of the chunk and each chunk
-    logger : Logger
-        A logger with which to log the upload.
 
     Returns
     -------
@@ -78,7 +78,7 @@ def upload_chunks(api_address: str, chunks: List[Tuple[int, bytes]], logger: Log
     response = requests.post(
         api_address + f"/chunk/upload/{chunks[0][0]}-of-{len(chunks)}",
         data=chunks[0][1])
-    logger.info("response from very first /chunk/upload was '%s'", response.json())
+    log.info("response from very first /chunk/upload was '%s'", response.json())
     assert response.status_code == HTTPStatus.OK, response.text
     chunk_set_uuid = response.json()['chunk_set_uuid']
 
@@ -86,18 +86,17 @@ def upload_chunks(api_address: str, chunks: List[Tuple[int, bytes]], logger: Log
     for i, chunk in chunks[1:]:
         response = requests.post(
             api_address + f"/chunk/upload/{chunk_set_uuid}/{i}-of-{len(chunks)}", data=chunk)
-        logger.info("response from subsequent /chunk/upload/uuid call was '%s'", response.text)
+        log.info("response from subsequent /chunk/upload/uuid call was '%s'", response.text)
         assert response.status_code == HTTPStatus.OK, response.text
 
     return chunk_set_uuid
 
 
-def download_chunks(api_address: str, chunk_set_uuid: str, logger: Logger) -> List[Tuple[int, bytes]]:
+def download_chunks(api_address: str, chunk_set_uuid: str, num_chunks: int) -> bytes:
     """
     Download data chunks.
 
-    Download a series of data chunks through the chunked transfer mechanism.
-    This method is for use on the client, not the server.
+    Download a series of data chunks sequentially through the chunked transfer mechanism.
 
     Parameters
     ----------
@@ -105,53 +104,51 @@ def download_chunks(api_address: str, chunk_set_uuid: str, logger: Logger) -> Li
         The full URL of the API, including port but not including endpoint, e.g. "http://127.0.0.1:9999"
     chunk_set_uuid : str
         The uuid of the chunk set to download.
-    logger : Logger
-        A logger with which to log the download.
+    num_chunks : int
+        The number of chunks to download.
 
     Returns
     -------
-    chunks : List[Tuple[int, bytes]]
-        A list of tuples containing the ordinal number of the chunk and each chunk
+    bytes
+        The concatenated data from all the chunks.
 
     Raises
     ------
     RuntimeError
         If the download fails after a number of retries.
     """
-    # fetch the data
-    data_chunks = []
-    finished = False
-
-    while not finished:
+    chunks = []
+    for chunk_ordinal in range(1, num_chunks + 1):
         retries = 0
         success = False
-
         while retries < CHUNK_DOWNLOAD_RETRIES and not success:
             try:
                 this_chunk_response = requests.get(
-                    f"{api_address}/chunk/download/{chunk_set_uuid}",
+                    f"{api_address}/chunk/download/{chunk_set_uuid}/{chunk_ordinal}",
                     timeout=CHUNK_DOWNLOAD_TIMEOUT_S
                 )
-                if this_chunk_response.status_code == HTTPStatus.NO_CONTENT:
-                    finished = True
-                else:
-                    data_chunks.append((
-                        struct.unpack('i', this_chunk_response.content[:4])[0],
-                        this_chunk_response.content[4:]))
+                this_chunk_response.raise_for_status()
+                ordinal = struct.unpack('i', this_chunk_response.content[:4])[0]
+                chunk = this_chunk_response.content[4:]
+                assert ordinal == chunk_ordinal, (
+                    f"Chunk ordinal mismatch: expected {chunk_ordinal}, got {ordinal}")
+                chunks.append(chunk)
                 success = True
-
             except (requests.Timeout, requests.ConnectionError) as e:
                 retries += 1
-                logger.warning(
+                log.warning(
                     "Request failed with %s. Retrying (%s/%s)...", e, retries, CHUNK_DOWNLOAD_RETRIES)
                 time.sleep(1)  # Small backoff before retrying
-
         if not success:
             error_message = (
                 f"Failed to download from chunk set {chunk_set_uuid} after {CHUNK_DOWNLOAD_TIMEOUT_S} attempts.")
-            logger.error(error_message)
+            log.error(error_message)
             raise RuntimeError(error_message)
-    return data_chunks
+
+    # Let the server know that we are done with this data and it can be deleted.
+    requests.delete(f'{api_address}/chunk/delete/{chunk_set_uuid}')
+
+    return b''.join(chunks)
 
 
 def split_into_chunks(data: bytes, chunk_size: int = MAX_CHUNK_SIZE_BYTES) -> List[Tuple[int, bytes]]:
@@ -174,23 +171,6 @@ def split_into_chunks(data: bytes, chunk_size: int = MAX_CHUNK_SIZE_BYTES) -> Li
         enumerate((data[i:i + chunk_size] for i in range(0, len(data), chunk_size)), start=1))
 
 
-def merge_from_chunks(chunks: List[Tuple[int, bytes]]) -> bytes:
-    """
-    Merge data that has been split into compressed chunks back into a single message.
-
-    Parameters
-    ----------
-    chunks : List[Tuple[int, bytes]]
-        A list of tuples containing the ordinal number of the chunk and each chunk
-
-    Returns
-    -------
-    bytes
-        The merged data
-    """
-    return b''.join([dc[1] for dc in sorted(chunks, key=lambda x: x[0])])
-
-
 def deserialise_dataframe(data: bytes) -> Union[DataFrame, GeoDataFrame]:
     """
     Convert a binary serialised feather table to pandas dataframe.
@@ -223,7 +203,7 @@ def deserialise_dataframe(data: bytes) -> Union[DataFrame, GeoDataFrame]:
                 raise ValueError("Couldn't deserialise table format") from e
         else:
             raise ValueError("Couldn't deserialise table format") from e
-    return df
+    return df  # type: ignore
 
 
 def serialise_dataframe(df: Union[DataFrame, GeoDataFrame]) -> bytes:

specklia/client.py

@@ -231,9 +231,12 @@ class Specklia:
         # stream and deserialise the results
         if response_dict['num_chunks'] > 0:
             gdf = chunked_transfer.deserialise_dataframe(
-                chunked_transfer.merge_from_chunks(
-                    chunked_transfer.download_chunks(
-                        self.server_url, response_dict['chunk_set_uuid'], _log)))
+                chunked_transfer.download_chunks(
+                    self.server_url,
+                    response_dict['chunk_set_uuid'],
+                    response_dict['num_chunks'],
+                )
+            )
         else:
             gdf = gpd.GeoDataFrame()
 
@@ -312,12 +315,14 @@ class Specklia:
         # serialise and upload each dataframe
         upload_points = []
         for n in new_points:
+            chunks = chunked_transfer.split_into_chunks(
+                chunked_transfer.serialise_dataframe(n['gdf']))
             chunk_set_uuid = chunked_transfer.upload_chunks(
-                self.server_url, chunked_transfer.split_into_chunks(
-                    chunked_transfer.serialise_dataframe(n['gdf'])), _log)
+                self.server_url, chunks)
             upload_points.append({
                 'source': n['source'],
-                'chunk_set_uuid': chunk_set_uuid
+                'chunk_set_uuid': chunk_set_uuid,
+                'num_chunks': len(chunks),
             })
             del n
 

{specklia-1.9.38.dist-info → specklia-1.9.39.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: specklia
-Version: 1.9.38
+Version: 1.9.39
 Summary: Python client for Specklia, a geospatial point cloud database by Earthwave.
 Home-page: https://specklia.earthwave.co.uk/
 Author: Earthwave Ltd

specklia-1.9.39.dist-info/RECORD

@@ -0,0 +1,9 @@
+specklia/__init__.py,sha256=ePVHqq642NocoE8tS0cNTd0B5wJdUB7r3y815oQXD6A,51
+specklia/chunked_transfer.py,sha256=H7JUJqi_e8ftSPURG-B59900-S23PXFGXBrH3_lfRBw,7910
+specklia/client.py,sha256=ujSkx62VIuOJ3FfTon7rAztXkzZCQl5J6QEmFwK8aP8,42183
+specklia/utilities.py,sha256=fs9DOSq-0hdgOlGAnPY_og5QngDcu3essVAupz6ychM,5170
+specklia-1.9.39.dist-info/LICENCE,sha256=kjWTA-TtT_rJtsWuAgWvesvu01BytVXgt_uCbeQgjOg,1061
+specklia-1.9.39.dist-info/METADATA,sha256=rBIu805Ndrde3QgrZIIvwWHaf4mC3cdmubphHfujJj8,3082
+specklia-1.9.39.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+specklia-1.9.39.dist-info/top_level.txt,sha256=XgU53UpAJbqEni5EjJaPdQPYuNx16Geg2I5A9lo1BQw,9
+specklia-1.9.39.dist-info/RECORD,,

specklia-1.9.38.dist-info/RECORD

@@ -1,9 +0,0 @@
-specklia/__init__.py,sha256=ePVHqq642NocoE8tS0cNTd0B5wJdUB7r3y815oQXD6A,51
-specklia/chunked_transfer.py,sha256=sQGh_0M3DcJw4Uu1Upb7zA9L1syupI9JXVL_94OoNEs,8343
-specklia/client.py,sha256=IlOr3CP_0BJpB0oOvkvHFgtiqmmFynpUriDM3NHUVeU,42088
-specklia/utilities.py,sha256=fs9DOSq-0hdgOlGAnPY_og5QngDcu3essVAupz6ychM,5170
-specklia-1.9.38.dist-info/LICENCE,sha256=kjWTA-TtT_rJtsWuAgWvesvu01BytVXgt_uCbeQgjOg,1061
-specklia-1.9.38.dist-info/METADATA,sha256=Tc8LwvdAQPpbxOSzcZiTQmICzBo_jhuMvzbMvYfdOCA,3082
-specklia-1.9.38.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-specklia-1.9.38.dist-info/top_level.txt,sha256=XgU53UpAJbqEni5EjJaPdQPYuNx16Geg2I5A9lo1BQw,9
-specklia-1.9.38.dist-info/RECORD,,