learning-loop-node 0.10.9__tar.gz → 0.10.10__tar.gz

{learning_loop_node-0.10.9 → learning_loop_node-0.10.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.9
+Version: 0.10.10
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.10.9 → learning_loop_node-0.10.10}/learning_loop_node/detector/outbox.py

@@ -14,9 +14,9 @@ from multiprocessing import Event
 from multiprocessing.synchronize import Event as SyncEvent
 from typing import List, Optional, Tuple, Union
 
+import aiohttp
 import PIL
 import PIL.Image  # type: ignore
-import requests
 from fastapi.encoders import jsonable_encoder
 
 from ..data_classes import Detections
@@ -64,7 +64,10 @@ class Outbox():
             detections = Detections()
         if not tags:
             tags = []
-        identifier = datetime.now().isoformat(sep='_', timespec='milliseconds')
+        identifier = datetime.now().isoformat(sep='_', timespec='microseconds')
+        if os.path.exists(self.path + '/' + identifier):
+            self.log.error('Directory with identifier %s already exists', identifier)
+            return
         tmp = f'{GLOBALS.data_folder}/tmp/{identifier}'
         detections.tags = tags
         detections.date = identifier
@@ -97,11 +100,11 @@ class Outbox():
         self.log.info('continuous upload started')
         assert self.shutdown_event is not None
         while not self.shutdown_event.is_set():
-            self.upload()
+            await self.upload()
             await asyncio.sleep(5)
         self.log.info('continuous upload ended')
 
-    def upload(self):
+    async def upload(self):
         items = self.get_data_files()
         if not items:
             self.log.debug('No images found to upload')
@@ -113,11 +116,11 @@ class Outbox():
             if self.shutdown_event.is_set():
                 break
             try:
-                self._upload_batch(batch_items)
+                await self._upload_batch(batch_items)
             except Exception:
                 self.log.exception('Could not upload files')
 
-    def _upload_batch(self, items: List[str]):
+    async def _upload_batch(self, items: List[str]):
 
         # NOTE: keys are not relevant for the server, but using a fixed key like 'files'
         # results in a post failure on the first run of the test in a docker environment (WTF)
@@ -127,7 +130,8 @@ class Outbox():
         data += [('files', open(f'{item}/image.jpg', 'rb')) for item in items]
 
         try:
-            response = requests.post(self.target_uri, files=data, timeout=self.UPLOAD_TIMEOUT_S)
+            async with aiohttp.ClientSession() as session:
+                response = await session.post(self.target_uri, data=data, timeout=self.UPLOAD_TIMEOUT_S)
         except Exception:
             self.log.exception('Could not upload images')
             return
@@ -136,21 +140,21 @@ class Outbox():
             for _, file in data:
                 file.close()
 
-        if response.status_code == 200:
+        if response.status == 200:
             self.upload_counter += len(items)
             for item in items:
                 shutil.rmtree(item, ignore_errors=True)
             self.log.info('Uploaded %s images successfully', len(items))
 
-        elif response.status_code == 422:
+        elif response.status == 422:
             if len(items) == 1:
                 self.log.error('Broken content in image: %s\n Skipping.', items[0])
                 shutil.rmtree(items[0], ignore_errors=True)
                 return
 
             self.log.exception('Broken content in batch. Splitting and retrying')
-            self._upload_batch(items[:len(items)//2])
-            self._upload_batch(items[len(items)//2:])
+            await self._upload_batch(items[:len(items)//2])
+            await self._upload_batch(items[len(items)//2:])
         else:
             self.log.error('Could not upload images: %s', response.content)
 

{learning_loop_node-0.10.9 → learning_loop_node-0.10.10}/learning_loop_node/tests/detector/test_client_communication.py

@@ -121,3 +121,24 @@ async def test_rest_outbox_mode(test_detector_node: DetectorNode):
     check_switch_to_mode('stopped')
     check_switch_to_mode('continuous_upload')
     check_switch_to_mode('stopped')
+
+
+async def test_api_responsive_during_large_upload(test_detector_node: DetectorNode):
+    assert len(get_outbox_files(test_detector_node.outbox)) == 0
+
+    with open(test_image_path, 'rb') as f:
+        image_bytes = f.read()
+
+    for _ in range(100):
+        test_detector_node.outbox.save(image_bytes)
+
+    outbox_size_early = len(get_outbox_files(test_detector_node.outbox))
+    await asyncio.sleep(5)  # NOTE: we wait 5 seconds because the continuous upload is running every 5 seconds
+
+    # check if api is still responsive
+    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/outbox_mode', timeout=2)
+    assert response.status_code == 200
+
+    await asyncio.sleep(5)
+    outbox_size_late = len(get_outbox_files(test_detector_node.outbox))
+    assert outbox_size_early > outbox_size_late > 0, 'The outbox should have been partially emptied.'

{learning_loop_node-0.10.9 → learning_loop_node-0.10.10}/learning_loop_node/tests/detector/test_outbox.py

@@ -52,7 +52,7 @@ async def test_outbox_upload_is_successful(test_outbox: Outbox):
     await asyncio.sleep(1)
     test_outbox.save(get_test_image_binary())
     assert await wait_for_outbox_count(test_outbox, 2)
-    test_outbox.upload()
+    await test_outbox.upload()
     assert await wait_for_outbox_count(test_outbox, 0)
     assert test_outbox.upload_counter == 2
 

{learning_loop_node-0.10.9 → learning_loop_node-0.10.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "learning_loop_node"
-version = "v0.10.9"
+version = "v0.10.10"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = ["Zauberzeug GmbH <info@zauberzeug.com>"]
 license = "MIT"