learning-loop-node 0.10.9__py3-none-any.whl → 0.10.10__py3-none-any.whl

learning_loop_node/detector/outbox.py

@@ -14,9 +14,9 @@ from multiprocessing import Event
 from multiprocessing.synchronize import Event as SyncEvent
 from typing import List, Optional, Tuple, Union
 
+import aiohttp
 import PIL
 import PIL.Image  # type: ignore
-import requests
 from fastapi.encoders import jsonable_encoder
 
 from ..data_classes import Detections
@@ -64,7 +64,10 @@ class Outbox():
             detections = Detections()
         if not tags:
             tags = []
-        identifier = datetime.now().isoformat(sep='_', timespec='milliseconds')
+        identifier = datetime.now().isoformat(sep='_', timespec='microseconds')
+        if os.path.exists(self.path + '/' + identifier):
+            self.log.error('Directory with identifier %s already exists', identifier)
+            return
         tmp = f'{GLOBALS.data_folder}/tmp/{identifier}'
         detections.tags = tags
         detections.date = identifier
@@ -97,11 +100,11 @@ class Outbox():
         self.log.info('continuous upload started')
         assert self.shutdown_event is not None
         while not self.shutdown_event.is_set():
-            self.upload()
+            await self.upload()
             await asyncio.sleep(5)
         self.log.info('continuous upload ended')
 
-    def upload(self):
+    async def upload(self):
         items = self.get_data_files()
         if not items:
             self.log.debug('No images found to upload')
@@ -113,11 +116,11 @@ class Outbox():
             if self.shutdown_event.is_set():
                 break
             try:
-                self._upload_batch(batch_items)
+                await self._upload_batch(batch_items)
             except Exception:
                 self.log.exception('Could not upload files')
 
-    def _upload_batch(self, items: List[str]):
+    async def _upload_batch(self, items: List[str]):
 
         # NOTE: keys are not relevant for the server, but using a fixed key like 'files'
         # results in a post failure on the first run of the test in a docker environment (WTF)
@@ -127,7 +130,8 @@ class Outbox():
         data += [('files', open(f'{item}/image.jpg', 'rb')) for item in items]
 
         try:
-            response = requests.post(self.target_uri, files=data, timeout=self.UPLOAD_TIMEOUT_S)
+            async with aiohttp.ClientSession() as session:
+                response = await session.post(self.target_uri, data=data, timeout=self.UPLOAD_TIMEOUT_S)
         except Exception:
             self.log.exception('Could not upload images')
             return
@@ -136,21 +140,21 @@ class Outbox():
             for _, file in data:
                 file.close()
 
-        if response.status_code == 200:
+        if response.status == 200:
             self.upload_counter += len(items)
             for item in items:
                 shutil.rmtree(item, ignore_errors=True)
             self.log.info('Uploaded %s images successfully', len(items))
 
-        elif response.status_code == 422:
+        elif response.status == 422:
             if len(items) == 1:
                 self.log.error('Broken content in image: %s\n Skipping.', items[0])
                 shutil.rmtree(items[0], ignore_errors=True)
                 return
 
             self.log.exception('Broken content in batch. Splitting and retrying')
-            self._upload_batch(items[:len(items)//2])
-            self._upload_batch(items[len(items)//2:])
+            await self._upload_batch(items[:len(items)//2])
+            await self._upload_batch(items[len(items)//2:])
         else:
             self.log.error('Could not upload images: %s', response.content)
 

learning_loop_node/tests/detector/test_client_communication.py

@@ -121,3 +121,24 @@ async def test_rest_outbox_mode(test_detector_node: DetectorNode):
     check_switch_to_mode('stopped')
     check_switch_to_mode('continuous_upload')
     check_switch_to_mode('stopped')
+
+
+async def test_api_responsive_during_large_upload(test_detector_node: DetectorNode):
+    assert len(get_outbox_files(test_detector_node.outbox)) == 0
+
+    with open(test_image_path, 'rb') as f:
+        image_bytes = f.read()
+
+    for _ in range(100):
+        test_detector_node.outbox.save(image_bytes)
+
+    outbox_size_early = len(get_outbox_files(test_detector_node.outbox))
+    await asyncio.sleep(5)  # NOTE: we wait 5 seconds because the continuous upload is running every 5 seconds
+
+    # check if api is still responsive
+    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/outbox_mode', timeout=2)
+    assert response.status_code == 200
+
+    await asyncio.sleep(5)
+    outbox_size_late = len(get_outbox_files(test_detector_node.outbox))
+    assert outbox_size_early > outbox_size_late > 0, 'The outbox should have been partially emptied.'

learning_loop_node/tests/detector/test_outbox.py

@@ -52,7 +52,7 @@ async def test_outbox_upload_is_successful(test_outbox: Outbox):
     await asyncio.sleep(1)
     test_outbox.save(get_test_image_binary())
     assert await wait_for_outbox_count(test_outbox, 2)
-    test_outbox.upload()
+    await test_outbox.upload()
     assert await wait_for_outbox_count(test_outbox, 0)
     assert test_outbox.upload_counter == 2
 

{learning_loop_node-0.10.9.dist-info → learning_loop_node-0.10.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.9
+Version: 0.10.10
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.10.9.dist-info → learning_loop_node-0.10.10.dist-info}/RECORD

@@ -15,7 +15,7 @@ learning_loop_node/detector/detector_node.py,sha256=jaz4TiHNVFd8p7NQ6Zcrsro9c-X9
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=TD346I9ymtIP0_CJXCIKMRuiXbfVVanXNu_iHAwDd7Q,3318
 learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=s2FuwZ-tD_5obkSutstjc8pE_hLGbrv9WjrEO9t8rJ8,1011
-learning_loop_node/detector/outbox.py,sha256=kRC_ZSvNNxEeJg0qmImBWNiKYjcgHiPAmRQB4VlQ-Uk,7828
+learning_loop_node/detector/outbox.py,sha256=MHHP4rnGaV8JxDSig96KZN4hSQ3i9z6-7WYhTTrMtp0,8082
 learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/rest/about.py,sha256=-PNqlQI_tzRvoSI_UR9rX8-5GeiENNpRDQ4Ylw3wYVs,607
 learning_loop_node/detector/rest/backdoor_controls.py,sha256=38axRG66Z3_Q6bYKa7Hw-ldChEAu-dJcBM_Sl_17Ozo,1725
@@ -46,8 +46,8 @@ learning_loop_node/tests/detector/inbox_filter/test_relevance_group.py,sha256=Xj
 learning_loop_node/tests/detector/inbox_filter/test_unexpected_observations_count.py,sha256=MWC7PbaCy14jjRw0_oilkXj6gymAsUZXHJdzNW5m2D4,1639
 learning_loop_node/tests/detector/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/detector/test.jpg,sha256=msA-vHPmvPiro_D102Qmn1fn4vNfooqYYEXPxZUmYpk,161390
-learning_loop_node/tests/detector/test_client_communication.py,sha256=fnTyzP3XV1i6YwB9-GEDj2ifc_Fr_FtnjDhPoRYX2Ec,5261
-learning_loop_node/tests/detector/test_outbox.py,sha256=1sqTLRrCkUsoYz67Z1L03wIet1N3Nxlil3b5VLEK644,2995
+learning_loop_node/tests/detector/test_client_communication.py,sha256=9r1LULGmRqKihRgG0v7-CkQryRCsyyahoLy8QyjsOTU,6128
+learning_loop_node/tests/detector/test_outbox.py,sha256=5RMKQfuu1-rvpVCpEtt_D70bYgma-sIrTHWxHdTdU9Y,3001
 learning_loop_node/tests/detector/test_relevance_filter.py,sha256=3VLhHKaxPzLYmiNZagvgg9ZHkPhWk4_-qpmkJw36wBU,2046
 learning_loop_node/tests/detector/testing_detector.py,sha256=FeQroV85IvsT8dmalQBqf1FLNt_buCtZK3-lgtmbrBI,542
 learning_loop_node/tests/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -88,6 +88,6 @@ learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvg
 learning_loop_node/trainer/trainer_logic.py,sha256=PJxiO1chPdvpq8UTtzv_nVam9CouCswX9b1FnRwT2Tw,8411
 learning_loop_node/trainer/trainer_logic_generic.py,sha256=AzllMMiUPP_CMkjIVqse8wY50Cg5RDnk5y5ERVUjtZg,25801
 learning_loop_node/trainer/trainer_node.py,sha256=dV-kcTIxhHsep_xIXdGc_AaeJM1mFlQNnwUpTkG4btg,4110
-learning_loop_node-0.10.9.dist-info/METADATA,sha256=DaJ_D49CGjlIbcX1xagMN3DPRxasYxFWiZBfDJqph5A,10383
-learning_loop_node-0.10.9.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.10.9.dist-info/RECORD,,
+learning_loop_node-0.10.10.dist-info/METADATA,sha256=d4e08fjXLZ4nieGSIE97nLYl1sapDta_W4mA-Uh05oM,10384
+learning_loop_node-0.10.10.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.10.10.dist-info/RECORD,,