learning-loop-node 0.9.3__py3-none-any.whl → 0.10.1__py3-none-any.whl

learning_loop_node/data_exchanger.py

@@ -2,23 +2,19 @@ import asyncio
 import logging
 import os
 import shutil
-import time
 import zipfile
 from glob import glob
 from http import HTTPStatus
 from io import BytesIO
-from time import perf_counter
+from time import time
 from typing import Dict, List, Optional
 
-import aiofiles
-from tqdm.asyncio import tqdm
+import aiofiles  # type: ignore
 
 from .data_classes import Context
-from .helpers.misc import create_resource_paths, create_task
+from .helpers.misc import create_resource_paths, create_task, is_valid_image
 from .loop_communication import LoopCommunicator
 
-check_jpeg = shutil.which('jpeginfo') is not None
-
 
 class DownloadError(Exception):
 
@@ -26,201 +22,151 @@ class DownloadError(Exception):
         super().__init__(*args)
         self.cause = cause
 
+    def __str__(self) -> str:
+        return f'DownloadError: {self.cause}'
+
 
 class DataExchanger():
 
     def __init__(self, context: Optional[Context], loop_communicator: LoopCommunicator):
-        self.context = context
+        """Exchanges data with the learning loop via the loop_communicator (rest api).
+
+        Args:
+            context (Optional[Context]): The context of the node. This is the organization and project name.
+            loop_communicator (LoopCommunicator): The loop_communicator to use for communication with the learning loop.
+
+        Note:
+            The context can be set later with the set_context method.
+        """
+        self.set_context(context)
+        self.progress = 0.0
         self.loop_communicator = loop_communicator
+
+        self.check_jpeg = shutil.which('jpeginfo') is not None
+        if self.check_jpeg:
+            logging.info('Detected command line tool "jpeginfo". Images will be checked for validity')
+        else:
+            logging.error('Missing command line tool "jpeginfo". We cannot check for validity of images.')
+
+    def set_context(self, context: Optional[Context]) -> None:
+        self._context = context
         self.progress = 0.0
 
-    def set_context(self, context: Context):
-        self.context = context
+    @property
+    def context(self) -> Context:
+        assert self._context, 'DataExchanger: Context was not set yet.. call set_context() first.'
+        return self._context
 
-    async def fetch_image_ids(self, query_params: Optional[str] = '') -> List[str]:
-        if self.context is None:
-            logging.warning('context was not set yet')
-            return []
+    # ---------------------------- END OF INIT ----------------------------
+
+    async def fetch_image_uuids(self, query_params: Optional[str] = '') -> List[str]:
+        """Fetch image uuids from the learning loop data endpoint."""
+        logging.info(f'Fetching image uuids for {self.context.organization}/{self.context.project}..')
 
         response = await self.loop_communicator.get(f'/{self.context.organization}/projects/{self.context.project}/data?{query_params}')
         assert response.status_code == 200, response
         return (response.json())['image_ids']
 
-    async def download_images_data(self, ids: List[str]) -> List[Dict]:
-        '''Download image annotations etc.'''
-        if self.context is None:
-            logging.warning('context was not set yet')
-            return []
-
-        return await self._download_images_data(self.context.organization, self.context.project, ids)
-
-    async def download_images(self, image_ids: List[str], image_folder: str) -> None:
-        '''Download images. Will skip existing images'''
-        if self.context is None:
-            logging.warning('context was not set yet')
-            return
-
-        new_image_ids = await asyncio.get_event_loop().run_in_executor(None, DataExchanger.filter_existing_images, image_ids, image_folder)
-        paths, ids = create_resource_paths(self.context.organization, self.context.project, new_image_ids)
-        await self._download_images(paths, ids, image_folder)
-
-    @staticmethod
-    async def delete_corrupt_images(image_folder: str) -> None:
-        logging.info('deleting corrupt images')
-        n_deleted = 0
-        for image in glob(f'{image_folder}/*.jpg'):
-            if not await DataExchanger.is_valid_image(image):
-                logging.debug(f'  deleting image {image}')
-                os.remove(image)
-                n_deleted += 1
-
-        logging.info(f'deleted {n_deleted} images')
-
-    @staticmethod
-    def filter_existing_images(all_image_ids, image_folder) -> List[str]:
-        logging.info(f'### Going to filter {len(all_image_ids)} images ids')
-        start = perf_counter()
-        ids = [os.path.splitext(os.path.basename(image))[0]
-               for image in glob(f'{image_folder}/*.jpg')]
-        logging.info(f'found {len(ids)} images on disc')
-        result = [id for id in all_image_ids if id not in ids]
-        end = perf_counter()
-        logging.info(f'calculated {len(result)} new image ids, which took {end-start:0.2f} seconds')
-        return result
-
-    def jepeg_check_info(self):
-        if check_jpeg:
-            logging.info('Detected command line tool "jpeginfo". Images will be checked for validity')
-        else:
-            logging.error('Missing command line tool "jpeginfo". We can not check for validity of images.')
+    async def download_images_data(self, image_uuids: List[str], chunk_size: int = 100) -> List[Dict]:
+        """Download image annotations, tags, set and other information for the given image uuids."""
+        logging.info(f'Fetching annotations, tags, sets, etc. for {len(image_uuids)} images..')
 
-    async def _download_images_data(self, organization: str, project: str, image_ids: List[str], chunk_size: int = 100) -> List[Dict]:
-        logging.info('fetching annotations and other image data')
-        num_image_ids = len(image_ids)
-        self.jepeg_check_info()
-        images_data = []
+        num_image_ids = len(image_uuids)
         if num_image_ids == 0:
             logging.info('got empty list. No images were downloaded')
-            return images_data
-        starttime = time.time()
+            return []
+
         progress_factor = 0.5 / num_image_ids  # 50% of progress is for downloading data
-        for i in tqdm(range(0, num_image_ids, chunk_size), position=0, leave=True):
+        images_data: List[Dict] = []
+        for i in range(0, num_image_ids, chunk_size):
             self.progress = i * progress_factor
-            chunk_ids = image_ids[i:i+chunk_size]
-            response = await self.loop_communicator.get(f'/{organization}/projects/{project}/images?ids={",".join(chunk_ids)}')
+            chunk_ids = image_uuids[i:i+chunk_size]
+            response = await self.loop_communicator.get(f'/{self.context.organization}/projects/{self.context.project}/images?ids={",".join(chunk_ids)}')
             if response.status_code != 200:
-                logging.error(
-                    f'Error during downloading list of images. Statuscode is {response.status_code}')
+                logging.error(f'Error {response.status_code} during downloading image data. Continue with next batch..')
                 continue
             images_data += response.json()['images']
-            total_time = round(time.time() - starttime, 1)
-            if images_data:
-                per100 = total_time / len(images_data) * 100
-                logging.debug(f'[+] Performance: {total_time} sec total. Per 100 : {per100:.1f} sec')
-            else:
-                logging.debug(f'[+] Performance: {total_time} sec total.')
+
         return images_data
 
-    async def _download_images(self, paths: List[str], image_ids: List[str], image_folder: str, chunk_size: int = 10) -> None:
-        num_image_ids = len(image_ids)
-        if num_image_ids == 0:
-            logging.debug('got empty list. No images were downloaded')
+    async def download_images(self, image_uuids: List[str], image_folder: str, chunk_size: int = 10) -> None:
+        """Downloads images (actual image data). Will skip existing images"""
+        logging.info(f'Downloading {len(image_uuids)} images (actual image data).. skipping existing images.')
+        if not image_uuids:
             return
-        logging.info('fetching image files')
-        starttime = time.time()
+
+        existing_uuids = {os.path.splitext(os.path.basename(image))[0] for image in glob(f'{image_folder}/*.jpg')}
+        new_image_uuids = [id for id in image_uuids if id not in existing_uuids]
+
+        paths, _ = create_resource_paths(self.context.organization, self.context.project, new_image_uuids)
+        num_image_ids = len(image_uuids)
         os.makedirs(image_folder, exist_ok=True)
 
         progress_factor = 0.5 / num_image_ids  # second 50% of progress is for downloading images
-        for i in tqdm(range(0, num_image_ids, chunk_size), position=0, leave=True):
+        for i in range(0, num_image_ids, chunk_size):
             self.progress = 0.5 + i * progress_factor
             chunk_paths = paths[i:i+chunk_size]
-            chunk_ids = image_ids[i:i+chunk_size]
+            chunk_ids = image_uuids[i:i+chunk_size]
             tasks = []
             for j, chunk_j in enumerate(chunk_paths):
-                tasks.append(create_task(self.download_one_image(chunk_j, chunk_ids[j], image_folder)))
+                start = time()
+                tasks.append(create_task(self._download_one_image(chunk_j, chunk_ids[j], image_folder)))
+                await asyncio.sleep(max(0, 0.02 - (time() - start)))  # prevent too many requests at once
             await asyncio.gather(*tasks)
-            total_time = round(time.time() - starttime, 1)
-            per100 = total_time / (i + len(tasks)) * 100
-            logging.debug(f'[+] Performance (image files): {total_time} sec total. Per 100 : {per100:.1f}')
 
-    async def download_one_image(self, path: str, image_id: str, image_folder: str) -> None:
+    async def _download_one_image(self, path: str, image_id: str, image_folder: str) -> None:
         response = await self.loop_communicator.get(path)
         if response.status_code != HTTPStatus.OK:
-            logging.error(f'bad status code {response.status_code} for {path}')
+            logging.error(f'bad status code {response.status_code} for {path}. Details: {response.text}')
             return
         filename = f'{image_folder}/{image_id}.jpg'
         async with aiofiles.open(filename, 'wb') as f:
             await f.write(response.content)
-        if not await self.is_valid_image(filename):
+        if not await is_valid_image(filename, self.check_jpeg):
             os.remove(filename)
 
-    @staticmethod
-    async def is_valid_image(filename: str) -> bool:
-        if not os.path.isfile(filename) or os.path.getsize(filename) == 0:
-            return False
-        if not check_jpeg:
-            return True
-
-        info = await asyncio.create_subprocess_shell(
-            f'jpeginfo -c {filename}',
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE)
-        out, _ = await info.communicate()
-        return "OK" in out.decode()
-
-    async def download_model(self, target_folder: str, context: Context, model_id: str, model_format: str) -> List[str]:
-        path = f'/{context.organization}/projects/{context.project}/models/{model_id}/{model_format}/file'
+    async def download_model(self, target_folder: str, context: Context, model_uuid: str, model_format: str) -> List[str]:
+        """Downloads a model (and additional meta data like model.json) and returns the paths of the downloaded files.
+        Used before training a model (when continuing a finished training) or before detecting images.
+        """
+        logging.info(f'Downloading model data for uuid {model_uuid} from the loop to {target_folder}..')
+
+        path = f'/{context.organization}/projects/{context.project}/models/{model_uuid}/{model_format}/file'
         response = await self.loop_communicator.get(path, requires_login=False)
         if response.status_code != 200:
             content = response.json()
-            logging.error(
-                f'could not download {self.loop_communicator.base_url}/{path}: {response.status_code}, content: {content}')
+            logging.error(f'could not download loop/{path}: {response.status_code}, content: {content}')
             raise DownloadError(content['detail'])
         try:
             provided_filename = response.headers.get(
                 "Content-Disposition").split("filename=")[1].strip('"')
             content = response.content
         except:
-            logging.error(f'Error during downloading model {path}:')
-            try:
-                logging.exception(response.json())
-            except Exception:
-                pass
+            logging.exception(f'Error during downloading model {path}:')
             raise
 
-        # unzip and place downloaded model
         tmp_path = f'/tmp/{os.path.splitext(provided_filename)[0]}'
         shutil.rmtree(tmp_path, ignore_errors=True)
         with zipfile.ZipFile(BytesIO(content), 'r') as zip_:
             zip_.extractall(tmp_path)
 
-        logging.info(f'---- downloaded model {model_id} to {tmp_path}.')
-
         created_files = []
-        files = glob(f'{tmp_path}/**/*', recursive=True)
-        for file in files:
+        for file in glob(f'{tmp_path}/**/*', recursive=True):
             new_file = shutil.move(file, target_folder)
-            logging.info(f'moved model file {os.path.basename(file)} to {new_file}.')
             created_files.append(new_file)
-        return created_files
 
-    async def upload_model(self, context: Context, files: List[str], model_id: str, mformat: str) -> None:
-        response = await self.loop_communicator.put(f'/{context.organization}/projects/{context.project}/models/{model_id}/{mformat}/file', files=files)
-        if response.status_code != 200:
-            msg = f'---- could not upload model with id {model_id} and format {mformat}. Details: {response.text}'
-            raise Exception(msg)
-        logging.info(f'---- uploaded model with id {model_id} and format {mformat}.')
+        shutil.rmtree(tmp_path, ignore_errors=True)
+        logging.info(f'Downloaded model {model_uuid}({model_format}) to {target_folder}.')
+        return created_files
 
-    async def upload_model_for_training(self, context: Context, files: List[str], training_number: Optional[int], mformat: str) -> Optional[str]:
-        """Returns the new model uuid to use for detection."""
+    async def upload_model_get_uuid(self, context: Context, files: List[str], training_number: Optional[int], mformat: str) -> Optional[str]:
+        """Used by the trainers. Function returns the new model uuid to use for detection."""
         response = await self.loop_communicator.put(f'/{context.organization}/projects/{context.project}/trainings/{training_number}/models/latest/{mformat}/file', files=files)
         if response.status_code != 200:
-            msg = f'---- could not upload model for training {training_number} and format {mformat}. Details: {response.text}'
-            logging.error(msg)
+            logging.error(f'Could not upload model for training {training_number}, format {mformat}: {response.text}')
             response.raise_for_status()
             return None
-        else:
-            uploaded_model = response.json()
-            logging.info(
-                f'---- uploaded model for training {training_number} and format {mformat}. Model id is {uploaded_model}')
-            return uploaded_model['id']
+
+        uploaded_model = response.json()
+        logging.info(f'Uploaded model for training {training_number}, format {mformat}. Response is: {uploaded_model}')
+        return uploaded_model['id']

learning_loop_node/detector/__init__.py

@@ -1 +0,0 @@
-

learning_loop_node/detector/detector_node.py

@@ -14,7 +14,7 @@ from fastapi.encoders import jsonable_encoder
 from fastapi_socketio import SocketManager
 from socketio import AsyncClient
 
-from ..data_classes import Category, Context, Detections, DetectionStatus, ModelInformation, NodeState, Shape
+from ..data_classes import Category, Context, Detections, DetectionStatus, ModelInformation, Shape
 from ..data_classes.socket_response import SocketResponse
 from ..data_exchanger import DataExchanger, DownloadError
 from ..globals import GLOBALS
@@ -34,9 +34,8 @@ from .rest.operation_mode import OperationMode
 class DetectorNode(Node):
 
     def __init__(self, name: str, detector: DetectorLogic, uuid: Optional[str] = None, use_backdoor_controls: bool = False) -> None:
-        super().__init__(name, uuid)
+        super().__init__(name, uuid, 'detector', False)
         self.detector_logic = detector
-        self.needs_login = False
         self.organization = environment_reader.organization()
         self.project = environment_reader.project()
         assert self.organization and self.project, 'Detector node needs an organization and an project'
@@ -170,6 +169,8 @@ class DetectorNode(Node):
         def _connect(sid, environ, auth) -> None:
             self.connected_clients.append(sid)
 
+        print('>>>>>>>>>>>>>>>>>>>>>>> setting up sio server', flush=True)
+
         self.sio_server = SocketManager(app=self)
         self.sio_server.on('detect', _detect)
         self.sio_server.on('info', _info)
@@ -185,7 +186,9 @@ class DetectorNode(Node):
             if not update_to_model_id:
                 self.log.info('could not check for updates')
                 return
-            if self.detector_logic.is_initialized:  # TODO: solve race condition !!!
+
+            # TODO: solve race condition (it should not be required to recheck if model_info is not None, but it is!)
+            if self.detector_logic.is_initialized:
                 model_info = self.detector_logic._model_info  # pylint: disable=protected-access
                 if model_info is not None:
                     self.log.info(f'Current model: {model_info.version} with id {model_info.id}')
@@ -220,8 +223,7 @@ class DetectorNode(Node):
                     await self.data_exchanger.download_model(target_model_folder,
                                                              Context(organization=self.organization,
                                                                      project=self.project),
-                                                             update_to_model_id,
-                                                             self.detector_logic.model_format)
+                                                             update_to_model_id, self.detector_logic.model_format)
                     try:
                         os.unlink(model_symlink)
                         os.remove(model_symlink)
@@ -256,7 +258,7 @@ class DetectorNode(Node):
             name=self.name,
             state=self.status.state,
             errors=self.status.errors,
-            uptime=int((datetime.now() - self.startup_time).total_seconds()),
+            uptime=int((datetime.now() - self.startup_datetime).total_seconds()),
             operation_mode=self.operation_mode,
             current_model=current_model,
             target_model=self.target_model,
@@ -272,13 +274,11 @@ class DetectorNode(Node):
             return False
 
         assert socket_response.payload is not None
+        # TODO This is weird because target_model_version is stored in self and target_model_id is returned
         self.target_model = socket_response.payload['target_model_version']
         self.log.info(f'After sending status. Target_model is {self.target_model}')
         return socket_response.payload['target_model_id']
 
-    async def get_state(self):
-        return NodeState.Online  # NOTE At the moment only trainer-nodes use a meaningful state
-
     async def set_operation_mode(self, mode: OperationMode):
         self.operation_mode = mode
         await self.send_status()
@@ -353,9 +353,6 @@ class DetectorNode(Node):
             classification_detection.category_id = category_id
         return detections
 
-    def get_node_type(self):
-        return 'detector'
-
     def register_sio_events(self, sio_client: AsyncClient):
         pass
 

learning_loop_node/detector/inbox_filter/cam_observation_history.py

@@ -1,20 +1,17 @@
 import os
 from typing import List, Union
 
-from learning_loop_node.data_classes import (BoxDetection,
-                                             ClassificationDetection,
-                                             Detections, Observation,
-                                             PointDetection,
-                                             SegmentationDetection)
+from learning_loop_node.data_classes import (BoxDetection, ClassificationDetection, Detections, Observation,
+                                             PointDetection, SegmentationDetection)
 
 
 class CamObservationHistory:
-    def __init__(self):
+    def __init__(self) -> None:
         self.reset_time = 3600
         self.recent_observations: List[Observation] = []
         self.iou_threshold = 0.5
 
-    def forget_old_detections(self):
+    def forget_old_detections(self) -> None:
         self.recent_observations = [detection
                                     for detection in self.recent_observations
                                     if not detection.is_older_than(self.reset_time)]

learning_loop_node/detector/outbox.py

@@ -53,7 +53,6 @@ class Outbox():
         with open(tmp + '/image.json', 'w') as f:
             json.dump(jsonable_encoder(asdict(detections)), f)
 
-        # TODO sometimes No such file or directory: '/tmp/learning_loop_lib_data/tmp/2023-09-07_13:27:38.399/image.jpg'
         with open(tmp + '/image.jpg', 'wb') as f:
             f.write(image)
 

learning_loop_node/detector/rest/about.py

@@ -16,6 +16,7 @@ async def get_about(request: Request):
         curl http://localhost/about
     '''
     app: 'DetectorNode' = request.app
+
     return {
         'operation_mode': app.operation_mode.value,
         'state': app.status.state,

learning_loop_node/detector/tests/conftest.py

@@ -12,7 +12,6 @@ import socketio
 import uvicorn
 
 from learning_loop_node import DetectorNode
-from learning_loop_node.data_classes.general import Category, ModelInformation
 from learning_loop_node.detector.outbox import Outbox
 from learning_loop_node.globals import GLOBALS
 

learning_loop_node/detector/tests/test_client_communication.py

@@ -5,7 +5,7 @@ import pytest
 import requests
 
 from learning_loop_node import DetectorNode
-from learning_loop_node.data_classes import Category, ModelInformation
+from learning_loop_node.data_classes import ModelInformation
 from learning_loop_node.detector.tests.conftest import get_outbox_files
 from learning_loop_node.globals import GLOBALS
 
@@ -88,15 +88,17 @@ async def test_sio_upload(test_detector_node: DetectorNode, sio_client):
     assert len(get_outbox_files(test_detector_node.outbox)) == 2, 'There should be one image and one .json file.'
 
 
+# NOTE: This test seems to be flaky.
 async def test_about_endpoint(test_detector_node: DetectorNode):
-    await asyncio.sleep(1)
+    await asyncio.sleep(3)
     response = requests.get(f'http://localhost:{GLOBALS.detector_port}/about', timeout=30)
 
     assert response.status_code == 200
     response_dict = json.loads(response.content)
+    assert response_dict['model_info']
     model_information = ModelInformation.from_dict(response_dict['model_info'])
 
     assert response_dict['operation_mode'] == 'idle'
     assert response_dict['state'] == 'online'
     assert response_dict['target_model'] == '1.1'
-    assert any([c.name == 'purple point' for c in model_information.categories])
+    assert any(c.name == 'purple point' for c in model_information.categories)

learning_loop_node/detector/tests/test_outbox.py

@@ -9,6 +9,8 @@ from learning_loop_node.data_classes import Detections
 from learning_loop_node.detector.detector_node import DetectorNode
 from learning_loop_node.detector.outbox import Outbox
 
+# pylint: disable=redefined-outer-name
+
 
 @pytest.fixture()
 def test_outbox():

learning_loop_node/detector/tests/testing_detector.py

@@ -4,7 +4,7 @@ import numpy as np
 
 from learning_loop_node import DetectorLogic
 from learning_loop_node.conftest import get_dummy_detections
-from learning_loop_node.data_classes import Category, Detections, ModelInformation
+from learning_loop_node.data_classes import Detections
 
 
 class TestingDetectorLogic(DetectorLogic):
@@ -20,10 +20,3 @@ class TestingDetectorLogic(DetectorLogic):
     def evaluate(self, image: np.ndarray) -> Detections:
         logging.info('evaluating')
         return self.det_to_return
-
-        # return Detections(
-        #     box_detections=[BoxDetection(category_name='some_category_name', x=1, y=2, height=3, width=4,
-        #                                  model_name='some_model', confidence=.42, category_id='some_id')],
-        #     point_detections=[PointDetection(category_name='some_category_name_2', x=10, y=12,
-        #                                      model_name='some_model', confidence=.42, category_id='some_id')]
-        # )

learning_loop_node/globals.py

@@ -1,8 +1,8 @@
 
 class Globals():
-    def __init__(self):
+    def __init__(self) -> None:
         self.data_folder: str = '/data'
-        self.detector_port: int = 5004  # TODO move to tests
+        self.detector_port: int = 5004  # NOTE used for tests
 
 
 GLOBALS = Globals()

learning_loop_node/helpers/gdrive_downloader.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-import requests
+import requests  # type: ignore
 
 # https://stackoverflow.com/a/39225272/4082686