learning-loop-node 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

learning_loop_node/data_classes/general.py

@@ -59,6 +59,8 @@ class ModelInformation():
         default=None, metadata={"description": "The path of the parent directory of the model in the file system."})
     model_size: Optional[str] = field(default=None, metadata={
                                       "description": "The size of the model (i.e. the specification or variant of the model architecture)."})
+    conf_threshold: Optional[float] = field(default=None, metadata={
+        "description": "The minimum confidence the trainer chose for the detections of this model, if it chose one."})
 
     @property
     def context(self):

learning_loop_node/data_classes/training.py

@@ -103,13 +103,19 @@ class Training():
         assert self.image_data is not None, 'Image data not set'
         return [image['id'] for image in self.image_data]
 
-    def train_image_count(self) -> int:
-        assert self.image_data is not None, 'Image data not set'
-        return len([image for image in self.image_data if image['set'] == 'train'])
+    def train_images(self) -> list[dict]:
+        return self._images_of_set('train')
+
+    def val_images(self) -> list[dict]:
+        """The images to select models on: the `val` images if there are any, else the test images."""
+        return self._images_of_set('val') or self.test_images()
+
+    def test_images(self) -> list[dict]:
+        return self._images_of_set('test')
 
-    def test_image_count(self) -> int:
+    def _images_of_set(self, set_name: str) -> list[dict]:
         assert self.image_data is not None, 'Image data not set'
-        return len([image for image in self.image_data if image['set'] == 'test'])
+        return [image for image in self.image_data if image['set'] == set_name]
 
 
 @dataclass(**KWONLY_SLOTS)

learning_loop_node/data_exchanger.py

@@ -12,9 +12,11 @@ from typing import Any, Dict, List, Optional
 import aiofiles  # type: ignore
 
 from .data_classes import Context
-from .helpers.misc import create_resource_paths, create_task, is_valid_image
+from .helpers.misc import create_resource_paths, create_task, is_valid_image, is_valid_uuid4
 from .loop_communication import LoopCommunicator
-from .trainer.exceptions import CriticalError
+from .trainer.exceptions import CriticalError, ModelUploadFailedError
+
+logger = logging.getLogger(__name__)
 
 
 class DownloadError(Exception):
@@ -171,20 +173,41 @@ class DataExchanger():
         logging.info('Downloaded model %s(%s) to %s.', model_uuid, model_format, target_folder)
         return created_files
 
-    async def upload_model_get_uuid(self, context: Context, files: List[str], training_number: Optional[int], mformat: str) -> str:
+    async def upload_model_get_uuid(self, context: Context, files: List[str], training_number: Optional[int], mformat: str) -> str | None:
         """Used by the trainers. Function returns the new model uuid to use for detection.
+        A loop that already stores the model in this format answers 409 and should name the model's uuid there.
 
-        :return: The new model uuid.
-        :raise CriticalError: If the upload does not return status code 200.
+        :return: The new model uuid, or None if the loop already stores the model in this format but names no valid uuid.
+        :raise CriticalError: If the loop rejects the upload with a client error (other than 408 and 429).
+        :raise ModelUploadFailedError: If the loop could not store the model, e.g. on a server error.
         """
-        response = await self.loop_communicator.put(f'/{context.organization}/projects/{context.project}/trainings/{training_number}/models/latest/{mformat}/file', files=files)
-        if response.status_code != 200:
-            logging.error('Could not upload model for training %s, format %s: %s',
-                          training_number, mformat, response.text)
-            raise CriticalError(
-                f'Could not upload model for training {training_number}, format {mformat}: {response.text}')
+        path = f'/{context.organization}/projects/{context.project}/trainings/{training_number}/models/latest/{mformat}/file'
+        response = await self.loop_communicator.put(path, files=files)
+        if response.status_code == HTTPStatus.CONFLICT:
+            try:
+                stored_model_id = response.json()['detail']['id']
+            except (ValueError, KeyError, TypeError):  # older loops name no id
+                stored_model_id = None
+            if not is_valid_uuid4(stored_model_id):
+                logger.warning('The loop already stores the model of training %s in format %s, but names no id',
+                               training_number, mformat)
+                return None
+            logger.info('The loop already stores model %s of training %s in format %s',
+                        stored_model_id, training_number, mformat)
+            return stored_model_id
+        if response.status_code != HTTPStatus.OK:
+            logger.error('Could not upload model for training %s, format %s: %s %s',
+                         training_number, mformat, response.status_code, response.text)
+            error = CriticalError if _rejects_upload(response.status_code) else ModelUploadFailedError
+            raise error(
+                f'Could not upload model for training {training_number}, format {mformat}: {response.status_code} {response.text}')
 
         uploaded_model = response.json()
-        logging.info('Uploaded model for training %s, format %s. Response is: %s',
-                     training_number, mformat, uploaded_model)
+        logger.info('Uploaded model for training %s, format %s. Response is: %s',
+                    training_number, mformat, uploaded_model)
         return uploaded_model['id']
+
+
+def _rejects_upload(status_code: int) -> bool:
+    """A client error other than 408 and 429 answers the same however often the upload is sent."""
+    return 400 <= status_code < 500 and status_code not in (HTTPStatus.REQUEST_TIMEOUT, HTTPStatus.TOO_MANY_REQUESTS)

learning_loop_node/detector/detector_node.py

@@ -6,6 +6,7 @@ import os
 import shutil
 import subprocess
 import sys
+from collections.abc import Sequence
 from dataclasses import asdict, dataclass
 from datetime import datetime
 from typing import Dict, List, Literal, Optional, Union
@@ -30,7 +31,7 @@ from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
 from ..globals import GLOBALS
 from ..helpers import background_tasks, environment_reader, run
-from ..helpers.misc import numpy_image_from_dict
+from ..helpers.misc import ImageData, image_from_dict, jpg_bytes_to_numpy_array
 from ..node import Node
 from .categories import category_by_name
 from .detector_logic import DetectorLogic, DetectorLogicFactory
@@ -230,9 +231,10 @@ class DetectorNode(Node):
 
             The data dict has the following schema:
             - image: The image data as dictionary:
-              - bytes: bytes of the ndarray
-              - dtype: data type of the ndarray
-              - shape: shape of the ndarray
+              - format: 'raw' (default) or 'jpeg' (optional)
+              - bytes: bytes of the ndarray, or the JPEG-encoded image for format 'jpeg'
+              - dtype: data type of the ndarray (raw only)
+              - shape: shape of the ndarray (raw only)
             - camera_id: Optional camera ID
             - tags: Optional list of tags
             - source: Optional source string
@@ -240,7 +242,7 @@ class DetectorNode(Node):
             - creation_date: Optional creation date in isoformat string
             """
             try:
-                image = numpy_image_from_dict(data['image'])
+                image = image_from_dict(data['image'])
             except Exception:
                 self.log.exception('could not parse image from socketio')
                 return {'error': 'could not parse image from data'}
@@ -252,7 +254,7 @@ class DetectorNode(Node):
                     tags=data.get('tags', []),
                     source=data.get('source', None),
                     autoupload=data.get('autoupload', 'filtered'),
-                    creation_date=data.get('creation_date', None)
+                    creation_date=data.get('creation_date', None),
                 )
                 return jsonable_encoder(asdict(det))
             except DetectorUnavailableError as e:
@@ -271,7 +273,7 @@ class DetectorNode(Node):
             """
             try:
                 images_data = data['images']
-                images = [numpy_image_from_dict(image) for image in images_data]
+                images = [image_from_dict(image) for image in images_data]
             except Exception:
                 self.log.exception('could not parse images from socketio')
                 return {'error': 'could not parse images from data'}
@@ -283,7 +285,7 @@ class DetectorNode(Node):
                     camera_id=data.get('camera_id', None),
                     source=data.get('source', None),
                     autoupload=data.get('autoupload', 'filtered'),
-                    creation_date=data.get('creation_date', None)
+                    creation_date=data.get('creation_date', None),
                 )
                 return jsonable_encoder(asdict(det))
             except DetectorUnavailableError as e:
@@ -336,9 +338,11 @@ class DetectorNode(Node):
 
             The data dict must contain:
             - image: The image data as dictionary with the following keys:
-                - bytes: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`)
-                - dtype: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.)
-                - shape: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`)
+                - format: 'raw' (default) or 'jpeg' (optional)
+                - bytes: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`),
+                  or the JPEG-encoded image for format 'jpeg'
+                - dtype: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.) (raw only)
+                - shape: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`) (raw only)
             - metadata: The metadata for the image (optional). Besides detections and annotations it
               may carry `tags`, `source`, `created` and `state`. A `state` of e.g. "trash" makes the
               loop file the image into that state instead of its default one.
@@ -362,7 +366,7 @@ class DetectorNode(Node):
                 image_metadata = ImageMetadata()
 
             try:
-                image = numpy_image_from_dict(data['image'])
+                image = image_from_dict(data['image'])
             except Exception:
                 self.log.exception('could not parse image from socketio')
                 return {'error': 'could not parse image from data'}
@@ -607,7 +611,7 @@ class DetectorNode(Node):
             self.log.error('could not reload app')
 
     async def get_detections(self,
-                             image: np.ndarray,
+                             image: ImageData,
                              tags: List[str],
                              *,
                              camera_id: Optional[str] = None,
@@ -620,11 +624,15 @@ class DetectorNode(Node):
         Used when an image is received via REST or SocketIO.
         This function infers the detections from the image,
         cares about uploading to the loop and returns the detections as ImageMetadata object.
+        The image is a pixel array or JPEG bytes; JPEG bytes are decoded for the inference
+        and stored as they are if the image is uploaded.
         Raises exception if no model is loaded.
         """
+        _unwrap_detector(self._detector)  # fails fast, before the decode
+        array = await _pixel_array(image)
         async with self.detection_lock:
             detector = _unwrap_detector(self._detector)
-            metadata = await run.io_bound(detector.logic.evaluate, image)
+            metadata = await run.io_bound(detector.logic.evaluate, array)
 
         metadata.tags.extend(tags)
         metadata.source = source
@@ -646,7 +654,7 @@ class DetectorNode(Node):
         return metadata
 
     async def get_batch_detections(self,
-                                   images: List[np.ndarray],
+                                   images: Sequence[ImageData],
                                    tags: List[str],
                                    *,
                                    camera_id: Optional[str] = None,
@@ -658,11 +666,15 @@ class DetectorNode(Node):
 
         This function infers the detections from all images,
         cares about uploading to the loop and returns the detections as a list of ImageMetadata.
+        Each image is a pixel array or JPEG bytes; JPEG bytes are decoded for the inference
+        and stored as they are if the image is uploaded.
         Raises exception if no model is loaded.
         """
+        _unwrap_detector(self._detector)  # fails fast, before the decode
+        arrays = await asyncio.gather(*(_pixel_array(image) for image in images))
         async with self.detection_lock:
             detector = _unwrap_detector(self._detector)
-            all_detections = await run.io_bound(detector.logic.batch_evaluate, images)
+            all_detections = await run.io_bound(detector.logic.batch_evaluate, arrays)
 
         for metadata in all_detections.items:
             metadata.tags.extend(tags)
@@ -687,14 +699,14 @@ class DetectorNode(Node):
 
     async def upload_images(
             self, *,
-            images: List[np.ndarray],
+            images: Sequence[ImageData],
             images_metadata: Optional[ImagesMetadata] = None,
             upload_priority: bool = False
     ) -> None:
         """Save images to the outbox using an asyncio executor.
         Used by SIO and REST upload endpoints.
 
-        :param images: List of images to upload
+        :param images: List of images to upload, pixel arrays or JPEG bytes
         :param images_metadata: Optional metadata for all images
         :param upload_priority: Whether to upload the images with priority
         :raises ValueError: If the number of images and number of metadata items do not match
@@ -760,6 +772,13 @@ class DetectorUnavailableError(Exception):
     pass
 
 
+async def _pixel_array(image: ImageData) -> np.ndarray:
+    """The pixels of an image; JPEG bytes are decoded in a thread, as that takes a while for a camera image."""
+    if isinstance(image, np.ndarray):
+        return image
+    return await run.io_bound(jpg_bytes_to_numpy_array, image)
+
+
 def _unwrap_detector(state: '_DetectorState') -> _ActiveDetector:
     """Return the active detector or raise DetectorUnavailableError."""
     match state:

learning_loop_node/detector/inbox_filter/relevance_filter.py

@@ -1,8 +1,7 @@
 from typing import Dict, List, Optional
 
-import numpy as np
-
 from ...data_classes.image_metadata import ImageMetadata
+from ...helpers.misc import ImageData
 from ..outbox import Outbox
 from .cam_observation_history import CamObservationHistory
 
@@ -17,7 +16,7 @@ class RelevanceFilter():
     async def may_upload_detections(self,
                                     image_metadata: ImageMetadata,
                                     cam_id: Optional[str],
-                                    image: np.ndarray) -> List[str]:
+                                    image: ImageData) -> List[str]:
         """Check if the detection should be uploaded to the outbox.
         If so, upload it and return the list of causes for the upload.
         """

learning_loop_node/detector/outbox.py

@@ -1,5 +1,4 @@
 import asyncio
-import io
 import json
 import logging
 import os
@@ -17,15 +16,13 @@ from typing import List, Optional, Tuple, TypeVar, Union
 
 import aiohttp
 import numpy as np
-import PIL
-import PIL.Image  # type: ignore
 from fastapi.encoders import jsonable_encoder
 
 from ..data_classes import ImageMetadata
 from ..enums import OutboxMode
 from ..globals import GLOBALS
 from ..helpers import environment_reader, run
-from ..helpers.misc import numpy_array_to_jpg_bytes
+from ..helpers.misc import ImageData, numpy_array_to_jpg_bytes
 
 T = TypeVar('T')
 
@@ -80,16 +77,17 @@ class Outbox():
             self.upload_folders.append(file)
 
     async def save(self,
-                   image: np.ndarray,
+                   image: ImageData,
                    image_metadata: Optional[ImageMetadata] = None,
                    upload_priority: bool = False) -> None:
         """
-        Save an image and its metadata to disk. 
+        Save an image and its metadata to disk.
 
+        A pixel array is encoded as JPEG here, image bytes are stored unchanged for the loop to validate.
         The data will be picked up by the continuous upload process.
         """
 
-        jpg_bytes = numpy_array_to_jpg_bytes(image)
+        jpg_bytes = numpy_array_to_jpg_bytes(image) if isinstance(image, np.ndarray) else image
 
         if image_metadata is None:
             image_metadata = ImageMetadata()
@@ -262,14 +260,6 @@ class Outbox():
         else:
             self.log.error('Could not upload images: %s', response.content)
 
-    def _is_valid_jpg(self, image: bytes) -> bool:
-        try:
-            _ = PIL.Image.open(io.BytesIO(image), formats=['JPEG'])
-            return True
-        except Exception:
-            self.log.exception('Invalid jpg image')
-            return False
-
     async def ensure_continuous_upload_stopped(self) -> bool:
         self.log.debug('Outbox: Ensuring continuous upload')
         if not self._upload_process_alive():

learning_loop_node/detector/rest/detect.py

@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Literal, Optional
 from fastapi import APIRouter, File, Header, Request, UploadFile
 
 from ...data_classes.image_metadata import ImageMetadata
-from ...helpers.misc import jpg_bytes_to_numpy_array
 
 if TYPE_CHECKING:
     from ..detector_node import DetectorNode
@@ -42,7 +41,7 @@ async def http_detect(
         raise Exception(f'Uploaded file {file.filename} is no image file.') from exc
 
     try:
-        detections = await node.get_detections(image=jpg_bytes_to_numpy_array(file_bytes),
+        detections = await node.get_detections(image=file_bytes,
                                                camera_id=camera_id or None,
                                                tags=tags.split(',') if tags else [],
                                                source=source,

learning_loop_node/detector/rest/upload.py

@@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, List, Optional
 from fastapi import APIRouter, File, Query, Request, UploadFile
 
 from ...data_classes.image_metadata import ImageMetadata, ImagesMetadata
-from ...helpers.misc import jpg_bytes_to_numpy_array
 
 if TYPE_CHECKING:
     from ..detector_node import DetectorNode
@@ -29,14 +28,8 @@ async def upload_image(request: Request,
     """
     node: 'DetectorNode' = request.app
 
-    files_bytes = [await file.read() for file in files]
-    image_metadatas = []
-    images = []
-    for file_bytes in files_bytes:
-        images.append(jpg_bytes_to_numpy_array(file_bytes))
-        image_metadatas.append(ImageMetadata(source=source, created=creation_date))
-
-    images_metadata = ImagesMetadata(items=image_metadatas)
+    images = [await file.read() for file in files]
+    images_metadata = ImagesMetadata(items=[ImageMetadata(source=source, created=creation_date) for _ in images])
 
     await node.upload_images(images=images,
                              images_metadata=images_metadata,

learning_loop_node/helpers/misc.py

@@ -10,12 +10,12 @@ import sys
 from dataclasses import asdict
 from glob import glob
 from time import perf_counter
-from typing import Any, Coroutine, Dict, List, Optional, Tuple, TypeVar
+from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, TypeVar, Union, get_args
 from uuid import UUID, uuid4
 
 import numpy as np
 import pynvml
-from PIL import Image
+from PIL import Image, ImageOps
 
 from ..data_classes.general import Context
 from ..data_classes.socket_response import SocketResponse
@@ -220,31 +220,56 @@ def create_training_folder(project_folder: str, trainings_id: str) -> str:
     return training_folder
 
 
+ImageData = Union[np.ndarray, bytes]
+"""An image as it was received: either a pixel array or a JPEG-encoded image."""
+
+ImageFormat = Literal['raw', 'jpeg']
+"""How the `bytes` of an image dict are encoded: a pixel array in C order described by `dtype` and `shape`, or a JPEG."""
+
+
 def jpg_bytes_to_numpy_array(jpg_bytes: bytes) -> np.ndarray:
-    """Convert jpg bytes to numpy array."""
-    image = Image.open(io.BytesIO(jpg_bytes))
+    """Decode image bytes into a pixel array, rotated as the EXIF orientation tag says."""
+    image = ImageOps.exif_transpose(Image.open(io.BytesIO(jpg_bytes)))
     return np.array(image)
 
 
 def numpy_array_to_jpg_bytes(image_array: np.ndarray) -> bytes:
-    """Convert jpg bytes to numpy array."""
+    """Encode a pixel array as JPEG."""
     buffer = io.BytesIO()
     Image.fromarray(image_array).save(buffer, format="JPEG")
     jpg_bytes = buffer.getvalue()
     return jpg_bytes
 
 
+def image_format_of(image_data: Dict) -> ImageFormat:
+    """The format an image dict declares, 'raw' if it declares none."""
+    image_format = image_data.get('format', 'raw')
+    if image_format not in get_args(ImageFormat):
+        raise ValueError(f'unknown image format: {image_format}')
+    return image_format
+
+
 def numpy_image_from_dict(image_data: Dict) -> np.ndarray:
     """
     Convert image dict to numpy array.
 
     The image_data dict should have the following keys:
-    - bytes: bytes of the image in C order (row-major)
-    - dtype: data type of the pixel values
-    - shape: shape of the image (height, width, channels)
+    - format: 'raw' (default) or 'jpeg' (optional)
+    - bytes: bytes of the image in C order (row-major), or the JPEG-encoded image for format 'jpeg'
+    - dtype: data type of the pixel values (raw only)
+    - shape: shape of the image (height, width, channels) (raw only)
     """
+    if image_format_of(image_data) == 'jpeg':
+        return jpg_bytes_to_numpy_array(image_data['bytes'])
 
     image_bytes = image_data['bytes']
     image_dtype = image_data['dtype']
     image_shape = image_data['shape']
     return np.frombuffer(image_bytes, dtype=image_dtype).reshape(image_shape, order='C')
+
+
+def image_from_dict(image_data: Dict) -> ImageData:
+    """The image of an image dict as received: the JPEG bytes for format 'jpeg', the pixel array otherwise."""
+    if image_format_of(image_data) == 'jpeg':
+        return image_data['bytes']
+    return numpy_image_from_dict(image_data)

learning_loop_node/trainer/exceptions.py

@@ -5,6 +5,12 @@ class CriticalError(Exception):
     """
 
 
+class ModelUploadFailedError(Exception):
+    """Raised when the loop could not store an uploaded model, e.g. because it answered with a server error.
+    Unlike a CriticalError it keeps the training, so the upload is tried again later.
+    """
+
+
 class TrainingRefusedError(Exception):
     """Raised when a trainer cannot take on a training it was asked to begin; nothing was set up."""
 

learning_loop_node/trainer/io_helpers.py

@@ -55,6 +55,7 @@ class ActiveTrainingIO:
         self.context = context
 
         self.mup_path = f'{training_folder}/model_uploading_progress.txt'
+        self.muuid_path = f'{training_folder}/uploaded_model_uuid.txt'
         # string with placeholder gor index
         self.det_path = f'{training_folder}' + '/detections_{0}.json'
         self.dufi_path = f'{training_folder}/detection_uploading_json_index.txt'
@@ -73,6 +74,17 @@ class ActiveTrainingIO:
         with open(self.mup_path, 'r') as f:
             return f.read().split(',')
 
+    def save_uploaded_model_uuid(self, model_uuid: str) -> None:
+        """Replaces the file in one step, so a crash while writing cannot leave it empty."""
+        with open(f'{self.muuid_path}.tmp', 'w') as f:
+            f.write(model_uuid)
+        os.replace(f'{self.muuid_path}.tmp', self.muuid_path)
+
+    def load_uploaded_model_uuid(self) -> str:
+        """:raise FileNotFoundError: If no upload has named the model uuid yet."""
+        with open(self.muuid_path) as f:
+            return f.read()
+
     # detections
 
     def _get_detection_file_names(self) -> List[Path]:

learning_loop_node/trainer/rest/backdoor_controls.py

@@ -38,8 +38,7 @@ async def reset(request: Request):
     trainer_node: 'TrainerNode' = request.app
     async with trainer_node.repeat_loop_lock:
 
-        await trainer_node.trainer_logic.stop()  # NOTE first stop may only kill running training process
-        await trainer_node.trainer_logic.stop()
+        await trainer_node.trainer_logic.abort()
         trainer_node.last_training_io.delete()
         trainer_node.status.reset_all_errors()
 

learning_loop_node/trainer/trainer_logic.py

@@ -25,6 +25,8 @@ class TrainerLogic(TrainerLogicGeneric):
         super().__init__(model_format)
         self._detection_progress: Optional[float] = None
         self._executor: Optional[Executor] = None
+        self._process_started: bool = False
+        self._stop_requested: bool = False
         self.start_training_task: Optional[Coroutine] = None
         self.inference_batch_size = self._environment_vars.inference_batch_size
 
@@ -47,10 +49,13 @@ class TrainerLogic(TrainerLogicGeneric):
         previous_state = TrainerState.TrainModelDownloaded
         error_key = 'run_training'
         self._executor = Executor(self.training.training_folder)
+        self._process_started = False
+        self._stop_requested = False
         self.training.training_state = TrainerState.TrainingRunning
 
         try:
             await self._start_training()
+            self._process_started = True
             last_sync_time = datetime.now()
 
             while True:
@@ -77,11 +82,17 @@ class TrainerLogic(TrainerLogicGeneric):
             # if self.executor.return_code != 0:
             #     raise TrainingError(cause=f'Executor returned with error code: {self.executor.return_code}')
 
-        except TrainingError:
+        except TrainingError as e:
             logging.exception('Exception in trainer_logic._train')
+            if self._stop_requested:
+                # ending the process may be what failed it; a stopped training must not start over, and the
+                # wind-down uploads the model it has (or ends with a CriticalError if there is none)
+                logging.warning('Training process failed after a stop (%s), winding down with its model', e.cause)
+                return
             self.training.training_state = previous_state
             raise
         finally:
+            self._process_started = False
             # shield so the subprocess is stopped even when the task is cancelled (e.g. on abort())
             if self._executor and self._executor.is_running():
                 await asyncio.shield(self.executor.stop_and_wait())
@@ -147,15 +158,14 @@ class TrainerLogic(TrainerLogicGeneric):
 
     # ---------------------------------------- OVERWRITTEN METHODS ----------------------------------------
 
-    async def stop(self) -> None:
-        """If executor is running, stop it. Else cancel training task."""
-        print('===============> stop received in trainer_logic.', flush=True)
-        if not self.training_active:
+    async def _stop_running_training(self) -> None:
+        """End the training process; before it has started, abort."""
+        if not self._process_started:
+            await super()._stop_running_training()
             return
+        self._stop_requested = True
         if self._executor and self._executor.is_running():
             await self.executor.stop_and_wait()
-            return
-        await super().stop()
 
     # ---------------------------------------- ABSTRACT METHODS ----------------------------------------
 

learning_loop_node/trainer/trainer_logic_generic.py

@@ -29,13 +29,22 @@ from .downloader import TrainingsDownloader
 from .exceptions import CriticalError, NodeNeedsRestartError, TrainingRefusedError
 from .io_helpers import ActiveTrainingIO, EnvironmentVars, LastTrainingIO
 from .training_slot import TrainingSlot, training_slot_from_env
+from .validation_set import assign_validation_set
 
 if TYPE_CHECKING:
     from .trainer_node import TrainerNode
 
+BASE_RETRY_DELAY = 5  # seconds before the first retry of a failed state
+MAX_RETRY_DELAY = 300  # seconds
+
 logger = logging.getLogger('learning_loop_node.trainer_logic_generic')
 
 
+def _retry_delay(failures: int) -> float:
+    """The wait before the next attempt of a state that has failed `failures` times in a row."""
+    return min(BASE_RETRY_DELAY * 2 ** min(failures - 1, 16), MAX_RETRY_DELAY)
+
+
 class TrainerLogicGeneric(ABC):
 
     def __init__(self, model_format: str, training_slot: TrainingSlot | None = None) -> None:
@@ -45,6 +54,7 @@ class TrainerLogicGeneric(ABC):
         # '/{context.organization}/projects/{context.project}/models/{model_id}/{model_format}/file'
         self.model_format: str = model_format
         self.errors = Errors()
+        self._state_failures: Dict[str, int] = {}  # failed attempts in a row, by error key
 
         self.training_task: Optional[asyncio.Task] = None
         self.shutdown_event: asyncio.Event = asyncio.Event()
@@ -264,6 +274,7 @@ class TrainerLogicGeneric(ABC):
         Either via `begin_training` or `try_continue_run_if_incomplete`.
         """
         self.errors.reset_all()
+        self._state_failures.clear()
         try:
             await self.training_slot.wait_for(self._slot_claim(self.training.context, self.training.training_number))
             await self._training_loop()
@@ -318,11 +329,14 @@ class TrainerLogicGeneric(ABC):
     async def _perform_state(self, error_key: str, state_during: TrainerState, state_after: TrainerState, action: Callable[[], Coroutine], reset_early=False):
         '''
         Perform a training state and handle errors.
-        - If the loop sends a StopTraining event, this will raise a CancelledError.
+        - If the training is aborted, the CancelledError moves it to ReadyForCleanup; on shutdown it is re-raised.
         - States can raise a CriticalError indicating that there is no point in retrying the state.
         - If any other error occurs, the error is stored in the errors object and the state is reset to the previous state.
+          The state is retried after a wait until the training is aborted.
         '''
 
+        if failures := self._state_failures.get(error_key):
+            await asyncio.sleep(_retry_delay(failures))
         await asyncio.sleep(0.1)
         logger.info('Performing state: %s', state_during)
         previous_state = self.training.training_state
@@ -351,6 +365,7 @@ class TrainerLogicGeneric(ABC):
             print('Error in %s - Exception: %s', state_during, e, flush=True)
             self.errors.set(error_key, str(e))
             logger.exception('Error in %s - Exception: %s', state_during, e)
+            self._state_failures[error_key] = self._state_failures.get(error_key, 0) + 1
             self.training.training_state = previous_state
             return
         else:
@@ -359,14 +374,15 @@ class TrainerLogicGeneric(ABC):
                 self.errors.reset(error_key)
             self.training.training_state = state_after
 
+        self._state_failures.pop(error_key, None)
         self.last_training_io.save(self.training)
 
     async def _prepare(self) -> None:
-        """Downloads images to the images_folder and saves annotations to training.data.image_data.
-        """
+        """Downloads images to the images_folder and saves annotations to training.data.image_data."""
         self.node.data_exchanger.set_context(self.training.context)
         downloader = TrainingsDownloader(self.node.data_exchanger)
         image_data, skipped_image_count = await downloader.download_training_data(self.training.images_folder)
+        assign_validation_set(image_data, self.training.hyperparameters)
 
         self.training.image_data = image_data
         self.training.skipped_image_count = skipped_image_count
@@ -406,11 +422,12 @@ class TrainerLogicGeneric(ABC):
         try:
             new_best_model = self._get_new_best_training_state()
             if new_best_model:
+                sets = [image['set'] for image in self.training.image_data or []]
                 new_training = TrainingOut(trainer_id=self.node.uuid,
                                            trainer_name=self.node.name,
                                            confusion_matrix=new_best_model.confusion_matrix,
-                                           train_image_count=self.training.train_image_count(),
-                                           test_image_count=self.training.test_image_count(),
+                                           train_image_count=sets.count('train') + sets.count('val'),
+                                           test_image_count=sets.count('test'),
                                            hyperparameters=self.hyperparameters_for_state_sync,
                                            best_epoch=new_best_model.epoch)
                 await asyncio.sleep(0.1)  # NOTE needed for tests.
@@ -440,10 +457,13 @@ class TrainerLogicGeneric(ABC):
         Note that with the latest trainers the conversion to (.wts) is done by the trainer.
         The conversion from .wts to .engine is done by the detector (needs to be done on target hardware).
         Note that trainer may train with different classes, which is why we send an initial model.json file.
+        A format the loop already stores counts as uploaded.
+        The model UUID is kept beside the upload progress, so a retry after a restart still knows it.
 
         :return: The new model UUID.
-        :raise CriticalError: If the latest model files cannot be obtained.
-        :raise Exception: If the response for the model upload attempt is invalid.
+        :raise CriticalError: If the latest model files cannot be obtained,
+            or the model is uploaded but its UUID is unknown.
+        :raise ModelUploadFailedError: If the loop could not store a model file; the state is retried.
         """
 
         files = await self._get_latest_model_files()
@@ -456,25 +476,27 @@ class TrainerLogicGeneric(ABC):
 
         already_uploaded_formats = self.active_training_io.load_model_upload_progress()
 
-        model_uuid = None
         for file_format in [f for f in files if f not in already_uploaded_formats]:
-            _files = files[file_format] + [self._dump_categories_to_json()]
+            _files = files[file_format] + [self._dump_model_json()]
             assert len([f for f in _files if 'model.json' in f]) == 1, "model.json must be included exactly once"
 
             model_uuid = await self.node.data_exchanger.upload_model_get_uuid(context, _files, self.training.training_number, file_format)
+            if model_uuid is not None:
+                self.active_training_io.save_uploaded_model_uuid(model_uuid)
 
             already_uploaded_formats.append(file_format)
             self.active_training_io.save_model_upload_progress(already_uploaded_formats)
 
-        if not model_uuid:
-            raise Exception('Invalid response for model upload attempt')
-
-        return model_uuid
+        try:
+            return self.active_training_io.load_uploaded_model_uuid()
+        except FileNotFoundError as e:
+            raise CriticalError('The model is uploaded, but the loop never answered with its id') from e
 
-    def _dump_categories_to_json(self) -> str:
-        """Dumps the categories to a json file and returns the path to the file.
+    def _dump_model_json(self) -> str:
+        """Dumps the categories and the trainer's model metadata to a json file and returns the path to the file.
         """
-        content = {'categories': [asdict(c) for c in self._training.categories], } if self._training else None
+        content = {'categories': [asdict(c) for c in self._training.categories],
+                   **self._get_model_metadata()} if self._training else None
         json_path = '/tmp/model.json'
         with open(json_path, 'w') as f:
             json.dump(content, f)
@@ -496,12 +518,17 @@ class TrainerLogicGeneric(ABC):
 
     async def on_shutdown(self) -> None:
         self.shutdown_event.set()
-        await self.stop()
-        await self.stop()
+        await self.abort()
 
-    async def stop(self):
-        """Stop the training. Default is to abort.
-        """
+    async def stop(self) -> None:
+        """Stop a running training but keep its model. In every other state a stop does nothing."""
+        if not self.training_active or self.training.training_state != TrainerState.TrainingRunning:
+            logger.info('Received stop in state %s. This is ignored.', self.training.training_state if self.training_active else None)
+            return
+        await self._stop_running_training()
+
+    async def _stop_running_training(self) -> None:
+        """End the running training so it winds down with its model. This base class cannot, so it aborts."""
         await self.abort()
 
     async def abort(self):
@@ -587,6 +614,13 @@ class TrainerLogicGeneric(ABC):
         """
         raise NotImplementedError
 
+    def _get_model_metadata(self) -> dict[str, Any]:
+        """Called when the files of `_get_latest_model_files` are uploaded.
+        Returns values the trainer determined for this model, merged into its model.json.
+        The keys must be fields of `ModelInformation` (e.g. `conf_threshold`) so detectors can read them.
+        """
+        return {}
+
     @abstractmethod
     async def _clear_training_data(self, training_folder: str) -> None:
         """Called after a training has finished. Deletes all data that is not needed anymore after a training run. 

learning_loop_node/trainer/validation_set.py

@@ -0,0 +1,63 @@
+"""Which images a training selects its models on.
+
+A node selects models (best checkpoint, early stopping, confidence threshold) on
+`Training.val_images()` and reads `Training.test_images()` only for the counts it reports. It never
+decides between the two itself: `val_images()` always holds a set to select on, which is the test
+images unless the training has `val` images of its own. A node may notice that both hold the same
+images and spare itself a second pass, but nothing more.
+
+Where the `val` images come from, in this order:
+
+1. The loop sent a dedicated validation set (`set == 'val'`); it is used as it is.
+2. The `compute_validation_set` hyperparameter is true: `assign_validation_set` moves about a tenth of the
+   train images to `val`, chosen by a hash of the image id, so an image stays on its side when new
+   images arrive or a training is continued.
+3. Otherwise there are no `val` images and `val_images()` falls back to the test images, so the
+   model is selected on the test set.
+"""
+
+import hashlib
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from .exceptions import CriticalError
+
+logger = logging.getLogger(__name__)
+
+COMPUTE_VALIDATION_SET = 'compute_validation_set'
+"""The hyperparameter that asks to split a validation set off the train images when the loop sent none."""
+
+VALIDATION_FRACTION = 0.1
+
+
+def compute_validation_set(hyperparameters: Mapping[str, Any]) -> bool:
+    """Whether `compute_validation_set` is set, defaulting to false; anything but a boolean is a CriticalError."""
+    value = hyperparameters.get(COMPUTE_VALIDATION_SET, False)
+    if not isinstance(value, bool):
+        raise CriticalError(f"'{COMPUTE_VALIDATION_SET}' must be true or false, got {value!r}")
+    return value
+
+
+def assign_validation_set(image_data: list[dict], hyperparameters: Mapping[str, Any]) -> None:
+    """Move part of the train images to `val` if `compute_validation_set` asks for it and the loop sent no `val` images.
+    """
+    if any(image['set'] == 'val' for image in image_data):
+        logger.info('Selecting models on the validation set of the loop')
+        return
+    if not compute_validation_set(hyperparameters):
+        return
+
+    train_images = [image for image in image_data if image['set'] == 'train']
+    if len(train_images) < 2:
+        raise CriticalError(f'{len(train_images)} training image(s) are too few to split off a validation set, '
+                            'add more training images')
+    val_images = [image for image in train_images if _position(image) < VALIDATION_FRACTION] \
+        or [min(train_images, key=_position)]
+    for image in val_images:
+        image['set'] = 'val'
+    logger.info('Split %d of %d training images off as validation set', len(val_images), len(train_images))
+
+
+def _position(image: dict) -> float:
+    return int(hashlib.sha256(image['id'].encode()).hexdigest()[:8], 16) / 16**8

{learning_loop_node-0.25.0.dist-info → learning_loop_node-0.27.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.25.0
+Version: 0.27.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
@@ -114,9 +114,10 @@ Example usage:
 To use the **SocketIO** inference EPs, the caller needs to connect to the detector node's SocketIO server and emit the `detect` or `batch_detect` event with the image data and image metadata. The `detect` endpoint receives a dictionary, with the following entries:
 
 - `image`: The image data as dictionary with the following keys:
-  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`)
-  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.)
-  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`)
+  - `format`: `raw` (default) or `jpeg`
+  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`), or the JPEG-encoded image for format `jpeg`
+  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.) (raw only)
+  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`) (raw only)
 - `camera_id`: optional camera identifier (string) used to improve the autoupload filtering
 - `tags`: optional list of tags to add to the image in the learning loop
 - `source`: optional source string
@@ -127,6 +128,8 @@ The `batch_detect` endpoint receives a dictionary, with the same entries as the
 
 - `images`: List of image data dictionaries, each with the same structure as the `image` entry in the `detect` endpoint
 
+Sending a camera's JPEG as it is (`format: 'jpeg'`) saves the bandwidth of a raw pixel array and spares the detector a re-encode for the upload.
+
 Example code can be found [in the rosys implementation](https://github.com/zauberzeug/rosys/blob/main/rosys/vision/detector_hardware.py).
 
 ### Upload API
@@ -144,9 +147,10 @@ Example:
 The detector also has a **SocketIO** upload endpoint that can be used to upload images and detections to the learning loop. The function receives a dictionary, with the following entries:
 
 - `image`: the image data as dictionary with the following keys:
-  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`)
-  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.)
-  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`)
+  - `format`: `raw` (default) or `jpeg`
+  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`), or the JPEG-encoded image for format `jpeg`
+  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.) (raw only)
+  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`) (raw only)
 - `metadata`: a dictionary representing the image metadata. If metadata contains detections and/or annotations, UUIDs for the classes are automatically determined based on the category names. Metadata should follow the schema of the `ImageMetadata` data class.
 - `upload_priority`: Optional boolean flag to prioritize the upload (defaults to False)
 
@@ -345,3 +349,4 @@ After a short time the converted model should be available as well.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
   - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings
+  - `conf_threshold` (optional): the minimum confidence the trainer chose for this model's detections

{learning_loop_node-0.25.0.dist-info → learning_loop_node-0.27.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 learning_loop_node/__init__.py,sha256=LzNoXBrcL6yJTyisuwJokZ9LTiYQwY3yRY3tmy83FlI,410
-learning_loop_node/data_exchanger.py,sha256=nd9JNPLn9amIeTcSIyUPpbE97ORAcb5yNphvmpgWSUQ,9095
+learning_loop_node/data_exchanger.py,sha256=Rlql4WSDgF6uUivH77deoIl2vnTKL1rlEhQmAUmJ9IU,10642
 learning_loop_node/globals.py,sha256=a50Qg4z2205Zcw4Cs8z417kn1x8s78o2FJfZB9-ZZI4,211
 learning_loop_node/loop_communication.py,sha256=d1tJZ9T_y22FrEq1XzvidgH6d-byGxFq3ovLvPBS4FA,7611
 learning_loop_node/node.py,sha256=CABqe-sqUlcwi9CYVXBkrfxh0Xgtzj3AFLQcupSX-Ms,12158
@@ -12,29 +12,29 @@ learning_loop_node/data_classes/__init__.py,sha256=_2dxfVkkI9kNM955Y7ZE8RLgfpegZ
 learning_loop_node/data_classes/annotation_data.py,sha256=NfMlTv2_5AfVY_JDM4tbjETFjSN2S2I2LJJPMMcDT50,966
 learning_loop_node/data_classes/annotations.py,sha256=ha7uuWmZqW-LU2vYrkvfnaVpvf5C2xdm826IVVBu0w0,1848
 learning_loop_node/data_classes/detections.py,sha256=hGSKc1elk4Drp4XUjQ3F9HGeRpyLvA0q064b-S9lIug,5335
-learning_loop_node/data_classes/general.py,sha256=y_seXgqCWGoUCFzwZKt8eG8rAlANIG8VhzVVivZLklU,7334
+learning_loop_node/data_classes/general.py,sha256=LTRtA_sPUKbXkNrka01_o8qwRN6r56tQJCSHPNdg_MI,7522
 learning_loop_node/data_classes/image_metadata.py,sha256=QPLJ4gcVj33xaaCFF1n5e8G6higOJ6iIKy3YhGZMfkw,2459
 learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
-learning_loop_node/data_classes/training.py,sha256=TybwcCDf_NUaDUaOj30lPm-7Z3Qk9XFRibEX5qIv96Y,5737
+learning_loop_node/data_classes/training.py,sha256=OlbPhm5kYHaZqgpz4_DfWnW4DQ_DK91AurReDhvWpRo,5944
 learning_loop_node/detector/__init__.py,sha256=wroy0W1X5MX16C-peA-UHYrcdXet7GhleIlC_Pqa6hI,128
 learning_loop_node/detector/categories.py,sha256=qgquuop3geKXXsNBurO0eC2b64h_ojL37RBYMv0ZzOg,1147
 learning_loop_node/detector/detector_logic.py,sha256=XJaHNiw5PaHoCWnJ-qsxdUELLevvztYx54TQ8KcHl18,1479
-learning_loop_node/detector/detector_node.py,sha256=GC5CaGcW_qVxyolDEz9U5W7y0ejZGEu9FZ__ckLxd7g,35728
+learning_loop_node/detector/detector_node.py,sha256=3e6UU1fl6IVkEgcfJTiOE-W30yebXcd5IpPOBYWV2mM,36950
 learning_loop_node/detector/exceptions.py,sha256=C6KbNPlSbtfgDrZx2Hbhm7Suk9jVoR3fMRCO0CkrMsQ,196
 learning_loop_node/detector/geometry.py,sha256=bZCF9xuK7qy6Ipv5aQYrTlwSmjaiUz5VEUYnc1zCe-w,1101
-learning_loop_node/detector/outbox.py,sha256=HT0GHv6rsQzdbCD8DAT23m1-Q1XEvA_KgwAJLONyX_0,12506
+learning_loop_node/detector/outbox.py,sha256=mtMxdcComjRyM6qc-O1fBv9ABCau4-oCCV442hMaryQ,12351
 learning_loop_node/detector/postprocess.py,sha256=fPBbgE6Obkej709zoFPGqA2gnbJNbpbMuZogilKy4pI,7274
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=slPGm87TbBCfDUdpRTAM5yOz1An8Br5X_2HFJOGSrFg,3540
-learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=81XQV6HST2hSH-In7VF8B4BfubgUdRHTLcc16VLjUxc,1518
+learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=kFwXIEE6ddkJ6qq8vfTLMAUP7FayOGgyrNHBhLtxVD0,1535
 learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/rest/about.py,sha256=evHJ2svUZY_DFz0FSef5u9c5KW4Uc3GL7EbPinG9-dg,583
 learning_loop_node/detector/rest/backdoor_controls.py,sha256=ZNaFOvC0OLWNtcLiG-NIqS_y1kkLP4csgk3CHhp8Gis,885
-learning_loop_node/detector/rest/detect.py,sha256=BHcQbSuC1sYHRKl2wNchiasb3A4HJk9tVia11e9TS6Y,2597
+learning_loop_node/detector/rest/detect.py,sha256=xr6VVCiH-wzvwTzTVAnzBR0vc_gTPwGL9OSSK2PJiOI,2518
 learning_loop_node/detector/rest/model_version_control.py,sha256=P4FOG0U9HT6QtCoNt-1s1pT6drtgdVjGZWEuCAyuNmA,1370
 learning_loop_node/detector/rest/operation_mode.py,sha256=eHOqbPdyfgzLtlz8sYs3WjESQ_kvxEIEP-cMuDQnsDI,1562
 learning_loop_node/detector/rest/outbox_mode.py,sha256=H8coDNbgLGEfXmKQrhtXWeUHBAHpnrdZktuHXQz0xis,1148
-learning_loop_node/detector/rest/upload.py,sha256=MditQ4Tu9PRzW6vhO-I9TJcnjOFpD83XnHx-6uUURkU,1708
+learning_loop_node/detector/rest/upload.py,sha256=qNXTLXY2b4A7Wu2GDhKU9bkfF8-QyFd-zdqWld3Jj0o,1483
 learning_loop_node/enums/__init__.py,sha256=tjSrhztIQ8W656_QuXfTbbVNtH_wDXP5hpYZgzfgRhc,285
 learning_loop_node/enums/annotator.py,sha256=mtTAw-8LJIrHcYkBjYHCZuhYEEHS6QzSK8k6BhLusvQ,285
 learning_loop_node/enums/detector.py,sha256=Qvm5LWWR9BfsDxHEQ8YzaPaUuSmp4BescYuV4X4ikwE,512
@@ -46,25 +46,26 @@ learning_loop_node/helpers/entrypoint.py,sha256=U4vHFqgke6mXjDL-E4MND8RO7P1k2VBG
 learning_loop_node/helpers/environment_reader.py,sha256=zsoFysxNxBCOJzUwX4Pbm3ftYhxx5zsbOHQfD9VcfvA,1937
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
 learning_loop_node/helpers/log_conf.py,sha256=hqVAa_9NnYEU6N0dcOKmph82p7MpgKqeF_eomTLYzWY,961
-learning_loop_node/helpers/misc.py,sha256=zfy1SumqNBwBw6-BYignQJpwMeMf1k_MA3xhYt_GiG4,8939
+learning_loop_node/helpers/misc.py,sha256=ury0HOuUxDwdYNxmiDuaWOw77lUXv0DwqQhrHqoliY4,10158
 learning_loop_node/helpers/run.py,sha256=_uox-j3_K_bL3yCAwy3JYSOiIxrnhzVxyxWpCe8_J9U,876
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/batch_size.py,sha256=LjA-zyY4hdnSMy6ZsdQOhWbmd5H_DI-9LG3yFInv04E,5476
 learning_loop_node/trainer/cuda.py,sha256=ehpfWqPBCVFu06SnGsjmWbSDRJ8sZc-L_nCtemRLUOU,12567
 learning_loop_node/trainer/downloader.py,sha256=AIyEKM4XYtd6VgmXrP3VayV9DpJzdURK1Brx81ePNSM,1470
-learning_loop_node/trainer/exceptions.py,sha256=hhd23WvuqV-6fiJo3vCNLN0hmF6TxAH9C5Bc7pCtmUM,940
+learning_loop_node/trainer/exceptions.py,sha256=uCqGzni5CO8R8pHnVjsucuM4S00UxY35qxJaWIaqqjk,1186
 learning_loop_node/trainer/executor.py,sha256=XQ1TwgOF2pQv7bgzSaljDWG4I1ySjYXL8r9c_p9MX1Q,3947
-learning_loop_node/trainer/io_helpers.py,sha256=zWl0-HLP8zxOG3ybm76vXgEozULj_1QRl95jkrF126k,7255
+learning_loop_node/trainer/io_helpers.py,sha256=fmSh2-9sEfBavYxfa_zT4FfZMByT3KxDbA3O57I8iBY,7834
 learning_loop_node/trainer/metrics.py,sha256=H1jQh5fHN3-cTdMTuH8mF_uODV0T9DBdqcPUD-ZqFAI,1439
 learning_loop_node/trainer/subprocess.py,sha256=WxuTly0zhpClEjTd9wqoI_-7YTNMaiZpRHxu00vttKw,2647
 learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
-learning_loop_node/trainer/trainer_logic.py,sha256=Yaw1GNTC05rVLQczY6vKUgkymBYfm3_0XaR5C0q-CP4,9159
-learning_loop_node/trainer/trainer_logic_generic.py,sha256=OtjdZlixz-2UXZkG_o5PaCdXCFvwgGuD5J4I9n5bROY,28812
+learning_loop_node/trainer/trainer_logic.py,sha256=x15bEy7uTfaGGHGYRGUl2olMeL0x0QW0vlu8nXZOaGg,9769
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=j_SBUejnZZqxvwzrNzwS1XifscaKwxlbnMJczzIETdA,31154
 learning_loop_node/trainer/trainer_node.py,sha256=GU4LyFA-JC9s4d-PbyKhKWyGMMIzTMqKJFNxM6tDJE4,5316
 learning_loop_node/trainer/training_slot.py,sha256=Njq6Z0T2Lqlr24ftoIp11Pw2OxCo9Z2RGn1tuFhHqLY,6737
+learning_loop_node/trainer/validation_set.py,sha256=3gyqw88jadYRYDTUZVsaE0o5LGShmqaJykjr0sdNLJo,2907
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/trainer/rest/backdoor_controls.py,sha256=ZnK8ypY5r_q0-YZbtaOxhQThzuZvMsQHM5gJGESd_dE,5131
-learning_loop_node-0.25.0.dist-info/METADATA,sha256=UOGaOXyynobyOFSEROGxOPH49odFh2-V2IGZ_LhQNsI,19855
-learning_loop_node-0.25.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-learning_loop_node-0.25.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
-learning_loop_node-0.25.0.dist-info/RECORD,,
+learning_loop_node/trainer/rest/backdoor_controls.py,sha256=bCjrQvqtgd2DtyYNgftkcDe9AFHExFLtfdW_jISLwuk,5026
+learning_loop_node-0.27.0.dist-info/METADATA,sha256=6U988q8DyrBZ1fVd6Dcoqm3wWQuXglUbQ-jiORrbsuE,20317
+learning_loop_node-0.27.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+learning_loop_node-0.27.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
+learning_loop_node-0.27.0.dist-info/RECORD,,

{learning_loop_node-0.25.0.dist-info → learning_loop_node-0.27.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (84.0.0)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any