learning-loop-node 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

learning_loop_node/data_exchanger.py

@@ -12,9 +12,11 @@ from typing import Any, Dict, List, Optional
 import aiofiles  # type: ignore
 
 from .data_classes import Context
-from .helpers.misc import create_resource_paths, create_task, is_valid_image
+from .helpers.misc import create_resource_paths, create_task, is_valid_image, is_valid_uuid4
 from .loop_communication import LoopCommunicator
-from .trainer.exceptions import CriticalError
+from .trainer.exceptions import CriticalError, ModelUploadFailedError
+
+logger = logging.getLogger(__name__)
 
 
 class DownloadError(Exception):
@@ -171,20 +173,41 @@ class DataExchanger():
         logging.info('Downloaded model %s(%s) to %s.', model_uuid, model_format, target_folder)
         return created_files
 
-    async def upload_model_get_uuid(self, context: Context, files: List[str], training_number: Optional[int], mformat: str) -> str:
+    async def upload_model_get_uuid(self, context: Context, files: List[str], training_number: Optional[int], mformat: str) -> str | None:
         """Used by the trainers. Function returns the new model uuid to use for detection.
+        A loop that already stores the model in this format answers 409 and should name the model's uuid there.
 
-        :return: The new model uuid.
-        :raise CriticalError: If the upload does not return status code 200.
+        :return: The new model uuid, or None if the loop already stores the model in this format but names no valid uuid.
+        :raise CriticalError: If the loop rejects the upload with a client error (other than 408 and 429).
+        :raise ModelUploadFailedError: If the loop could not store the model, e.g. on a server error.
         """
-        response = await self.loop_communicator.put(f'/{context.organization}/projects/{context.project}/trainings/{training_number}/models/latest/{mformat}/file', files=files)
-        if response.status_code != 200:
-            logging.error('Could not upload model for training %s, format %s: %s',
-                          training_number, mformat, response.text)
-            raise CriticalError(
-                f'Could not upload model for training {training_number}, format {mformat}: {response.text}')
+        path = f'/{context.organization}/projects/{context.project}/trainings/{training_number}/models/latest/{mformat}/file'
+        response = await self.loop_communicator.put(path, files=files)
+        if response.status_code == HTTPStatus.CONFLICT:
+            try:
+                stored_model_id = response.json()['detail']['id']
+            except (ValueError, KeyError, TypeError):  # older loops name no id
+                stored_model_id = None
+            if not is_valid_uuid4(stored_model_id):
+                logger.warning('The loop already stores the model of training %s in format %s, but names no id',
+                               training_number, mformat)
+                return None
+            logger.info('The loop already stores model %s of training %s in format %s',
+                        stored_model_id, training_number, mformat)
+            return stored_model_id
+        if response.status_code != HTTPStatus.OK:
+            logger.error('Could not upload model for training %s, format %s: %s %s',
+                         training_number, mformat, response.status_code, response.text)
+            error = CriticalError if _rejects_upload(response.status_code) else ModelUploadFailedError
+            raise error(
+                f'Could not upload model for training {training_number}, format {mformat}: {response.status_code} {response.text}')
 
         uploaded_model = response.json()
-        logging.info('Uploaded model for training %s, format %s. Response is: %s',
-                     training_number, mformat, uploaded_model)
+        logger.info('Uploaded model for training %s, format %s. Response is: %s',
+                    training_number, mformat, uploaded_model)
         return uploaded_model['id']
+
+
+def _rejects_upload(status_code: int) -> bool:
+    """A client error other than 408 and 429 answers the same however often the upload is sent."""
+    return 400 <= status_code < 500 and status_code not in (HTTPStatus.REQUEST_TIMEOUT, HTTPStatus.TOO_MANY_REQUESTS)

learning_loop_node/detector/detector_node.py

@@ -6,6 +6,7 @@ import os
 import shutil
 import subprocess
 import sys
+from collections.abc import Sequence
 from dataclasses import asdict, dataclass
 from datetime import datetime
 from typing import Dict, List, Literal, Optional, Union
@@ -30,7 +31,7 @@ from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
 from ..globals import GLOBALS
 from ..helpers import background_tasks, environment_reader, run
-from ..helpers.misc import numpy_image_from_dict
+from ..helpers.misc import ImageData, image_from_dict, jpg_bytes_to_numpy_array
 from ..node import Node
 from .categories import category_by_name
 from .detector_logic import DetectorLogic, DetectorLogicFactory
@@ -230,9 +231,10 @@ class DetectorNode(Node):
 
             The data dict has the following schema:
             - image: The image data as dictionary:
-              - bytes: bytes of the ndarray
-              - dtype: data type of the ndarray
-              - shape: shape of the ndarray
+              - format: 'raw' (default) or 'jpeg' (optional)
+              - bytes: bytes of the ndarray, or the JPEG-encoded image for format 'jpeg'
+              - dtype: data type of the ndarray (raw only)
+              - shape: shape of the ndarray (raw only)
             - camera_id: Optional camera ID
             - tags: Optional list of tags
             - source: Optional source string
@@ -240,7 +242,7 @@ class DetectorNode(Node):
             - creation_date: Optional creation date in isoformat string
             """
             try:
-                image = numpy_image_from_dict(data['image'])
+                image = image_from_dict(data['image'])
             except Exception:
                 self.log.exception('could not parse image from socketio')
                 return {'error': 'could not parse image from data'}
@@ -252,7 +254,7 @@ class DetectorNode(Node):
                     tags=data.get('tags', []),
                     source=data.get('source', None),
                     autoupload=data.get('autoupload', 'filtered'),
-                    creation_date=data.get('creation_date', None)
+                    creation_date=data.get('creation_date', None),
                 )
                 return jsonable_encoder(asdict(det))
             except DetectorUnavailableError as e:
@@ -271,7 +273,7 @@ class DetectorNode(Node):
             """
             try:
                 images_data = data['images']
-                images = [numpy_image_from_dict(image) for image in images_data]
+                images = [image_from_dict(image) for image in images_data]
             except Exception:
                 self.log.exception('could not parse images from socketio')
                 return {'error': 'could not parse images from data'}
@@ -283,7 +285,7 @@ class DetectorNode(Node):
                     camera_id=data.get('camera_id', None),
                     source=data.get('source', None),
                     autoupload=data.get('autoupload', 'filtered'),
-                    creation_date=data.get('creation_date', None)
+                    creation_date=data.get('creation_date', None),
                 )
                 return jsonable_encoder(asdict(det))
             except DetectorUnavailableError as e:
@@ -336,9 +338,11 @@ class DetectorNode(Node):
 
             The data dict must contain:
             - image: The image data as dictionary with the following keys:
-                - bytes: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`)
-                - dtype: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.)
-                - shape: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`)
+                - format: 'raw' (default) or 'jpeg' (optional)
+                - bytes: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`),
+                  or the JPEG-encoded image for format 'jpeg'
+                - dtype: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.) (raw only)
+                - shape: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`) (raw only)
             - metadata: The metadata for the image (optional). Besides detections and annotations it
               may carry `tags`, `source`, `created` and `state`. A `state` of e.g. "trash" makes the
               loop file the image into that state instead of its default one.
@@ -362,7 +366,7 @@ class DetectorNode(Node):
                 image_metadata = ImageMetadata()
 
             try:
-                image = numpy_image_from_dict(data['image'])
+                image = image_from_dict(data['image'])
             except Exception:
                 self.log.exception('could not parse image from socketio')
                 return {'error': 'could not parse image from data'}
@@ -607,7 +611,7 @@ class DetectorNode(Node):
             self.log.error('could not reload app')
 
     async def get_detections(self,
-                             image: np.ndarray,
+                             image: ImageData,
                              tags: List[str],
                              *,
                              camera_id: Optional[str] = None,
@@ -620,11 +624,15 @@ class DetectorNode(Node):
         Used when an image is received via REST or SocketIO.
         This function infers the detections from the image,
         cares about uploading to the loop and returns the detections as ImageMetadata object.
+        The image is a pixel array or JPEG bytes; JPEG bytes are decoded for the inference
+        and stored as they are if the image is uploaded.
         Raises exception if no model is loaded.
         """
+        _unwrap_detector(self._detector)  # fails fast, before the decode
+        array = await _pixel_array(image)
         async with self.detection_lock:
             detector = _unwrap_detector(self._detector)
-            metadata = await run.io_bound(detector.logic.evaluate, image)
+            metadata = await run.io_bound(detector.logic.evaluate, array)
 
         metadata.tags.extend(tags)
         metadata.source = source
@@ -646,7 +654,7 @@ class DetectorNode(Node):
         return metadata
 
     async def get_batch_detections(self,
-                                   images: List[np.ndarray],
+                                   images: Sequence[ImageData],
                                    tags: List[str],
                                    *,
                                    camera_id: Optional[str] = None,
@@ -658,11 +666,15 @@ class DetectorNode(Node):
 
         This function infers the detections from all images,
         cares about uploading to the loop and returns the detections as a list of ImageMetadata.
+        Each image is a pixel array or JPEG bytes; JPEG bytes are decoded for the inference
+        and stored as they are if the image is uploaded.
         Raises exception if no model is loaded.
         """
+        _unwrap_detector(self._detector)  # fails fast, before the decode
+        arrays = await asyncio.gather(*(_pixel_array(image) for image in images))
         async with self.detection_lock:
             detector = _unwrap_detector(self._detector)
-            all_detections = await run.io_bound(detector.logic.batch_evaluate, images)
+            all_detections = await run.io_bound(detector.logic.batch_evaluate, arrays)
 
         for metadata in all_detections.items:
             metadata.tags.extend(tags)
@@ -687,14 +699,14 @@ class DetectorNode(Node):
 
     async def upload_images(
             self, *,
-            images: List[np.ndarray],
+            images: Sequence[ImageData],
             images_metadata: Optional[ImagesMetadata] = None,
             upload_priority: bool = False
     ) -> None:
         """Save images to the outbox using an asyncio executor.
         Used by SIO and REST upload endpoints.
 
-        :param images: List of images to upload
+        :param images: List of images to upload, pixel arrays or JPEG bytes
         :param images_metadata: Optional metadata for all images
         :param upload_priority: Whether to upload the images with priority
         :raises ValueError: If the number of images and number of metadata items do not match
@@ -760,6 +772,13 @@ class DetectorUnavailableError(Exception):
     pass
 
 
+async def _pixel_array(image: ImageData) -> np.ndarray:
+    """The pixels of an image; JPEG bytes are decoded in a thread, as that takes a while for a camera image."""
+    if isinstance(image, np.ndarray):
+        return image
+    return await run.io_bound(jpg_bytes_to_numpy_array, image)
+
+
 def _unwrap_detector(state: '_DetectorState') -> _ActiveDetector:
     """Return the active detector or raise DetectorUnavailableError."""
     match state:

learning_loop_node/detector/inbox_filter/relevance_filter.py

@@ -1,8 +1,7 @@
 from typing import Dict, List, Optional
 
-import numpy as np
-
 from ...data_classes.image_metadata import ImageMetadata
+from ...helpers.misc import ImageData
 from ..outbox import Outbox
 from .cam_observation_history import CamObservationHistory
 
@@ -17,7 +16,7 @@ class RelevanceFilter():
     async def may_upload_detections(self,
                                     image_metadata: ImageMetadata,
                                     cam_id: Optional[str],
-                                    image: np.ndarray) -> List[str]:
+                                    image: ImageData) -> List[str]:
         """Check if the detection should be uploaded to the outbox.
         If so, upload it and return the list of causes for the upload.
         """

learning_loop_node/detector/outbox.py

@@ -1,5 +1,4 @@
 import asyncio
-import io
 import json
 import logging
 import os
@@ -17,15 +16,13 @@ from typing import List, Optional, Tuple, TypeVar, Union
 
 import aiohttp
 import numpy as np
-import PIL
-import PIL.Image  # type: ignore
 from fastapi.encoders import jsonable_encoder
 
 from ..data_classes import ImageMetadata
 from ..enums import OutboxMode
 from ..globals import GLOBALS
 from ..helpers import environment_reader, run
-from ..helpers.misc import numpy_array_to_jpg_bytes
+from ..helpers.misc import ImageData, numpy_array_to_jpg_bytes
 
 T = TypeVar('T')
 
@@ -80,16 +77,17 @@ class Outbox():
             self.upload_folders.append(file)
 
     async def save(self,
-                   image: np.ndarray,
+                   image: ImageData,
                    image_metadata: Optional[ImageMetadata] = None,
                    upload_priority: bool = False) -> None:
         """
-        Save an image and its metadata to disk. 
+        Save an image and its metadata to disk.
 
+        A pixel array is encoded as JPEG here, image bytes are stored unchanged for the loop to validate.
         The data will be picked up by the continuous upload process.
         """
 
-        jpg_bytes = numpy_array_to_jpg_bytes(image)
+        jpg_bytes = numpy_array_to_jpg_bytes(image) if isinstance(image, np.ndarray) else image
 
         if image_metadata is None:
             image_metadata = ImageMetadata()
@@ -262,14 +260,6 @@ class Outbox():
         else:
             self.log.error('Could not upload images: %s', response.content)
 
-    def _is_valid_jpg(self, image: bytes) -> bool:
-        try:
-            _ = PIL.Image.open(io.BytesIO(image), formats=['JPEG'])
-            return True
-        except Exception:
-            self.log.exception('Invalid jpg image')
-            return False
-
     async def ensure_continuous_upload_stopped(self) -> bool:
         self.log.debug('Outbox: Ensuring continuous upload')
         if not self._upload_process_alive():

learning_loop_node/detector/rest/detect.py

@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Literal, Optional
 from fastapi import APIRouter, File, Header, Request, UploadFile
 
 from ...data_classes.image_metadata import ImageMetadata
-from ...helpers.misc import jpg_bytes_to_numpy_array
 
 if TYPE_CHECKING:
     from ..detector_node import DetectorNode
@@ -42,7 +41,7 @@ async def http_detect(
         raise Exception(f'Uploaded file {file.filename} is no image file.') from exc
 
     try:
-        detections = await node.get_detections(image=jpg_bytes_to_numpy_array(file_bytes),
+        detections = await node.get_detections(image=file_bytes,
                                                camera_id=camera_id or None,
                                                tags=tags.split(',') if tags else [],
                                                source=source,

learning_loop_node/detector/rest/upload.py

@@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, List, Optional
 from fastapi import APIRouter, File, Query, Request, UploadFile
 
 from ...data_classes.image_metadata import ImageMetadata, ImagesMetadata
-from ...helpers.misc import jpg_bytes_to_numpy_array
 
 if TYPE_CHECKING:
     from ..detector_node import DetectorNode
@@ -29,14 +28,8 @@ async def upload_image(request: Request,
     """
     node: 'DetectorNode' = request.app
 
-    files_bytes = [await file.read() for file in files]
-    image_metadatas = []
-    images = []
-    for file_bytes in files_bytes:
-        images.append(jpg_bytes_to_numpy_array(file_bytes))
-        image_metadatas.append(ImageMetadata(source=source, created=creation_date))
-
-    images_metadata = ImagesMetadata(items=image_metadatas)
+    images = [await file.read() for file in files]
+    images_metadata = ImagesMetadata(items=[ImageMetadata(source=source, created=creation_date) for _ in images])
 
     await node.upload_images(images=images,
                              images_metadata=images_metadata,

learning_loop_node/enums/trainer.py

@@ -4,7 +4,9 @@ from enum import Enum
 
 class TrainerState(str, Enum):
     Idle = 'idle'
+    Blocked = 'blocked'  # idle, but unable to take a training
     Initialized = 'initialized'
+    WaitingForSlot = 'waiting_for_slot'
     Preparing = 'preparing'
     DataDownloading = 'data_downloading'
     DataDownloaded = 'data_downloaded'

learning_loop_node/helpers/misc.py

@@ -10,12 +10,12 @@ import sys
 from dataclasses import asdict
 from glob import glob
 from time import perf_counter
-from typing import Any, Coroutine, Dict, List, Optional, Tuple, TypeVar
+from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, TypeVar, Union, get_args
 from uuid import UUID, uuid4
 
 import numpy as np
 import pynvml
-from PIL import Image
+from PIL import Image, ImageOps
 
 from ..data_classes.general import Context
 from ..data_classes.socket_response import SocketResponse
@@ -150,7 +150,7 @@ def ensure_socket_response(func):
             if isinstance(value, bool):
                 return asdict(SocketResponse.from_bool(value))
             if isinstance(value, SocketResponse):
-                return value
+                return asdict(value)
             if (args[0] in ['connect', 'disconnect', 'connect_error']):
                 return value
             if value is None:
@@ -220,31 +220,56 @@ def create_training_folder(project_folder: str, trainings_id: str) -> str:
     return training_folder
 
 
+ImageData = Union[np.ndarray, bytes]
+"""An image as it was received: either a pixel array or a JPEG-encoded image."""
+
+ImageFormat = Literal['raw', 'jpeg']
+"""How the `bytes` of an image dict are encoded: a pixel array in C order described by `dtype` and `shape`, or a JPEG."""
+
+
 def jpg_bytes_to_numpy_array(jpg_bytes: bytes) -> np.ndarray:
-    """Convert jpg bytes to numpy array."""
-    image = Image.open(io.BytesIO(jpg_bytes))
+    """Decode image bytes into a pixel array, rotated as the EXIF orientation tag says."""
+    image = ImageOps.exif_transpose(Image.open(io.BytesIO(jpg_bytes)))
     return np.array(image)
 
 
 def numpy_array_to_jpg_bytes(image_array: np.ndarray) -> bytes:
-    """Convert jpg bytes to numpy array."""
+    """Encode a pixel array as JPEG."""
     buffer = io.BytesIO()
     Image.fromarray(image_array).save(buffer, format="JPEG")
     jpg_bytes = buffer.getvalue()
     return jpg_bytes
 
 
+def image_format_of(image_data: Dict) -> ImageFormat:
+    """The format an image dict declares, 'raw' if it declares none."""
+    image_format = image_data.get('format', 'raw')
+    if image_format not in get_args(ImageFormat):
+        raise ValueError(f'unknown image format: {image_format}')
+    return image_format
+
+
 def numpy_image_from_dict(image_data: Dict) -> np.ndarray:
     """
     Convert image dict to numpy array.
 
     The image_data dict should have the following keys:
-    - bytes: bytes of the image in C order (row-major)
-    - dtype: data type of the pixel values
-    - shape: shape of the image (height, width, channels)
+    - format: 'raw' (default) or 'jpeg' (optional)
+    - bytes: bytes of the image in C order (row-major), or the JPEG-encoded image for format 'jpeg'
+    - dtype: data type of the pixel values (raw only)
+    - shape: shape of the image (height, width, channels) (raw only)
     """
+    if image_format_of(image_data) == 'jpeg':
+        return jpg_bytes_to_numpy_array(image_data['bytes'])
 
     image_bytes = image_data['bytes']
     image_dtype = image_data['dtype']
     image_shape = image_data['shape']
     return np.frombuffer(image_bytes, dtype=image_dtype).reshape(image_shape, order='C')
+
+
+def image_from_dict(image_data: Dict) -> ImageData:
+    """The image of an image dict as received: the JPEG bytes for format 'jpeg', the pixel array otherwise."""
+    if image_format_of(image_data) == 'jpeg':
+        return image_data['bytes']
+    return numpy_image_from_dict(image_data)

learning_loop_node/trainer/exceptions.py

@@ -5,6 +5,24 @@ class CriticalError(Exception):
     """
 
 
+class ModelUploadFailedError(Exception):
+    """Raised when the loop could not store an uploaded model, e.g. because it answered with a server error.
+    Unlike a CriticalError it keeps the training, so the upload is tried again later.
+    """
+
+
+class TrainingRefusedError(Exception):
+    """Raised when a trainer cannot take on a training it was asked to begin; nothing was set up."""
+
+
+class TrainingSlotTakenError(TrainingRefusedError):
+    """Raised when a sibling node holds the training slot."""
+
+    def __init__(self, holder: str) -> None:
+        super().__init__(f'the training slot is held by {holder}')
+        self.holder = holder
+
+
 class InsufficientMemoryError(RuntimeError):
     """Raised when not even the smallest unit of work fits in memory."""
 

learning_loop_node/trainer/io_helpers.py

@@ -55,6 +55,7 @@ class ActiveTrainingIO:
         self.context = context
 
         self.mup_path = f'{training_folder}/model_uploading_progress.txt'
+        self.muuid_path = f'{training_folder}/uploaded_model_uuid.txt'
         # string with placeholder gor index
         self.det_path = f'{training_folder}' + '/detections_{0}.json'
         self.dufi_path = f'{training_folder}/detection_uploading_json_index.txt'
@@ -73,6 +74,17 @@ class ActiveTrainingIO:
         with open(self.mup_path, 'r') as f:
             return f.read().split(',')
 
+    def save_uploaded_model_uuid(self, model_uuid: str) -> None:
+        """Replaces the file in one step, so a crash while writing cannot leave it empty."""
+        with open(f'{self.muuid_path}.tmp', 'w') as f:
+            f.write(model_uuid)
+        os.replace(f'{self.muuid_path}.tmp', self.muuid_path)
+
+    def load_uploaded_model_uuid(self) -> str:
+        """:raise FileNotFoundError: If no upload has named the model uuid yet."""
+        with open(self.muuid_path) as f:
+            return f.read()
+
     # detections
 
     def _get_detection_file_names(self) -> List[Path]:

learning_loop_node/trainer/trainer_logic_generic.py

@@ -26,24 +26,34 @@ from ..helpers.misc import (
     is_valid_uuid4,
 )
 from .downloader import TrainingsDownloader
-from .exceptions import CriticalError, NodeNeedsRestartError
+from .exceptions import CriticalError, NodeNeedsRestartError, TrainingRefusedError
 from .io_helpers import ActiveTrainingIO, EnvironmentVars, LastTrainingIO
+from .training_slot import TrainingSlot, training_slot_from_env
 
 if TYPE_CHECKING:
     from .trainer_node import TrainerNode
 
+BASE_RETRY_DELAY = 5  # seconds before the first retry of a failed state
+MAX_RETRY_DELAY = 300  # seconds
+
 logger = logging.getLogger('learning_loop_node.trainer_logic_generic')
 
 
+def _retry_delay(failures: int) -> float:
+    """The wait before the next attempt of a state that has failed `failures` times in a row."""
+    return min(BASE_RETRY_DELAY * 2 ** min(failures - 1, 16), MAX_RETRY_DELAY)
+
+
 class TrainerLogicGeneric(ABC):
 
-    def __init__(self, model_format: str) -> None:
+    def __init__(self, model_format: str, training_slot: TrainingSlot | None = None) -> None:
 
         # NOTE: model_format is used in the file path for the model on the server:
         # It acts as a key for list of files (cf. _get_latest_model_files)
         # '/{context.organization}/projects/{context.project}/models/{model_id}/{model_format}/file'
         self.model_format: str = model_format
         self.errors = Errors()
+        self._state_failures: Dict[str, int] = {}  # failed attempts in a row, by error key
 
         self.training_task: Optional[asyncio.Task] = None
         self.shutdown_event: asyncio.Event = asyncio.Event()
@@ -54,6 +64,7 @@ class TrainerLogicGeneric(ABC):
         self._training: Optional[Training] = None
         self._active_training_io: Optional[ActiveTrainingIO] = None
         self._environment_vars = EnvironmentVars()
+        self.training_slot = training_slot or training_slot_from_env()
 
     # ---------------------------------------- PROPERTIES TO AVOID CHECKING FOR NONE ----------------------------------------
 
@@ -98,9 +109,17 @@ class TrainerLogicGeneric(ABC):
 
     @property
     def state(self) -> str:
-        """Returns the current state of the training. Used solely by the node in send_status().
+        """Returns the current state of the training.
+
+        Without a training the state is `idle`, or `blocked` while a sibling node holds the training slot.
+        A training resumed after a restart reports `waiting_for_slot` until it gets the slot: it has a
+        training, so it is never `blocked`.
         """
-        if (not self.training_active) or (self.training.training_state is None):
+        if not self.training_active:
+            return TrainerState.Blocked.value if self.training_slot.taken_by_sibling else TrainerState.Idle.value
+        if not self.training_slot.held:
+            return TrainerState.WaitingForSlot.value
+        if self.training.training_state is None:
             return TrainerState.Idle.value
         return self.training.training_state
 
@@ -213,8 +232,21 @@ class TrainerLogicGeneric(ABC):
 
     async def begin_training(self, organization: str, project: str, training_config: Dict) -> None:
         """Called on `begin_training` event from the Learning Loop.
+
+        The training slot is taken before anything is set up, so a refused training leaves the
+        previous training folders in place.
+
+        :raise TrainingRefusedError: if a training is active or a sibling node holds the training slot.
         """
-        self._init_new_training(Context(organization=organization, project=project), training_config)
+        if self.training_active:
+            raise TrainingRefusedError(f'training {self.training.id} is still active')
+        context = Context(organization=organization, project=project)
+        self.training_slot.claim(self._slot_claim(context, training_config.get('training_number')))
+        try:
+            self._init_new_training(context, training_config)
+        except Exception:
+            self.training_slot.release()
+            raise
         self._begin_training_task()
 
     def _begin_training_task(self) -> None:
@@ -241,7 +273,9 @@ class TrainerLogicGeneric(ABC):
         Either via `begin_training` or `try_continue_run_if_incomplete`.
         """
         self.errors.reset_all()
+        self._state_failures.clear()
         try:
+            await self.training_slot.wait_for(self._slot_claim(self.training.context, self.training.training_number))
             await self._training_loop()
         except asyncio.CancelledError:
             if not self.shutdown_event.is_set():
@@ -254,6 +288,12 @@ class TrainerLogicGeneric(ABC):
                 logger.info('CancelledError in _run - shutting down')
         except Exception:
             logger.exception('(Ignored) exception in trainer_logic._run:')
+        finally:
+            self._training = None
+            self.training_slot.release()
+
+    def _slot_claim(self, context: Context, training_number: Any) -> str:
+        return f'node {self.node.uuid}, training #{training_number} of {context.organization}/{context.project}'
 
     # ---------------------------------------- TRAINING STATES ----------------------------------------
 
@@ -291,8 +331,11 @@ class TrainerLogicGeneric(ABC):
         - If the loop sends a StopTraining event, this will raise a CancelledError.
         - States can raise a CriticalError indicating that there is no point in retrying the state.
         - If any other error occurs, the error is stored in the errors object and the state is reset to the previous state.
+          The state is retried after a wait until the training is aborted.
         '''
 
+        if failures := self._state_failures.get(error_key):
+            await asyncio.sleep(_retry_delay(failures))
         await asyncio.sleep(0.1)
         logger.info('Performing state: %s', state_during)
         previous_state = self.training.training_state
@@ -321,6 +364,7 @@ class TrainerLogicGeneric(ABC):
             print('Error in %s - Exception: %s', state_during, e, flush=True)
             self.errors.set(error_key, str(e))
             logger.exception('Error in %s - Exception: %s', state_during, e)
+            self._state_failures[error_key] = self._state_failures.get(error_key, 0) + 1
             self.training.training_state = previous_state
             return
         else:
@@ -329,6 +373,7 @@ class TrainerLogicGeneric(ABC):
                 self.errors.reset(error_key)
             self.training.training_state = state_after
 
+        self._state_failures.pop(error_key, None)
         self.last_training_io.save(self.training)
 
     async def _prepare(self) -> None:
@@ -410,10 +455,13 @@ class TrainerLogicGeneric(ABC):
         Note that with the latest trainers the conversion to (.wts) is done by the trainer.
         The conversion from .wts to .engine is done by the detector (needs to be done on target hardware).
         Note that trainer may train with different classes, which is why we send an initial model.json file.
+        A format the loop already stores counts as uploaded.
+        The model UUID is kept beside the upload progress, so a retry after a restart still knows it.
 
         :return: The new model UUID.
-        :raise CriticalError: If the latest model files cannot be obtained.
-        :raise Exception: If the response for the model upload attempt is invalid.
+        :raise CriticalError: If the latest model files cannot be obtained,
+            or the model is uploaded but its UUID is unknown.
+        :raise ModelUploadFailedError: If the loop could not store a model file; the state is retried.
         """
 
         files = await self._get_latest_model_files()
@@ -426,20 +474,21 @@ class TrainerLogicGeneric(ABC):
 
         already_uploaded_formats = self.active_training_io.load_model_upload_progress()
 
-        model_uuid = None
         for file_format in [f for f in files if f not in already_uploaded_formats]:
             _files = files[file_format] + [self._dump_categories_to_json()]
             assert len([f for f in _files if 'model.json' in f]) == 1, "model.json must be included exactly once"
 
             model_uuid = await self.node.data_exchanger.upload_model_get_uuid(context, _files, self.training.training_number, file_format)
+            if model_uuid is not None:
+                self.active_training_io.save_uploaded_model_uuid(model_uuid)
 
             already_uploaded_formats.append(file_format)
             self.active_training_io.save_model_upload_progress(already_uploaded_formats)
 
-        if not model_uuid:
-            raise Exception('Invalid response for model upload attempt')
-
-        return model_uuid
+        try:
+            return self.active_training_io.load_uploaded_model_uuid()
+        except FileNotFoundError as e:
+            raise CriticalError('The model is uploaded, but the loop never answered with its id') from e
 
     def _dump_categories_to_json(self) -> str:
         """Dumps the categories to a json file and returns the path to the file.

learning_loop_node/trainer/trainer_node.py

@@ -7,7 +7,10 @@ from typing import Dict, Optional
 from fastapi.encoders import jsonable_encoder
 from socketio import AsyncClient, exceptions
 
+from ..data_classes import SocketResponse
+from ..enums import TrainerState
 from ..node import Node
+from .exceptions import TrainingRefusedError
 from .io_helpers import LastTrainingIO
 from .rest import backdoor_controls
 from .trainer_logic_generic import TrainerLogicGeneric
@@ -64,7 +67,11 @@ class TrainerNode(Node):
         @sio_client.event
         async def begin_training(organization: str, project: str, details: Dict):
             self.log.info('received begin_training from server')
-            await self.trainer_logic.begin_training(organization, project, details)
+            try:
+                await self.trainer_logic.begin_training(organization, project, details)
+            except TrainingRefusedError as e:
+                self.log.warning('refused begin_training: %s', e)
+                return SocketResponse(success=False, error_msg=str(e), payload='refused')
             return True
 
         @sio_client.event
@@ -102,7 +109,7 @@ class TrainerNode(Node):
         if not self._idle_timeout:
             return
 
-        if self.trainer_logic.state == 'idle':
+        if self.trainer_logic.state in (TrainerState.Idle, TrainerState.Blocked):
             if self._first_idle_time is None:
                 self._first_idle_time = time.time()
             idle_time = time.time() - self._first_idle_time

learning_loop_node/trainer/training_slot.py

@@ -0,0 +1,207 @@
+"""The right to use the GPU, shared between the trainer nodes on one machine.
+
+A node holds the slot from the moment it accepts a training until the training is cleared.
+"""
+import asyncio
+import fcntl
+import logging
+import os
+import socket
+import weakref
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from .exceptions import TrainingSlotTakenError
+
+ENV_VAR = 'TRAINING_SLOT_LOCK'
+_CLAIM_ATTEMPTS = 3
+
+logger = logging.getLogger('learning_loop_node.training_slot')
+
+
+class TrainingSlot(ABC):
+    """The slot as a trainer uses it; an adapter implements `_acquire`, `_sibling_holder`, `release` and `held`."""
+
+    def claim(self, training: str) -> None:
+        """Take the slot for `training` without waiting; a no-op if this slot already holds it.
+
+        :raise TrainingSlotTakenError: if a sibling node holds the slot.
+        """
+        holder = self._describe(training)
+        for _ in range(_CLAIM_ATTEMPTS - 1):
+            try:
+                self._acquire(holder)
+                return
+            except TrainingSlotTakenError:
+                if self._sibling_holder() is not None:
+                    raise
+        self._acquire(holder)
+
+    async def wait_for(self, training: str) -> None:
+        """Take the slot for `training`, waiting as long as a sibling node holds it."""
+        try:
+            self.claim(training)
+            return
+        except TrainingSlotTakenError as e:
+            logger.info('%s - waiting', e)
+        while True:
+            await asyncio.sleep(1)
+            try:
+                self.claim(training)
+            except TrainingSlotTakenError:
+                continue
+            logger.info('acquired training slot')
+            return
+
+    @abstractmethod
+    def release(self) -> None:
+        """Give the slot back; a no-op if this slot does not hold it."""
+
+    @property
+    @abstractmethod
+    def held(self) -> bool:
+        """Whether this slot object holds the slot."""
+
+    @property
+    def taken_by_sibling(self) -> bool:
+        return not self.held and self._sibling_holder() is not None
+
+    @abstractmethod
+    def _acquire(self, holder: str) -> None:
+        """Take the slot without waiting; a no-op if this slot already holds it.
+
+        :raise TrainingSlotTakenError: if the slot is held elsewhere, including by a momentary probe.
+        """
+
+    @abstractmethod
+    def _sibling_holder(self) -> str | None:
+        """Who holds the slot, None if it is free or cannot be inspected; only asked while this slot does not hold it."""
+
+    @staticmethod
+    def _describe(training: str) -> str:
+        return f'pid {os.getpid()} on {socket.gethostname()}, {training}'
+
+
+class AlwaysFreeSlot(TrainingSlot):
+    """The slot of a node that has the GPU to itself."""
+
+    def __init__(self) -> None:
+        self._held = False
+
+    def release(self) -> None:
+        self._held = False
+
+    @property
+    def held(self) -> bool:
+        return self._held
+
+    def _acquire(self, holder: str) -> None:
+        self._held = True
+
+    def _sibling_holder(self) -> str | None:
+        return None
+
+
+class FileTrainingSlot(TrainingSlot):
+    """A slot backed by an exclusive `flock` on a file every sibling node mounts.
+
+    The kernel drops the lock with the process, so a node that dies in any way frees the slot.
+    The lock belongs to the inode: the file is only ever truncated and rewritten in place, never
+    replaced, and the mounted path is its directory.
+
+    The file's content names the holder and is diagnostic only — `_sibling_holder` tells a free slot
+    from a taken one by trying the lock, and that probe holds the lock for an instant, so a
+    claim that loses against a probe tries again. The file is made writable for everyone,
+    since sibling containers may run as different users.
+
+    A forked child shares the parent's lock and would keep it alive after the parent releases it,
+    so the child closes its copy right after the fork; the parent's lock is unaffected.
+    """
+
+    _instances: 'weakref.WeakSet[FileTrainingSlot]' = weakref.WeakSet()
+
+    def __init__(self, path: str | Path) -> None:
+        self.path = Path(path)
+        self._fd: int | None = None
+        FileTrainingSlot._instances.add(self)
+
+    def _acquire(self, holder: str) -> None:
+        if self._fd is not None:
+            return
+        fd = self._open()
+        try:
+            self._lock(fd)
+            os.ftruncate(fd, 0)
+            os.write(fd, holder.encode())
+        except BaseException:
+            os.close(fd)
+            raise
+        self._fd = fd
+
+    def release(self) -> None:
+        if self._fd is None:
+            return
+        try:
+            os.ftruncate(self._fd, 0)
+        except OSError:
+            logger.warning('cannot clear the holder from training slot %s', self.path, exc_info=True)
+        finally:
+            os.close(self._fd)
+            self._fd = None
+
+    @property
+    def held(self) -> bool:
+        return self._fd is not None
+
+    def _sibling_holder(self) -> str | None:
+        try:
+            fd = os.open(self.path, os.O_RDONLY)
+        except FileNotFoundError:
+            return None
+        except OSError:
+            logger.warning('cannot inspect training slot %s - treating it as free', self.path, exc_info=True)
+            return None
+        try:
+            self._lock(fd)
+        except TrainingSlotTakenError as e:
+            return e.holder
+        finally:
+            os.close(fd)
+        return None
+
+    @classmethod
+    def _drop_inherited_locks(cls) -> None:
+        for slot in list(cls._instances):
+            if slot._fd is not None:
+                os.close(slot._fd)
+                slot._fd = None
+
+    def _open(self) -> int:
+        fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o666)
+        try:
+            os.fchmod(fd, 0o666)
+        except PermissionError:
+            pass
+        return fd
+
+    @staticmethod
+    def _lock(fd: int) -> None:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError as e:
+            raise TrainingSlotTakenError(os.read(fd, 4096).decode(errors='replace') or 'unknown') from e
+
+os.register_at_fork(after_in_child=FileTrainingSlot._drop_inherited_locks)  # pylint: disable=protected-access
+
+
+def training_slot_from_env() -> TrainingSlot:
+    """The slot `TRAINING_SLOT_LOCK` names, or a free one if the variable is unset.
+
+    :raise FileNotFoundError: if the lock file's directory does not exist.
+    """
+    path = os.environ.get(ENV_VAR)
+    if not path:
+        return AlwaysFreeSlot()
+    if not Path(path).parent.is_dir():
+        raise FileNotFoundError(f'{ENV_VAR}={path}: the directory does not exist; mount it from the host')
+    return FileTrainingSlot(path)

{learning_loop_node-0.24.0.dist-info → learning_loop_node-0.26.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.24.0
+Version: 0.26.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
@@ -67,6 +67,7 @@ You can configure connection to our Learning Loop by specifying the following en
 | RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            | 0            |
 | KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            | 0            |
 | TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            | 0 (disabled) |
+| TRAINING_SLOT_LOCK       | -            | Lock file shared by the trainers on one GPU (see Trainer Node) | Trainer (opt.)          | - (disabled) |
 | USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) | 0            |
 
 Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
@@ -113,9 +114,10 @@ Example usage:
 To use the **SocketIO** inference EPs, the caller needs to connect to the detector node's SocketIO server and emit the `detect` or `batch_detect` event with the image data and image metadata. The `detect` endpoint receives a dictionary, with the following entries:
 
 - `image`: The image data as dictionary with the following keys:
-  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`)
-  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.)
-  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`)
+  - `format`: `raw` (default) or `jpeg`
+  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`), or the JPEG-encoded image for format `jpeg`
+  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.) (raw only)
+  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`) (raw only)
 - `camera_id`: optional camera identifier (string) used to improve the autoupload filtering
 - `tags`: optional list of tags to add to the image in the learning loop
 - `source`: optional source string
@@ -126,6 +128,8 @@ The `batch_detect` endpoint receives a dictionary, with the same entries as the
 
 - `images`: List of image data dictionaries, each with the same structure as the `image` entry in the `detect` endpoint
 
+Sending a camera's JPEG as it is (`format: 'jpeg'`) saves the bandwidth of a raw pixel array and spares the detector a re-encode for the upload.
+
 Example code can be found [in the rosys implementation](https://github.com/zauberzeug/rosys/blob/main/rosys/vision/detector_hardware.py).
 
 ### Upload API
@@ -143,9 +147,10 @@ Example:
 The detector also has a **SocketIO** upload endpoint that can be used to upload images and detections to the learning loop. The function receives a dictionary, with the following entries:
 
 - `image`: the image data as dictionary with the following keys:
-  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`)
-  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.)
-  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`)
+  - `format`: `raw` (default) or `jpeg`
+  - `bytes`: bytes of the ndarray (retrieved via `ndarray.tobytes(order='C')`), or the JPEG-encoded image for format `jpeg`
+  - `dtype`: data type of the ndarray as string (e.g. `uint8`, `float32`, etc.) (raw only)
+  - `shape`: shape of the ndarray as tuple of ints (e.g. `(480, 640, 3)`) (raw only)
 - `metadata`: a dictionary representing the image metadata. If metadata contains detections and/or annotations, UUIDs for the classes are automatically determined based on the category names. Metadata should follow the schema of the `ImageMetadata` data class.
 - `upload_priority`: Optional boolean flag to prioritize the upload (defaults to False)
 
@@ -250,6 +255,20 @@ Trainers fetch the images and anntoations from the Learning Loop to train new mo
 
 - if the command line tool "jpeginfo" is installed, the downloader will drop corrupted images automatically
 
+### Several trainers on one GPU
+
+Trainers that share a GPU (say, one per model architecture on the same machine) coordinate through
+a lock file: mount one host directory into each of them and point `TRAINING_SLOT_LOCK` at a file
+inside it, e.g. `/slot/training.lock`. A trainer takes the lock when it accepts a training and holds
+it until the training is cleaned up, including detection and upload. While a sibling holds it, the
+others report the state `blocked` instead of `idle`, so the Learning Loop neither offers them for a
+training nor dispatches queued jobs to them; a trainer that is sent `begin_training` regardless
+refuses it with `{"success": false, "error_msg": ...}` in the acknowledgement and sets nothing up. A
+training resumed after a restart cannot be refused — it waits for the lock instead, reporting
+`waiting_for_slot` meanwhile. The lock is an `flock`, so the kernel releases it whenever the holding
+process ends, however it ends — there is nothing to clean up after a crash. Mount the directory,
+not the file, and never replace the file: the lock belongs to its inode.
+
 ## Converter Node
 
 A Conveter Node converts models from one format into another.

{learning_loop_node-0.24.0.dist-info → learning_loop_node-0.26.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 learning_loop_node/__init__.py,sha256=LzNoXBrcL6yJTyisuwJokZ9LTiYQwY3yRY3tmy83FlI,410
-learning_loop_node/data_exchanger.py,sha256=nd9JNPLn9amIeTcSIyUPpbE97ORAcb5yNphvmpgWSUQ,9095
+learning_loop_node/data_exchanger.py,sha256=Rlql4WSDgF6uUivH77deoIl2vnTKL1rlEhQmAUmJ9IU,10642
 learning_loop_node/globals.py,sha256=a50Qg4z2205Zcw4Cs8z417kn1x8s78o2FJfZB9-ZZI4,211
 learning_loop_node/loop_communication.py,sha256=d1tJZ9T_y22FrEq1XzvidgH6d-byGxFq3ovLvPBS4FA,7611
 learning_loop_node/node.py,sha256=CABqe-sqUlcwi9CYVXBkrfxh0Xgtzj3AFLQcupSX-Ms,12158
@@ -19,51 +19,52 @@ learning_loop_node/data_classes/training.py,sha256=TybwcCDf_NUaDUaOj30lPm-7Z3Qk9
 learning_loop_node/detector/__init__.py,sha256=wroy0W1X5MX16C-peA-UHYrcdXet7GhleIlC_Pqa6hI,128
 learning_loop_node/detector/categories.py,sha256=qgquuop3geKXXsNBurO0eC2b64h_ojL37RBYMv0ZzOg,1147
 learning_loop_node/detector/detector_logic.py,sha256=XJaHNiw5PaHoCWnJ-qsxdUELLevvztYx54TQ8KcHl18,1479
-learning_loop_node/detector/detector_node.py,sha256=GC5CaGcW_qVxyolDEz9U5W7y0ejZGEu9FZ__ckLxd7g,35728
+learning_loop_node/detector/detector_node.py,sha256=3e6UU1fl6IVkEgcfJTiOE-W30yebXcd5IpPOBYWV2mM,36950
 learning_loop_node/detector/exceptions.py,sha256=C6KbNPlSbtfgDrZx2Hbhm7Suk9jVoR3fMRCO0CkrMsQ,196
 learning_loop_node/detector/geometry.py,sha256=bZCF9xuK7qy6Ipv5aQYrTlwSmjaiUz5VEUYnc1zCe-w,1101
-learning_loop_node/detector/outbox.py,sha256=HT0GHv6rsQzdbCD8DAT23m1-Q1XEvA_KgwAJLONyX_0,12506
+learning_loop_node/detector/outbox.py,sha256=mtMxdcComjRyM6qc-O1fBv9ABCau4-oCCV442hMaryQ,12351
 learning_loop_node/detector/postprocess.py,sha256=fPBbgE6Obkej709zoFPGqA2gnbJNbpbMuZogilKy4pI,7274
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=slPGm87TbBCfDUdpRTAM5yOz1An8Br5X_2HFJOGSrFg,3540
-learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=81XQV6HST2hSH-In7VF8B4BfubgUdRHTLcc16VLjUxc,1518
+learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=kFwXIEE6ddkJ6qq8vfTLMAUP7FayOGgyrNHBhLtxVD0,1535
 learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/rest/about.py,sha256=evHJ2svUZY_DFz0FSef5u9c5KW4Uc3GL7EbPinG9-dg,583
 learning_loop_node/detector/rest/backdoor_controls.py,sha256=ZNaFOvC0OLWNtcLiG-NIqS_y1kkLP4csgk3CHhp8Gis,885
-learning_loop_node/detector/rest/detect.py,sha256=BHcQbSuC1sYHRKl2wNchiasb3A4HJk9tVia11e9TS6Y,2597
+learning_loop_node/detector/rest/detect.py,sha256=xr6VVCiH-wzvwTzTVAnzBR0vc_gTPwGL9OSSK2PJiOI,2518
 learning_loop_node/detector/rest/model_version_control.py,sha256=P4FOG0U9HT6QtCoNt-1s1pT6drtgdVjGZWEuCAyuNmA,1370
 learning_loop_node/detector/rest/operation_mode.py,sha256=eHOqbPdyfgzLtlz8sYs3WjESQ_kvxEIEP-cMuDQnsDI,1562
 learning_loop_node/detector/rest/outbox_mode.py,sha256=H8coDNbgLGEfXmKQrhtXWeUHBAHpnrdZktuHXQz0xis,1148
-learning_loop_node/detector/rest/upload.py,sha256=MditQ4Tu9PRzW6vhO-I9TJcnjOFpD83XnHx-6uUURkU,1708
+learning_loop_node/detector/rest/upload.py,sha256=qNXTLXY2b4A7Wu2GDhKU9bkfF8-QyFd-zdqWld3Jj0o,1483
 learning_loop_node/enums/__init__.py,sha256=tjSrhztIQ8W656_QuXfTbbVNtH_wDXP5hpYZgzfgRhc,285
 learning_loop_node/enums/annotator.py,sha256=mtTAw-8LJIrHcYkBjYHCZuhYEEHS6QzSK8k6BhLusvQ,285
 learning_loop_node/enums/detector.py,sha256=Qvm5LWWR9BfsDxHEQ8YzaPaUuSmp4BescYuV4X4ikwE,512
 learning_loop_node/enums/general.py,sha256=OSpcIpdFjuRByON2xBoiIXl1SgDREIqkwnaxkTFjnHU,164
-learning_loop_node/enums/trainer.py,sha256=VaD63guLO4aKgVfXT0EryPlXKQGegSET3Cp4R0uC_Ew,746
+learning_loop_node/enums/trainer.py,sha256=VdamD-k6Q8kMm_cwPA8iaEUfZvmTxLfsBHQpEi43Ahk,849
 learning_loop_node/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/helpers/background_tasks.py,sha256=sbbS8JO1ls0ZCbNW68KOT9QLOFyFoJRXwwAq78iOmoA,3256
 learning_loop_node/helpers/entrypoint.py,sha256=U4vHFqgke6mXjDL-E4MND8RO7P1k2VBGwdDOOoJvD9Y,3915
 learning_loop_node/helpers/environment_reader.py,sha256=zsoFysxNxBCOJzUwX4Pbm3ftYhxx5zsbOHQfD9VcfvA,1937
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
 learning_loop_node/helpers/log_conf.py,sha256=hqVAa_9NnYEU6N0dcOKmph82p7MpgKqeF_eomTLYzWY,961
-learning_loop_node/helpers/misc.py,sha256=qdN18Ukbw6eR6sFL4BxNq0EZzg2tVm2Cm8cTbFyy8QU,8931
+learning_loop_node/helpers/misc.py,sha256=ury0HOuUxDwdYNxmiDuaWOw77lUXv0DwqQhrHqoliY4,10158
 learning_loop_node/helpers/run.py,sha256=_uox-j3_K_bL3yCAwy3JYSOiIxrnhzVxyxWpCe8_J9U,876
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/batch_size.py,sha256=LjA-zyY4hdnSMy6ZsdQOhWbmd5H_DI-9LG3yFInv04E,5476
 learning_loop_node/trainer/cuda.py,sha256=ehpfWqPBCVFu06SnGsjmWbSDRJ8sZc-L_nCtemRLUOU,12567
 learning_loop_node/trainer/downloader.py,sha256=AIyEKM4XYtd6VgmXrP3VayV9DpJzdURK1Brx81ePNSM,1470
-learning_loop_node/trainer/exceptions.py,sha256=IRT7qvn7-RjXaqL8mF21ySm7JWhQtqm1Cl0Gi3DChVY,540
+learning_loop_node/trainer/exceptions.py,sha256=uCqGzni5CO8R8pHnVjsucuM4S00UxY35qxJaWIaqqjk,1186
 learning_loop_node/trainer/executor.py,sha256=XQ1TwgOF2pQv7bgzSaljDWG4I1ySjYXL8r9c_p9MX1Q,3947
-learning_loop_node/trainer/io_helpers.py,sha256=zWl0-HLP8zxOG3ybm76vXgEozULj_1QRl95jkrF126k,7255
+learning_loop_node/trainer/io_helpers.py,sha256=fmSh2-9sEfBavYxfa_zT4FfZMByT3KxDbA3O57I8iBY,7834
 learning_loop_node/trainer/metrics.py,sha256=H1jQh5fHN3-cTdMTuH8mF_uODV0T9DBdqcPUD-ZqFAI,1439
 learning_loop_node/trainer/subprocess.py,sha256=WxuTly0zhpClEjTd9wqoI_-7YTNMaiZpRHxu00vttKw,2647
 learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
 learning_loop_node/trainer/trainer_logic.py,sha256=Yaw1GNTC05rVLQczY6vKUgkymBYfm3_0XaR5C0q-CP4,9159
-learning_loop_node/trainer/trainer_logic_generic.py,sha256=IRP68ZGpnOX4OS03OwB7HmTUvk6nbBRmCjMCVupGE_Q,27196
-learning_loop_node/trainer/trainer_node.py,sha256=ZnydfHJ2ehAhb821Xke_09qthCE3vDf7iqeYUrliSSo,4938
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=fNvH00Bnu6LfOFVJlCB3WyDccN9sC8tKnPAD-7uZoro,30033
+learning_loop_node/trainer/trainer_node.py,sha256=GU4LyFA-JC9s4d-PbyKhKWyGMMIzTMqKJFNxM6tDJE4,5316
+learning_loop_node/trainer/training_slot.py,sha256=Njq6Z0T2Lqlr24ftoIp11Pw2OxCo9Z2RGn1tuFhHqLY,6737
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/rest/backdoor_controls.py,sha256=ZnK8ypY5r_q0-YZbtaOxhQThzuZvMsQHM5gJGESd_dE,5131
-learning_loop_node-0.24.0.dist-info/METADATA,sha256=rQNtas26mLdkkjI37mJ_jteRe7RtUO_3DpReanEUC2U,18609
-learning_loop_node-0.24.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-learning_loop_node-0.24.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
-learning_loop_node-0.24.0.dist-info/RECORD,,
+learning_loop_node-0.26.0.dist-info/METADATA,sha256=F-zF0LzeauG_ngUiRE8gDZO7qkY-gbdxjM7J9YMZe3g,20215
+learning_loop_node-0.26.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+learning_loop_node-0.26.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
+learning_loop_node-0.26.0.dist-info/RECORD,,

{learning_loop_node-0.24.0.dist-info → learning_loop_node-0.26.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (84.0.0)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any