learning-loop-node 0.27.2__py3-none-any.whl → 0.29.0__py3-none-any.whl

learning_loop_node/data_classes/__init__.py

@@ -26,10 +26,11 @@ from .general import (
     NodeState,
     NodeStatus,
 )
-from .image_metadata import ImageMetadata, ImagesMetadata
+from .image_metadata import CaptureMetadata, ImageMetadata, ImagesMetadata
 from .socket_response import SocketResponse
 from .training import (
     Errors,
+    Hyperparameter,
     PretrainedModel,
     Training,
     TrainingError,
@@ -43,8 +44,8 @@ __all__ = [
     'BoxDetection', 'ClassificationDetection', 'ImageMetadata', 'Observation', 'Point', 'PointDetection',
     'SegmentationDetection', 'Shape', 'Detections',
     'AnnotationNodeStatus', 'Category', 'Context', 'DetectorStatus', 'ErrorConfiguration',
-    'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse', 'ImagesMetadata',
+    'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse', 'ImagesMetadata', 'CaptureMetadata',
     'SocketResponse',
-    'Errors', 'PretrainedModel', 'Training',
+    'Errors', 'Hyperparameter', 'PretrainedModel', 'Training',
     'TrainingError', 'TrainingOut', 'TrainingStateData', 'TrainingStatus',
 ]

learning_loop_node/data_classes/image_metadata.py

@@ -56,3 +56,16 @@ class ImageMetadata():
 @dataclass(**KWONLY_SLOTS)
 class ImagesMetadata():
     items: List[ImageMetadata] = field(default_factory=list, metadata={'description': 'List of image metadata'})
+
+
+@dataclass(**KWONLY_SLOTS)
+class CaptureMetadata:
+    """Where an image comes from and how it is tagged."""
+    camera_id: str | None = field(default=None, metadata={
+        'description': 'Camera the image was taken with; selects the history of the relevance filter'})
+    tags: list[str] = field(default_factory=list, metadata={
+        'description': 'Tags of the image'})
+    source: str | None = field(default=None, metadata={
+        'description': 'Source of the image'})
+    creation_date: str | None = field(default=None, metadata={
+        'description': 'Creation date of the image'})

learning_loop_node/data_classes/training.py

@@ -3,7 +3,7 @@ import sys
 import time
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
 from uuid import uuid4
 
 from ..enums import TrainerState
@@ -22,6 +22,28 @@ class PretrainedModel():
     description: str
 
 
+HyperparameterType = Literal['int', 'float', 'bool', 'str', 'pairs']
+
+
+@dataclass(**KWONLY_SLOTS)
+class Hyperparameter:
+    """One hyperparameter a trainer accepts, reported to the loop so it can offer it in the project settings.
+
+    Plain data only, so it reaches the loop as it is. ``minimum`` and ``maximum`` apply to ``int`` and ``float``,
+    ``choices`` to ``str``. A ``pairs`` value is text of the form ``"key:value,..."``, which ``parse_pairs`` reads.
+    ``notes`` help whoever sets the value. ``expert`` ones are offered behind an expansion.
+    Whether a value can be changed when a training is started is the project's choice, not the trainer's.
+    """
+    name: str
+    type: HyperparameterType
+    default: Any
+    minimum: float | None = None
+    maximum: float | None = None
+    choices: list[str] | None = None
+    notes: str = ''
+    expert: bool = False
+
+
 @dataclass(**KWONLY_SLOTS)
 class TrainingStatus():
     id: str  # NOTE this must not be changed, but tests wont detect a change -> update tests!
@@ -33,6 +55,7 @@ class TrainingStatus():
     progress: Optional[float]
 
     pretrained_models: List[PretrainedModel] = field(default_factory=list)
+    hyperparameters: list[Hyperparameter] = field(default_factory=list)
     architecture: Optional[str] = None
     context: Optional[Context] = None
 

learning_loop_node/data_exchanger.py

@@ -14,7 +14,12 @@ import aiofiles  # type: ignore
 from .data_classes import Context
 from .helpers.misc import create_resource_paths, create_task, is_valid_image, is_valid_uuid4
 from .loop_communication import LoopCommunicator
-from .trainer.exceptions import CriticalError, ModelUploadFailedError
+from .trainer.exceptions import (
+    CriticalError,
+    ModelUploadFailedError,
+    TrainingResultsNotAnsweredError,
+    TrainingResultsUploadFailedError,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -207,6 +212,23 @@ class DataExchanger():
                     training_number, mformat, uploaded_model)
         return uploaded_model['id']
 
+    async def upload_training_results(self, context: Context, training_number: int, part: str, zip_path: str) -> None:
+        """Store one part of a training's results beside its latest model; a part sent again replaces the stored one.
+
+        :raise TrainingResultsUploadFailedError: If the loop answered with a client error other than 408.
+        :raise TrainingResultsNotAnsweredError: If the loop answered 408 or a server error.
+        """
+        path = (f'/{context.organization}/projects/{context.project}/trainings/{training_number}'
+                f'/training_results/{part}')
+        response = await self.loop_communicator.put(path, files=[zip_path], timeout=60*10)
+        if response.status_code == HTTPStatus.REQUEST_TIMEOUT or response.status_code >= 500:
+            raise TrainingResultsNotAnsweredError(f'The loop did not answer the upload of the {part} results of '
+                                                  f'training {training_number}: {response.status_code} {response.text}')
+        if response.status_code != HTTPStatus.OK:
+            raise TrainingResultsUploadFailedError(f'The loop did not store the {part} results of training '
+                                                   f'{training_number}: {response.status_code} {response.text}')
+        logger.info('Uploaded the %s results of training %s', part, training_number)
+
 
 def _rejects_upload(status_code: int) -> bool:
     """A client error other than 408 and 429 answers the same however often the upload is sent."""

learning_loop_node/detector/detector_node.py

@@ -19,6 +19,7 @@ from socketio import AsyncClient
 
 from ..data_classes import (
     AboutResponse,
+    CaptureMetadata,
     Context,
     DetectorStatus,
     ImageMetadata,
@@ -272,10 +273,16 @@ class DetectorNode(Node):
 
             Data dict follows the schema of the detect endpoint,
             but 'images' is a list of image dicts.
+            Each image dict may also carry its own:
+            - camera_id: replaces the camera_id of the batch
+            - tags: added to the tags of the batch
+            - source: replaces the source of the batch
+            - creation_date: replaces the creation_date of the batch
             """
             try:
                 images_data = data['images']
                 images = [image_from_dict(image) for image in images_data]
+                capture_metadata = [_capture_metadata(data, image) for image in images_data]
             except Exception:
                 self.log.exception('could not parse images from socketio')
                 return {'error': 'could not parse images from data'}
@@ -283,11 +290,8 @@ class DetectorNode(Node):
             try:
                 det = await self.get_batch_detections(
                     images=images,
-                    tags=data.get('tags', []),
-                    camera_id=data.get('camera_id', None),
-                    source=data.get('source', None),
+                    capture_metadata=capture_metadata,
                     autoupload=data.get('autoupload', 'filtered'),
-                    creation_date=data.get('creation_date', None),
                 )
                 return jsonable_encoder(asdict(det))
             except DetectorUnavailableError as e:
@@ -638,35 +642,17 @@ class DetectorNode(Node):
         array = await _pixel_array(image)
         async with self.detection_lock:
             detector = _unwrap_detector(self._detector)
-            metadata = await run.io_bound(detector.logic.evaluate, array)
-
-        metadata.tags.extend(tags)
-        metadata.source = source
-        metadata.created = creation_date
-
-        fix_shape_detections(metadata)
-        n_bo, n_cl = len(metadata.box_detections), len(metadata.classification_detections)
-        n_po, n_se = len(metadata.point_detections), len(metadata.segmentation_detections)
-        self.log.debug('Detected: %d boxes, %d points, %d segs, %d classes', n_bo, n_po, n_se, n_cl)
+            detections = await run.io_bound(detector.logic.evaluate, array)
 
-        if autoupload == 'filtered':
-            background_tasks.create(self.relevance_filter.may_upload_detections(metadata, camera_id, image))
-        elif autoupload == 'all':
-            background_tasks.create(self.outbox.save(image, metadata))
-        elif autoupload == 'disabled':
-            pass
-        else:
-            self.log.error('unknown autoupload value %s', autoupload)
-        return metadata
+        capture = CaptureMetadata(camera_id=camera_id, tags=tags, source=source, creation_date=creation_date)
+        self._finish_detection(detections, image, capture, autoupload)
+        return detections
 
     async def get_batch_detections(self,
                                    images: Sequence[ImageData],
-                                   tags: List[str],
+                                   capture_metadata: Sequence[CaptureMetadata],
                                    *,
-                                   camera_id: Optional[str] = None,
-                                   source: Optional[str] = None,
-                                   autoupload: str = 'filtered',
-                                   creation_date: Optional[str] = None) -> ImagesMetadata:
+                                   autoupload: str = 'filtered') -> ImagesMetadata:
         """
         Processing function for the detector node when a batch inference is requested via SocketIO.
 
@@ -674,35 +660,45 @@ class DetectorNode(Node):
         cares about uploading to the loop and returns the detections as a list of ImageMetadata.
         Each image is a pixel array or JPEG bytes; JPEG bytes are decoded for the inference
         and stored as they are if the image is uploaded.
+        `capture_metadata` holds the camera, tags, source and creation date of each image.
         Raises exception if no model is loaded.
         """
+        if len(capture_metadata) != len(images):
+            raise ValueError(f'got {len(capture_metadata)} capture metadata for {len(images)} images')
         _unwrap_detector(self._detector)  # fails fast, before the decode
         arrays = await asyncio.gather(*(_pixel_array(image) for image in images))
         async with self.detection_lock:
             detector = _unwrap_detector(self._detector)
             all_detections = await run.io_bound(detector.logic.batch_evaluate, arrays)
 
-        for metadata in all_detections.items:
-            metadata.tags.extend(tags)
-            metadata.source = source
-            metadata.created = creation_date
-
-        for detections, image in zip(all_detections.items, images):
-            fix_shape_detections(detections)
-            n_bo, n_cl = len(detections.box_detections), len(detections.classification_detections)
-            n_po, n_se = len(detections.point_detections), len(detections.segmentation_detections)
-            self.log.debug('Detected: %d boxes, %d points, %d segs, %d classes', n_bo, n_po, n_se, n_cl)
-
-            if autoupload == 'filtered':
-                background_tasks.create(self.relevance_filter.may_upload_detections(detections, camera_id, image))
-            elif autoupload == 'all':
-                background_tasks.create(self.outbox.save(image, detections))
-            elif autoupload == 'disabled':
-                pass
-            else:
-                self.log.error('unknown autoupload value %s', autoupload)
+        for detections, image, capture in zip(all_detections.items, images, capture_metadata):
+            self._finish_detection(detections, image, capture, autoupload)
         return all_detections
 
+    def _finish_detection(self,
+                          detections: ImageMetadata,
+                          image: ImageData,
+                          capture: CaptureMetadata,
+                          autoupload: str) -> None:
+        detections.tags.extend(capture.tags)
+        detections.source = capture.source
+        detections.created = capture.creation_date
+
+        fix_shape_detections(detections)
+        n_bo, n_cl = len(detections.box_detections), len(detections.classification_detections)
+        n_po, n_se = len(detections.point_detections), len(detections.segmentation_detections)
+        self.log.debug('Detected: %d boxes, %d points, %d segs, %d classes', n_bo, n_po, n_se, n_cl)
+
+        if autoupload == 'filtered':
+            background_tasks.create(
+                self.relevance_filter.may_upload_detections(detections, capture.camera_id, image))
+        elif autoupload == 'all':
+            background_tasks.create(self.outbox.save(image, detections))
+        elif autoupload == 'disabled':
+            pass
+        else:
+            self.log.error('unknown autoupload value %s', autoupload)
+
     async def upload_images(
             self, *,
             images: Sequence[ImageData],
@@ -785,6 +781,23 @@ async def _pixel_array(image: ImageData) -> np.ndarray:
     return await run.io_bound(jpg_bytes_to_numpy_array, image)
 
 
+def _capture_metadata(batch_data: dict, image_data: dict) -> CaptureMetadata:
+    """The values of one image dict of a batch: its own where it sets them, the batch's otherwise, and both tags."""
+    image_tags = image_data.get('tags') or []
+    if not isinstance(image_tags, list):
+        raise ValueError(f'tags of an image must be a list, got {type(image_tags).__name__}')
+    return CaptureMetadata(camera_id=_own_or_batch('camera_id', image_data, batch_data),
+                           tags=[*batch_data.get('tags', []), *image_tags],
+                           source=_own_or_batch('source', image_data, batch_data),
+                           creation_date=_own_or_batch('creation_date', image_data, batch_data))
+
+
+def _own_or_batch(key: str, image_data: dict, batch_data: dict) -> str | None:
+    """`None` counts as not set, so an image cannot clear a value of the batch."""
+    own = image_data.get(key)
+    return own if own is not None else batch_data.get(key)
+
+
 def _unwrap_detector(state: '_DetectorState') -> _ActiveDetector:
     """Return the active detector or raise DetectorUnavailableError."""
     match state:

learning_loop_node/trainer/exceptions.py

@@ -11,10 +11,29 @@ class ModelUploadFailedError(Exception):
     """
 
 
+class TrainingResultsUploadFailedError(Exception):
+    """Raised when the loop answered the upload of a part of a training's results with a client error other than 408.
+    The answer is final: sending the part again would get the same answer.
+    """
+
+
+class TrainingResultsNotAnsweredError(Exception):
+    """Raised when the loop gave up waiting for a part of a training's results (408) or answered with a server error;
+    the part may be sent again.
+    """
+
+
 class TrainingRefusedError(Exception):
     """Raised when a trainer cannot take on a training it was asked to begin; nothing was set up."""
 
 
+class HyperparameterMismatchError(Exception):
+    """Raised when a training's hyperparameters do not fit the trainer's declaration; nothing was set up.
+
+    Unlike a refusal, the training would fail on any trainer of the same declaration.
+    """
+
+
 class TrainingSlotTakenError(TrainingRefusedError):
     """Raised when a sibling node holds the training slot."""
 

learning_loop_node/trainer/hyperparameters.py

@@ -0,0 +1,180 @@
+"""Resolving the hyperparameters a training was sent against the ones its trainer declares.
+
+The loop sends only what the project sets, plus what the job overrides. Everything else comes from
+the trainer's declared defaults, so after :func:`resolve` a training holds the full set it runs
+with, and that is what gets reported back to the loop.
+"""
+
+import contextlib
+import logging
+import math
+from collections.abc import Mapping, Sequence
+from typing import Any, get_args
+
+from ..data_classes import Hyperparameter
+from ..data_classes.training import HyperparameterType
+from .batch_size import REQUESTED_BATCH_SIZE
+from .exceptions import CriticalError
+from .validation_set import COMPUTE_VALIDATION_SET
+
+logger = logging.getLogger(__name__)
+
+LIBRARY_NAMES = (REQUESTED_BATCH_SIZE, COMPUTE_VALIDATION_SET)
+"""Hyperparameters the library reads itself, so a trainer that does not declare them still uses them."""
+
+BOOL_STRINGS = {'true': True, '1': True, 'yes': True, 'on': True,
+                'false': False, '0': False, 'no': False, 'off': False}
+
+
+def resolve(declared: Sequence[Hyperparameter], received: Mapping[str, Any]) -> dict[str, Any]:
+    """Coerce and check ``received`` against ``declared`` and fill in the declared defaults.
+
+    ``declared`` is trusted to have passed :func:`check_declaration`. Names the trainer does not declare are kept
+    unchanged: old project configurations still carry them, and a resolved training carries the values its trainer
+    reported. An unset hyperparameter with a default of ``None`` is left out, so the trainer's own fallback applies.
+    Resolving twice gives the same result.
+
+    :raises CriticalError: If a value cannot be coerced to its type, is out of range or not one of its choices.
+    """
+    resolved = dict(received)
+    for hyperparameter in declared:
+        value = received.get(hyperparameter.name)
+        if _is_unset(hyperparameter, value):
+            if hyperparameter.default is None:
+                resolved.pop(hyperparameter.name, None)
+            else:
+                resolved[hyperparameter.name] = _coerce(hyperparameter, hyperparameter.default)
+        else:
+            resolved[hyperparameter.name] = _check(hyperparameter, _coerce(hyperparameter, value))
+    return resolved
+
+
+def check_declaration(declared: Sequence[Hyperparameter]) -> None:
+    """Check that ``declared`` is consistent, so that :func:`resolve` can rely on it.
+
+    A trainer's declaration does not change while it runs, so the node checks it once, when it is built. A default of
+    ``None`` stands for "the trainer decides" and is not checked.
+
+    :raises ValueError: If a name is declared twice, a hyperparameter contradicts itself or its default does not fit.
+    """
+    names = [h.name for h in declared]
+    if duplicates := sorted({name for name in names if names.count(name) > 1}):
+        raise ValueError(f'the trainer declares these hyperparameters twice: {", ".join(duplicates)}')
+    for hyperparameter in declared:
+        _check_one_declaration(hyperparameter)
+
+
+def parse_pairs(text: str) -> list[tuple[str, str]]:
+    """The ``key:value`` items of a ``pairs`` hyperparameter such as ``"uuid:uuid,uuid:uuid"``.
+
+    Empty items and the spaces around keys and values are skipped, so ``''`` holds no pairs.
+
+    :raises ValueError: If an item is not a key and a value separated by a single colon.
+    """
+    pairs = []
+    for item in (i.strip() for i in text.split(',')):
+        if not item:
+            continue
+        key, _, value = (part.strip() for part in item.partition(':'))
+        if not key or not value or ':' in value:
+            raise ValueError(f'"{item}" is not of the form "key:value"')
+        pairs.append((key, value))
+    return pairs
+
+
+def warn_about_undeclared(declared: Sequence[Hyperparameter], received: Mapping[str, Any]) -> None:
+    """Log the names in ``received`` that neither the trainer nor the library knows; they are not rejected."""
+    unknown = sorted(set(received) - {h.name for h in declared} - set(LIBRARY_NAMES))
+    if unknown:
+        logger.warning('these hyperparameters are not declared by this trainer: %s', ', '.join(unknown))
+
+
+def _check_one_declaration(hyperparameter: Hyperparameter) -> None:
+    def contradiction(reason: str) -> ValueError:
+        return ValueError(f"the trainer declares hyperparameter '{hyperparameter.name}' {reason}")
+
+    if hyperparameter.type not in get_args(HyperparameterType):
+        raise contradiction(f'with the unknown type {hyperparameter.type!r}')
+    if hyperparameter.type not in ('int', 'float') and (hyperparameter.minimum is not None or
+                                                        hyperparameter.maximum is not None):
+        raise contradiction(f'as {hyperparameter.type} with a minimum or maximum')
+    if hyperparameter.type != 'str' and hyperparameter.choices:
+        raise contradiction(f'as {hyperparameter.type} with choices')
+    for bound in (hyperparameter.minimum, hyperparameter.maximum):
+        if bound is not None and (isinstance(bound, bool) or not isinstance(bound, (int, float))):
+            raise contradiction(f'with the bound {bound!r}, which is not a number')
+    if (hyperparameter.minimum is not None and hyperparameter.maximum is not None and
+            hyperparameter.minimum > hyperparameter.maximum):
+        raise contradiction('with a minimum above its maximum')
+    if hyperparameter.default is not None:
+        try:
+            _check(hyperparameter, _coerce(hyperparameter, hyperparameter.default))
+        except CriticalError as e:
+            raise contradiction(f'with a default that does not fit: {e}') from None
+
+
+def _is_unset(hyperparameter: Hyperparameter, value: Any) -> bool:
+    """``None``, or the empty value the loop sends for a field left blank, unless it fits as a value of its own."""
+    if value is None:
+        return True
+    if value != '':
+        return False
+    try:
+        _check(hyperparameter, _coerce(hyperparameter, value))
+    except CriticalError:
+        return True
+    return False
+
+
+def _invalid(hyperparameter: Hyperparameter, requirement: str, value: Any) -> CriticalError:
+    return CriticalError(f"hyperparameter '{hyperparameter.name}' {requirement}, got {value!r}")
+
+
+def _coerce(hyperparameter: Hyperparameter, value: Any) -> Any:
+    if hyperparameter.type == 'pairs':
+        if not isinstance(value, str):
+            raise _invalid(hyperparameter, 'must be text of key:value pairs', value)
+        try:
+            return ','.join(f'{key}:{item}' for key, item in parse_pairs(value))
+        except ValueError as e:
+            raise CriticalError(f"hyperparameter '{hyperparameter.name}': {e}") from None
+    if hyperparameter.type == 'str':
+        if isinstance(value, bool) or not isinstance(value, (str, int, float)):
+            raise _invalid(hyperparameter, 'must be text', value)
+        return str(value)
+    if hyperparameter.type == 'bool':
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str) and (word := value.strip().lower()) in BOOL_STRINGS:
+            return BOOL_STRINGS[word]
+        with contextlib.suppress(TypeError, ValueError):
+            number = float(value)
+            if math.isfinite(number):
+                return number != 0
+        raise _invalid(hyperparameter, 'must be true or false', value)
+    if isinstance(value, bool):
+        raise _invalid(hyperparameter, 'must be a number', value)
+    if hyperparameter.type == 'int' and isinstance(value, (int, str)):
+        with contextlib.suppress(ValueError):
+            return int(value)
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        raise _invalid(hyperparameter, 'must be a number', value) from None
+    if not math.isfinite(number):
+        raise _invalid(hyperparameter, 'must be finite', value)
+    if hyperparameter.type == 'float':
+        return number
+    if not number.is_integer():
+        raise _invalid(hyperparameter, 'must be a whole number', value)
+    return int(number)
+
+
+def _check(hyperparameter: Hyperparameter, value: Any) -> Any:
+    if hyperparameter.minimum is not None and value < hyperparameter.minimum:
+        raise _invalid(hyperparameter, f'must be at least {hyperparameter.minimum}', value)
+    if hyperparameter.maximum is not None and value > hyperparameter.maximum:
+        raise _invalid(hyperparameter, f'must be at most {hyperparameter.maximum}', value)
+    if hyperparameter.choices and value not in hyperparameter.choices:
+        raise _invalid(hyperparameter, f"must be one of {', '.join(hyperparameter.choices)}", value)
+    return value

learning_loop_node/trainer/trainer_logic.py

@@ -26,7 +26,6 @@ class TrainerLogic(TrainerLogicGeneric):
         self._detection_progress: Optional[float] = None
         self._executor: Optional[Executor] = None
         self._process_started: bool = False
-        self._stop_requested: bool = False
         self.start_training_task: Optional[Coroutine] = None
         self.inference_batch_size = self._environment_vars.inference_batch_size
 

learning_loop_node/trainer/trainer_logic_generic.py

@@ -6,6 +6,7 @@ import sys
 import time
 from abc import ABC, abstractmethod
 from dataclasses import asdict
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Coroutine, Dict, List, Optional
 
 from fastapi.encoders import jsonable_encoder
@@ -13,6 +14,7 @@ from fastapi.encoders import jsonable_encoder
 from ..data_classes import (
     Context,
     Errors,
+    Hyperparameter,
     PretrainedModel,
     Training,
     TrainingOut,
@@ -25,8 +27,10 @@ from ..helpers.misc import (
     delete_all_training_folders,
     is_valid_uuid4,
 )
+from . import training_results
 from .downloader import TrainingsDownloader
-from .exceptions import CriticalError, NodeNeedsRestartError, TrainingRefusedError
+from .exceptions import CriticalError, HyperparameterMismatchError, NodeNeedsRestartError, TrainingRefusedError
+from .hyperparameters import resolve, warn_about_undeclared
 from .io_helpers import ActiveTrainingIO, EnvironmentVars, LastTrainingIO
 from .training_slot import TrainingSlot, training_slot_from_env
 from .validation_set import assign_validation_set
@@ -55,6 +59,7 @@ class TrainerLogicGeneric(ABC):
         self.model_format: str = model_format
         self.errors = Errors()
         self._state_failures: Dict[str, int] = {}  # failed attempts in a row, by error key
+        self._stop_requested: bool = False  # set by a trainer that stopped its training process; not kept on restart
 
         self.training_task: Optional[asyncio.Task] = None
         self.shutdown_event: asyncio.Event = asyncio.Event()
@@ -186,6 +191,12 @@ class TrainerLogicGeneric(ABC):
         """
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def provided_hyperparameters(self) -> list[Hyperparameter]:
+        """Returns the hyperparameters this trainer accepts; an empty list leaves them unchecked."""
+        raise NotImplementedError
+
     # ---------------------------------------- METHODS ----------------------------------------
 
     # NOTE: Trainings are started by the Learning Loop via the begin_training event
@@ -204,6 +215,7 @@ class TrainerLogicGeneric(ABC):
                                 progress=self.general_progress)
 
         status.pretrained_models = self.provided_pretrained_models
+        status.hyperparameters = self.provided_hyperparameters
         status.architecture = self.model_architecture
 
         if self._training:
@@ -224,9 +236,18 @@ class TrainerLogicGeneric(ABC):
 
     def _init_from_last_training(self) -> None:
         """Initializes a new training object from the last training saved on disc via last_training_io.
+
+        The hyperparameters are resolved again, since the training may have been saved before this trainer declared
+        them. If one no longer fits, the training continues with the values it was saved with, since it already runs
+        with them.
         """
         self._training = self.last_training_io.load()
         assert self._training is not None and self._training.training_folder is not None, 'could not restore training folder'
+        try:
+            self._training.hyperparameters = resolve(self.provided_hyperparameters, self._training.hyperparameters)
+        except CriticalError as e:
+            logger.warning('restored training does not fit the declared hyperparameters, continuing with the saved '
+                           'values: %s', e)
         logger.info('restored training: \n%s', self._training)
         self._active_training_io = ActiveTrainingIO(
             self._training.training_folder, self.node.loop_communicator, self._training.context)
@@ -234,13 +255,15 @@ class TrainerLogicGeneric(ABC):
     async def begin_training(self, organization: str, project: str, training_config: Dict) -> None:
         """Called on `begin_training` event from the Learning Loop.
 
-        The training slot is taken before anything is set up, so a refused training leaves the
-        previous training folders in place.
+        The hyperparameters are resolved and the training slot is taken before anything is set up, so a refused
+        training leaves the previous training folders in place.
 
         :raise TrainingRefusedError: if a training is active or a sibling node holds the training slot.
+        :raise HyperparameterMismatchError: if a hyperparameter does not fit its declaration.
         """
         if self.training_active:
             raise TrainingRefusedError(f'training {self.training.id} is still active')
+        training_config = self._with_resolved_hyperparameters(training_config)
         context = Context(organization=organization, project=project)
         self.training_slot.claim(self._slot_claim(context, training_config.get('training_number')))
         try:
@@ -250,6 +273,24 @@ class TrainerLogicGeneric(ABC):
             raise
         self._begin_training_task()
 
+    def _with_resolved_hyperparameters(self, training_config: dict) -> dict:
+        """Returns ``training_config`` with its hyperparameters resolved against the declared ones.
+
+        A config without a hyperparameters dict is returned as it is, for ``Training.generate_training`` to reject.
+
+        :raise HyperparameterMismatchError: if a hyperparameter does not fit its declaration.
+        """
+        declared = self.provided_hyperparameters
+        received = training_config.get('hyperparameters')
+        if not declared or not isinstance(received, dict):
+            return training_config
+        try:
+            resolved = resolve(declared, received)
+        except CriticalError as e:
+            raise HyperparameterMismatchError(str(e)) from e
+        warn_about_undeclared(declared, received)
+        return {**training_config, 'hyperparameters': resolved}
+
     def _begin_training_task(self) -> None:
         # NOTE: Task object is used to potentially cancel the task
         self.training_task = asyncio.get_event_loop().create_task(self._run())
@@ -378,7 +419,11 @@ class TrainerLogicGeneric(ABC):
         self.last_training_io.save(self.training)
 
     async def _prepare(self) -> None:
-        """Downloads images to the images_folder and saves annotations to training.data.image_data."""
+        """Downloads images to the images_folder and saves annotations to training.data.image_data.
+        Moves part of the train images to `val` if `compute_validation_set` asks for it (see `validation_set`).
+        A malformed `upload_examples` or `upload_logs` ends the training here, before it has run.
+        """
+        training_results.check_hyperparameters(self.training.hyperparameters)
         self.node.data_exchanger.set_context(self.training.context)
         downloader = TrainingsDownloader(self.node.data_exchanger)
         image_data, skipped_image_count = await downloader.download_training_data(self.training.images_folder)
@@ -451,11 +496,16 @@ class TrainerLogicGeneric(ABC):
         self.errors.reset(error_key)
 
     async def _upload_model(self) -> None:
-        """Uploads the latest model to the Learning Loop.
+        """Uploads the latest model and then its training results to the Learning Loop.
+        The results never fail the state (see `training_results`).
         """
         new_model_uuid = await self._upload_model_return_new_model_uuid(self.training.context)
         logger.info('Successfully uploaded model and received new model id: %s', new_model_uuid)
         self.training.model_uuid_for_detecting = new_model_uuid
+        await training_results.upload_results(self.training, model_uuid=new_model_uuid, node_name=self.node.name,
+                                              node_uuid=self.node.uuid, get_results=self._get_training_results,
+                                              stop_requested=self._stop_requested,
+                                              data_exchanger=self.node.data_exchanger)
 
     async def _upload_model_return_new_model_uuid(self, context: Context) -> str:
         """Upload model files, usually pytorch model (.pt) hyp.yaml and the converted .wts file.
@@ -620,6 +670,14 @@ class TrainerLogicGeneric(ABC):
         """
         raise NotImplementedError
 
+    def _get_training_results(self, training_folder: Path) -> dict[str, list[Path]]:
+        """Called when the model is uploaded. Returns the result files of the training by part, stored beside the model.
+        The parts are `core` (always uploaded) and `examples` and `logs` (uploaded only if `upload_examples` and
+        `upload_logs` are true); see `training_results`. Directories go in as a whole. Name only results, never the
+        dataset. Default: no results, and nothing is uploaded.
+        """
+        return {}
+
     def _get_model_metadata(self) -> dict[str, Any]:
         """Return the values the trainer determined for this model, to be merged into its model.json.
 

learning_loop_node/trainer/trainer_node.py

@@ -10,7 +10,8 @@ from socketio import AsyncClient, exceptions
 from ..data_classes import SocketResponse
 from ..enums import TrainerState
 from ..node import Node
-from .exceptions import TrainingRefusedError
+from .exceptions import HyperparameterMismatchError, TrainingRefusedError
+from .hyperparameters import check_declaration
 from .io_helpers import LastTrainingIO
 from .rest import backdoor_controls
 from .trainer_logic_generic import TrainerLogicGeneric
@@ -19,6 +20,8 @@ from .trainer_logic_generic import TrainerLogicGeneric
 class TrainerNode(Node):
 
     def __init__(self, name: str, trainer_logic: TrainerLogicGeneric, uuid: Optional[str] = None, use_backdoor_controls: bool = False):
+        """:raise ValueError: if the trainer declares hyperparameters that contradict themselves."""
+        check_declaration(trainer_logic.provided_hyperparameters)
         super().__init__(name, uuid=uuid, node_type='trainer')
         trainer_logic._node = self
         self.trainer_logic = trainer_logic
@@ -72,6 +75,10 @@ class TrainerNode(Node):
             except TrainingRefusedError as e:
                 self.log.warning('refused begin_training: %s', e)
                 return SocketResponse(success=False, error_msg=str(e), payload='refused')
+            except HyperparameterMismatchError as e:
+                # NOTE: not a refusal, which the loop answers by offering the same job again
+                self.log.warning('rejected begin_training: %s', e)
+                return SocketResponse.for_failure(str(e))
             return True
 
         @sio_client.event

learning_loop_node/trainer/training_results.py

@@ -0,0 +1,187 @@
+"""The results of a training — metrics, curves, logs — uploaded to the loop beside its model.
+
+A trainer names its result files in `_get_training_results`, grouped into the parts `core`, `examples` and
+`logs`. `upload_results` sends them once the model is uploaded and never fails the training: a part the loop
+answered is done, a part that got no answer or a server error is sent up to `MAX_ATTEMPTS` times, and
+anything else that goes wrong is logged and skips the part. `core` is always uploaded; `examples` and `logs`
+only if the training's `upload_examples` and `upload_logs` hyperparameters ask for them. Each part goes up as
+one zip that keeps the paths inside the training folder, and `core` additionally holds `meta.json`, where
+the results come from.
+"""
+import asyncio
+import json
+import logging
+import os
+import zipfile
+from collections.abc import Callable, Mapping
+from datetime import datetime, timezone
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from ..data_classes import Training
+from ..data_exchanger import DataExchanger
+from .exceptions import CriticalError, TrainingResultsNotAnsweredError, TrainingResultsUploadFailedError
+from .validation_set import bool_hyperparameter
+
+CORE = 'core'
+EXAMPLES = 'examples'
+LOGS = 'logs'
+PARTS = (CORE, EXAMPLES, LOGS)
+
+UPLOAD_EXAMPLES = 'upload_examples'
+UPLOAD_LOGS = 'upload_logs'
+OPTIONAL_PARTS = {EXAMPLES: UPLOAD_EXAMPLES, LOGS: UPLOAD_LOGS}
+"""The parts uploaded only if the hyperparameter they map to is true."""
+
+META_FILE = 'meta.json'
+RESULTS_FOLDER = 'training_results'
+"""Inside the training folder: the zips and the record of the parts that are done."""
+DONE_FILE = 'uploaded_parts.txt'
+
+MAX_ATTEMPTS = 3
+RETRY_DELAY = 5  # seconds before the second attempt, doubled before each further one
+
+COMPRESSED_SUFFIXES = {'.jpg', '.jpeg', '.png', '.webp', '.gif', '.mp4', '.zip', '.gz', '.tgz', '.xz', '.7z'}
+"""Files stored in a zip as they are; compressing them again costs time and saves almost nothing."""
+
+logger = logging.getLogger(__name__)
+
+
+def check_hyperparameters(hyperparameters: Mapping[str, Any]) -> None:
+    """Raise a CriticalError if `upload_examples` or `upload_logs` is set to anything but a boolean."""
+    for name in OPTIONAL_PARTS.values():
+        bool_hyperparameter(hyperparameters, name)
+
+
+async def upload_results(training: Training, *, model_uuid: str, node_name: str, node_uuid: str,
+                         get_results: Callable[[Path], Mapping[str, list[Path]]], stop_requested: bool,
+                         data_exchanger: DataExchanger) -> None:
+    """Upload the parts of `get_results(training_folder)` the training's hyperparameters ask for; never raises.
+
+    A part is recorded as done once it is answered or skipped, so a second call sends only the parts left.
+    """
+    training_folder = Path(training.training_folder)
+    results_folder = training_folder / RESULTS_FOLDER
+    try:
+        results = get_results(training_folder)
+        parts = _parts_to_upload(results, training.hyperparameters)
+        meta_content = _meta(training, model_uuid=model_uuid, node_name=node_name, node_uuid=node_uuid,
+                             stop_requested=stop_requested)
+        done = _load_done(results_folder)
+    except Exception:
+        logger.exception('Skipping the results of training %s', training.training_number)
+        return
+    for part in [p for p in parts if p not in done]:
+        try:
+            await _upload_part(training, part, results[part], meta_content if part == CORE else None, data_exchanger)
+        except TrainingResultsUploadFailedError as e:
+            logger.warning('Skipping the %s results of training %s: %s', part, training.training_number, e)
+        except Exception:
+            logger.exception('Skipping the %s results of training %s', part, training.training_number)
+        done.append(part)
+        try:
+            _save_done(results_folder, done)
+        except OSError:
+            logger.exception('Could not record the %s results of training %s as done', part, training.training_number)
+
+
+def _parts_to_upload(results: Mapping[str, list[Path]], hyperparameters: Mapping[str, Any]) -> list[str]:
+    """The parts of `results` to upload, in upload order: `core` first, the optional ones only if asked for.
+
+    A part not in `PARTS`, or one whose hyperparameter is malformed, is logged and left out.
+    """
+    parts = []
+    for part in results:
+        if part not in PARTS:
+            logger.warning('Leaving out the training results part %r, which is none of %s', part, ', '.join(PARTS))
+            continue
+        if part in OPTIONAL_PARTS:
+            try:
+                if not bool_hyperparameter(hyperparameters, OPTIONAL_PARTS[part]):
+                    continue
+            except CriticalError as e:
+                logger.error('Leaving out the %s results: %s', part, e)
+                continue
+        parts.append(part)
+    return sorted(parts, key=lambda part: part != CORE)
+
+
+def _meta(training: Training, *, model_uuid: str, node_name: str, node_uuid: str,
+          stop_requested: bool) -> dict[str, Any]:
+    """Where the results come from, so they explain themselves without the loop's database."""
+    return {
+        'node_name': node_name,
+        'node_uuid': node_uuid,
+        'organization': training.context.organization,
+        'project': training.context.project,
+        'training_number': training.training_number,
+        'model_uuid': model_uuid,
+        'start_time': datetime.fromtimestamp(training.start_time, timezone.utc).isoformat(),
+        'uploaded_at': datetime.now(timezone.utc).isoformat(),
+        'stop_requested': stop_requested,
+        'learning_loop_node_version': _library_version(),
+        'hyperparameters': dict(training.hyperparameters),
+    }
+
+
+async def _upload_part(training: Training, part: str, files: list[Path], part_meta: dict[str, Any] | None,
+                       data_exchanger: DataExchanger) -> None:
+    """Zip and send one part; send it again while it gets no answer, up to `MAX_ATTEMPTS` times."""
+    training_folder = Path(training.training_folder)
+    zip_path = training_folder / RESULTS_FOLDER / f'{part}.zip'
+    await asyncio.to_thread(_write_zip, files, training_folder, zip_path, part_meta)
+    for attempt in range(1, MAX_ATTEMPTS + 1):
+        try:
+            await data_exchanger.upload_training_results(training.context, training.training_number, part,
+                                                         str(zip_path))
+            return
+        except (TrainingResultsNotAnsweredError, httpx.TransportError):
+            if attempt == MAX_ATTEMPTS:
+                raise
+            await asyncio.sleep(RETRY_DELAY * 2 ** (attempt - 1))
+
+
+def _write_zip(files: list[Path], training_folder: Path, zip_path: Path, meta: dict[str, Any] | None) -> None:
+    """Zip `files` into `zip_path`; directories go in as a whole, except anything in the folder of `zip_path`.
+
+    A path inside `training_folder` keeps its path relative to it, any other its path relative to the parent of
+    the file or directory named. Files in `COMPRESSED_SUFFIXES` are stored, everything else compressed.
+    A file that does not exist is logged and left out.
+    """
+    results_folder = zip_path.parent
+    results_folder.mkdir(parents=True, exist_ok=True)
+    partial = zip_path.with_name(f'{zip_path.name}.partial')
+    with zipfile.ZipFile(partial, 'w', compression=zipfile.ZIP_DEFLATED) as zipf:
+        for file in files:
+            if not file.exists():
+                logger.warning('training result %s does not exist, leaving it out', file)
+                continue
+            root = training_folder if file.is_relative_to(training_folder) else file.parent
+            for path in sorted(file.rglob('*')) if file.is_dir() else [file]:
+                if path.is_file() and not path.is_relative_to(results_folder):
+                    stored = path.suffix.lower() in COMPRESSED_SUFFIXES
+                    zipf.write(path, str(path.relative_to(root)),
+                               compress_type=zipfile.ZIP_STORED if stored else zipfile.ZIP_DEFLATED)
+        if meta is not None:
+            zipf.writestr(META_FILE, json.dumps(meta, indent=1))
+    os.replace(partial, zip_path)
+
+
+def _load_done(results_folder: Path) -> list[str]:
+    path = results_folder / DONE_FILE
+    return path.read_text().split() if path.exists() else []
+
+
+def _save_done(results_folder: Path, done: list[str]) -> None:
+    results_folder.mkdir(parents=True, exist_ok=True)
+    (results_folder / DONE_FILE).write_text('\n'.join(done))
+
+
+def _library_version() -> str | None:
+    try:
+        return version('learning_loop_node')
+    except PackageNotFoundError:
+        return None

learning_loop_node/trainer/validation_set.py

@@ -21,6 +21,7 @@ import logging
 from collections.abc import Mapping
 from typing import Any
 
+from ..data_classes import Hyperparameter
 from .exceptions import CriticalError
 
 logger = logging.getLogger(__name__)
@@ -28,14 +29,25 @@ logger = logging.getLogger(__name__)
 COMPUTE_VALIDATION_SET = 'compute_validation_set'
 """The hyperparameter that asks to split a validation set off the train images when the loop sent none."""
 
+COMPUTE_VALIDATION_SET_HYPERPARAMETER = Hyperparameter(
+    name=COMPUTE_VALIDATION_SET, type='bool', default=False,
+    notes='Split about a tenth of the train images off to select the models on, so the test set is only reported on; '
+          'a validation set sent by the loop is used either way')
+"""The declaration of `compute_validation_set`, for a trainer to list among its `provided_hyperparameters`."""
+
 VALIDATION_FRACTION = 0.1
 
 
 def compute_validation_set(hyperparameters: Mapping[str, Any]) -> bool:
     """Whether `compute_validation_set` is set, defaulting to false; anything but a boolean is a CriticalError."""
-    value = hyperparameters.get(COMPUTE_VALIDATION_SET, False)
+    return bool_hyperparameter(hyperparameters, COMPUTE_VALIDATION_SET)
+
+
+def bool_hyperparameter(hyperparameters: Mapping[str, Any], name: str) -> bool:
+    """Whether the hyperparameter `name` is set, defaulting to false; anything but a boolean is a CriticalError."""
+    value = hyperparameters.get(name, False)
     if not isinstance(value, bool):
-        raise CriticalError(f"'{COMPUTE_VALIDATION_SET}' must be true or false, got {value!r}")
+        raise CriticalError(f"'{name}' must be true or false, got {value!r}")
     return value
 
 

{learning_loop_node-0.27.2.dist-info → learning_loop_node-0.29.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.27.2
+Version: 0.29.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
@@ -129,6 +129,8 @@ The `batch_detect` endpoint receives a dictionary, with the same entries as the
 
 - `images`: List of image data dictionaries, each with the same structure as the `image` entry in the `detect` endpoint
 
+The entries `camera_id`, `tags`, `source` and `creation_date` apply to every image of the batch. An image data dictionary may also carry its own: its `tags` are added to those of the batch, while its `camera_id`, `source` and `creation_date` replace those of the batch. This way a batch can hold images of several cameras, each filtered and uploaded as its own camera's.
+
 Sending a camera's JPEG as it is (`format: 'jpeg'`) saves the bandwidth of a raw pixel array and spares the detector a re-encode for the upload.
 
 Example code can be found [in the rosys implementation](https://github.com/zauberzeug/rosys/blob/main/rosys/vision/detector_hardware.py).
@@ -256,6 +258,17 @@ Trainers fetch the images and anntoations from the Learning Loop to train new mo
 
 - if the command line tool "jpeginfo" is installed, the downloader will drop corrupted images automatically
 
+### Training results
+
+Together with its model, a trainer uploads the results of the training to the Learning Loop, e.g. the
+metrics per epoch, the evaluation curves and the confusion matrix. The trainer decides which files
+they are, grouped into the parts `core`, `examples` and `logs`; a trainer that names none uploads
+only its model. `core` is always uploaded, the examples and the raw training log only if the
+training's `upload_examples` and `upload_logs` hyperparameters are true (both default to false).
+The results are sent right after the model and never stop the training: results the Learning Loop
+does not store, e.g. beyond the 5 GiB it keeps per training, are logged and skipped, and a part
+that gets no answer or a server error is sent up to three times.
+
 ### Several trainers on one GPU
 
 Trainers that share a GPU (say, one per model architecture on the same machine) coordinate through

{learning_loop_node-0.27.2.dist-info → learning_loop_node-0.29.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 learning_loop_node/__init__.py,sha256=LzNoXBrcL6yJTyisuwJokZ9LTiYQwY3yRY3tmy83FlI,410
-learning_loop_node/data_exchanger.py,sha256=Rlql4WSDgF6uUivH77deoIl2vnTKL1rlEhQmAUmJ9IU,10642
+learning_loop_node/data_exchanger.py,sha256=Ap5m_Bd_fHKz82YQiLQphAu5mwJ8pc9VllcinqTRrLo,12099
 learning_loop_node/globals.py,sha256=a50Qg4z2205Zcw4Cs8z417kn1x8s78o2FJfZB9-ZZI4,211
 learning_loop_node/loop_communication.py,sha256=d1tJZ9T_y22FrEq1XzvidgH6d-byGxFq3ovLvPBS4FA,7611
 learning_loop_node/node.py,sha256=CABqe-sqUlcwi9CYVXBkrfxh0Xgtzj3AFLQcupSX-Ms,12158
@@ -8,18 +8,18 @@ learning_loop_node/rest.py,sha256=5X9IVW9kf1gNf8jifGW9g_gI_-9TEeoMMOW16jvwpRE,15
 learning_loop_node/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/annotation/annotator_logic.py,sha256=BTaopkJZkIf1CI5lfsVKsxbxoUIbDJrevavuQUT5e_c,1000
 learning_loop_node/annotation/annotator_node.py,sha256=J5xwSnM5rwTWrTe-TI37J0JHKf_4PlDuABaHvgjYr_Q,4443
-learning_loop_node/data_classes/__init__.py,sha256=_2dxfVkkI9kNM955Y7ZE8RLgfpegZpX5tXkkPP_n6Fo,1354
+learning_loop_node/data_classes/__init__.py,sha256=1mrGtrDCxILDwmPcbLgjqYp0cg7ObKusFR3BDoSMQIQ,1428
 learning_loop_node/data_classes/annotation_data.py,sha256=NfMlTv2_5AfVY_JDM4tbjETFjSN2S2I2LJJPMMcDT50,966
 learning_loop_node/data_classes/annotations.py,sha256=ha7uuWmZqW-LU2vYrkvfnaVpvf5C2xdm826IVVBu0w0,1848
 learning_loop_node/data_classes/detections.py,sha256=hGSKc1elk4Drp4XUjQ3F9HGeRpyLvA0q064b-S9lIug,5335
 learning_loop_node/data_classes/general.py,sha256=0p6RhPOEVr_vBoG-TySTUKhjT2_AS9Jb8jEqSkg5ncc,7829
-learning_loop_node/data_classes/image_metadata.py,sha256=QPLJ4gcVj33xaaCFF1n5e8G6higOJ6iIKy3YhGZMfkw,2459
+learning_loop_node/data_classes/image_metadata.py,sha256=wRkDRHAhuiBs8ntXeIn5_brHA7bpmL1efQpq2LVq6mk,3058
 learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
-learning_loop_node/data_classes/training.py,sha256=OlbPhm5kYHaZqgpz4_DfWnW4DQ_DK91AurReDhvWpRo,5944
+learning_loop_node/data_classes/training.py,sha256=My56tpDHAaUlraACTh-ZJ3y2V6v23IkD_bkkvil9_S8,6902
 learning_loop_node/detector/__init__.py,sha256=wroy0W1X5MX16C-peA-UHYrcdXet7GhleIlC_Pqa6hI,128
 learning_loop_node/detector/categories.py,sha256=qgquuop3geKXXsNBurO0eC2b64h_ojL37RBYMv0ZzOg,1147
 learning_loop_node/detector/detector_logic.py,sha256=agPbkFvFuQhZ04qQleWji6iYrbXsKqVGL0fseq4OXXY,1554
-learning_loop_node/detector/detector_node.py,sha256=AB-uXNPGxRbQwjLIN35uyk1Qww6T1bF4fqWSbwV6F1A,37395
+learning_loop_node/detector/detector_node.py,sha256=4vq2gRb6lVs2lwvhWL4rC3S9D89xWGkGouy506g2h38,38314
 learning_loop_node/detector/exceptions.py,sha256=C6KbNPlSbtfgDrZx2Hbhm7Suk9jVoR3fMRCO0CkrMsQ,196
 learning_loop_node/detector/geometry.py,sha256=bZCF9xuK7qy6Ipv5aQYrTlwSmjaiUz5VEUYnc1zCe-w,1101
 learning_loop_node/detector/outbox.py,sha256=mtMxdcComjRyM6qc-O1fBv9ABCau4-oCCV442hMaryQ,12351
@@ -52,20 +52,22 @@ learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 learning_loop_node/trainer/batch_size.py,sha256=LjA-zyY4hdnSMy6ZsdQOhWbmd5H_DI-9LG3yFInv04E,5476
 learning_loop_node/trainer/cuda.py,sha256=ehpfWqPBCVFu06SnGsjmWbSDRJ8sZc-L_nCtemRLUOU,12567
 learning_loop_node/trainer/downloader.py,sha256=AIyEKM4XYtd6VgmXrP3VayV9DpJzdURK1Brx81ePNSM,1470
-learning_loop_node/trainer/exceptions.py,sha256=uCqGzni5CO8R8pHnVjsucuM4S00UxY35qxJaWIaqqjk,1186
+learning_loop_node/trainer/exceptions.py,sha256=31SzhQXf3vsU-2aqoQYTjvsebcHtkJGZnKGzSvgLEvo,1899
 learning_loop_node/trainer/executor.py,sha256=XQ1TwgOF2pQv7bgzSaljDWG4I1ySjYXL8r9c_p9MX1Q,3947
+learning_loop_node/trainer/hyperparameters.py,sha256=WtVh3PThIZs0Oe6IGQF73-WfdtHpf6wL6tsc0r9iUAY,8623
 learning_loop_node/trainer/io_helpers.py,sha256=fmSh2-9sEfBavYxfa_zT4FfZMByT3KxDbA3O57I8iBY,7834
 learning_loop_node/trainer/metrics.py,sha256=H1jQh5fHN3-cTdMTuH8mF_uODV0T9DBdqcPUD-ZqFAI,1439
 learning_loop_node/trainer/subprocess.py,sha256=WxuTly0zhpClEjTd9wqoI_-7YTNMaiZpRHxu00vttKw,2647
 learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
-learning_loop_node/trainer/trainer_logic.py,sha256=x15bEy7uTfaGGHGYRGUl2olMeL0x0QW0vlu8nXZOaGg,9769
-learning_loop_node/trainer/trainer_logic_generic.py,sha256=3OcI16ijZ4q0QSKZEKah7uhoEydjxy5vKNDm-xBCuk8,31611
-learning_loop_node/trainer/trainer_node.py,sha256=GU4LyFA-JC9s4d-PbyKhKWyGMMIzTMqKJFNxM6tDJE4,5316
+learning_loop_node/trainer/trainer_logic.py,sha256=ayBo0B9_2ZmoLW97-ahbRUpqz0jTgV7-LbWul2NC248,9726
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=q9QM4mGtIvp3ihMH4yuFvnO0G9t71ojeOya5OYAeKgE,35175
+learning_loop_node/trainer/trainer_node.py,sha256=viYXQvUnwWfFjMKsh5Y-uWPCY2I2HwQh92w7bnBa0XY,5830
+learning_loop_node/trainer/training_results.py,sha256=WOgT0SmNQ-38p0ZUnhcBWqWYbvEE9YkfGEDcRARJ3zE,8792
 learning_loop_node/trainer/training_slot.py,sha256=Njq6Z0T2Lqlr24ftoIp11Pw2OxCo9Z2RGn1tuFhHqLY,6737
-learning_loop_node/trainer/validation_set.py,sha256=3gyqw88jadYRYDTUZVsaE0o5LGShmqaJykjr0sdNLJo,2907
+learning_loop_node/trainer/validation_set.py,sha256=dHEBXqZoDVlWFyV6K5EZ4fXbXa6OuZxSw6fVw0qoJh0,3597
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/rest/backdoor_controls.py,sha256=bCjrQvqtgd2DtyYNgftkcDe9AFHExFLtfdW_jISLwuk,5026
-learning_loop_node-0.27.2.dist-info/METADATA,sha256=KZ4fva5K1YHTwNrGoQKVz4ooUGgNLuISoetBOE1g2_M,20629
-learning_loop_node-0.27.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-learning_loop_node-0.27.2.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
-learning_loop_node-0.27.2.dist-info/RECORD,,
+learning_loop_node-0.29.0.dist-info/METADATA,sha256=0VUirHSUUAuBSs2aUlgoWgSiCIGsA7UjscKnsZViyK8,21772
+learning_loop_node-0.29.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+learning_loop_node-0.29.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
+learning_loop_node-0.29.0.dist-info/RECORD,,