learning-loop-node 0.26.0__tar.gz → 0.27.0__tar.gz

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.26.0
+Version: 0.27.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
@@ -349,3 +349,4 @@ After a short time the converted model should be available as well.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
   - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings
+  - `conf_threshold` (optional): the minimum confidence the trainer chose for this model's detections

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/README.md

@@ -316,3 +316,4 @@ After a short time the converted model should be available as well.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
   - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings
+  - `conf_threshold` (optional): the minimum confidence the trainer chose for this model's detections

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/learning_loop_node/data_classes/general.py

@@ -59,6 +59,8 @@ class ModelInformation():
         default=None, metadata={"description": "The path of the parent directory of the model in the file system."})
     model_size: Optional[str] = field(default=None, metadata={
                                       "description": "The size of the model (i.e. the specification or variant of the model architecture)."})
+    conf_threshold: Optional[float] = field(default=None, metadata={
+        "description": "The minimum confidence the trainer chose for the detections of this model, if it chose one."})
 
     @property
     def context(self):

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/learning_loop_node/data_classes/training.py

@@ -103,13 +103,19 @@ class Training():
         assert self.image_data is not None, 'Image data not set'
         return [image['id'] for image in self.image_data]
 
-    def train_image_count(self) -> int:
-        assert self.image_data is not None, 'Image data not set'
-        return len([image for image in self.image_data if image['set'] == 'train'])
+    def train_images(self) -> list[dict]:
+        return self._images_of_set('train')
+
+    def val_images(self) -> list[dict]:
+        """The images to select models on: the `val` images if there are any, else the test images."""
+        return self._images_of_set('val') or self.test_images()
+
+    def test_images(self) -> list[dict]:
+        return self._images_of_set('test')
 
-    def test_image_count(self) -> int:
+    def _images_of_set(self, set_name: str) -> list[dict]:
         assert self.image_data is not None, 'Image data not set'
-        return len([image for image in self.image_data if image['set'] == 'test'])
+        return [image for image in self.image_data if image['set'] == set_name]
 
 
 @dataclass(**KWONLY_SLOTS)

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/learning_loop_node/trainer/rest/backdoor_controls.py

@@ -38,8 +38,7 @@ async def reset(request: Request):
     trainer_node: 'TrainerNode' = request.app
     async with trainer_node.repeat_loop_lock:
 
-        await trainer_node.trainer_logic.stop()  # NOTE first stop may only kill running training process
-        await trainer_node.trainer_logic.stop()
+        await trainer_node.trainer_logic.abort()
         trainer_node.last_training_io.delete()
         trainer_node.status.reset_all_errors()
 

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/learning_loop_node/trainer/trainer_logic.py

@@ -25,6 +25,8 @@ class TrainerLogic(TrainerLogicGeneric):
         super().__init__(model_format)
         self._detection_progress: Optional[float] = None
         self._executor: Optional[Executor] = None
+        self._process_started: bool = False
+        self._stop_requested: bool = False
         self.start_training_task: Optional[Coroutine] = None
         self.inference_batch_size = self._environment_vars.inference_batch_size
 
@@ -47,10 +49,13 @@ class TrainerLogic(TrainerLogicGeneric):
         previous_state = TrainerState.TrainModelDownloaded
         error_key = 'run_training'
         self._executor = Executor(self.training.training_folder)
+        self._process_started = False
+        self._stop_requested = False
         self.training.training_state = TrainerState.TrainingRunning
 
         try:
             await self._start_training()
+            self._process_started = True
             last_sync_time = datetime.now()
 
             while True:
@@ -77,11 +82,17 @@ class TrainerLogic(TrainerLogicGeneric):
             # if self.executor.return_code != 0:
             #     raise TrainingError(cause=f'Executor returned with error code: {self.executor.return_code}')
 
-        except TrainingError:
+        except TrainingError as e:
             logging.exception('Exception in trainer_logic._train')
+            if self._stop_requested:
+                # ending the process may be what failed it; a stopped training must not start over, and the
+                # wind-down uploads the model it has (or ends with a CriticalError if there is none)
+                logging.warning('Training process failed after a stop (%s), winding down with its model', e.cause)
+                return
             self.training.training_state = previous_state
             raise
         finally:
+            self._process_started = False
             # shield so the subprocess is stopped even when the task is cancelled (e.g. on abort())
             if self._executor and self._executor.is_running():
                 await asyncio.shield(self.executor.stop_and_wait())
@@ -147,15 +158,14 @@ class TrainerLogic(TrainerLogicGeneric):
 
     # ---------------------------------------- OVERWRITTEN METHODS ----------------------------------------
 
-    async def stop(self) -> None:
-        """If executor is running, stop it. Else cancel training task."""
-        print('===============> stop received in trainer_logic.', flush=True)
-        if not self.training_active:
+    async def _stop_running_training(self) -> None:
+        """End the training process; before it has started, abort."""
+        if not self._process_started:
+            await super()._stop_running_training()
             return
+        self._stop_requested = True
         if self._executor and self._executor.is_running():
             await self.executor.stop_and_wait()
-            return
-        await super().stop()
 
     # ---------------------------------------- ABSTRACT METHODS ----------------------------------------
 

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/learning_loop_node/trainer/trainer_logic_generic.py

@@ -29,6 +29,7 @@ from .downloader import TrainingsDownloader
 from .exceptions import CriticalError, NodeNeedsRestartError, TrainingRefusedError
 from .io_helpers import ActiveTrainingIO, EnvironmentVars, LastTrainingIO
 from .training_slot import TrainingSlot, training_slot_from_env
+from .validation_set import assign_validation_set
 
 if TYPE_CHECKING:
     from .trainer_node import TrainerNode
@@ -328,7 +329,7 @@ class TrainerLogicGeneric(ABC):
     async def _perform_state(self, error_key: str, state_during: TrainerState, state_after: TrainerState, action: Callable[[], Coroutine], reset_early=False):
         '''
         Perform a training state and handle errors.
-        - If the loop sends a StopTraining event, this will raise a CancelledError.
+        - If the training is aborted, the CancelledError moves it to ReadyForCleanup; on shutdown it is re-raised.
         - States can raise a CriticalError indicating that there is no point in retrying the state.
         - If any other error occurs, the error is stored in the errors object and the state is reset to the previous state.
           The state is retried after a wait until the training is aborted.
@@ -377,11 +378,11 @@ class TrainerLogicGeneric(ABC):
         self.last_training_io.save(self.training)
 
     async def _prepare(self) -> None:
-        """Downloads images to the images_folder and saves annotations to training.data.image_data.
-        """
+        """Downloads images to the images_folder and saves annotations to training.data.image_data."""
         self.node.data_exchanger.set_context(self.training.context)
         downloader = TrainingsDownloader(self.node.data_exchanger)
         image_data, skipped_image_count = await downloader.download_training_data(self.training.images_folder)
+        assign_validation_set(image_data, self.training.hyperparameters)
 
         self.training.image_data = image_data
         self.training.skipped_image_count = skipped_image_count
@@ -421,11 +422,12 @@ class TrainerLogicGeneric(ABC):
         try:
             new_best_model = self._get_new_best_training_state()
             if new_best_model:
+                sets = [image['set'] for image in self.training.image_data or []]
                 new_training = TrainingOut(trainer_id=self.node.uuid,
                                            trainer_name=self.node.name,
                                            confusion_matrix=new_best_model.confusion_matrix,
-                                           train_image_count=self.training.train_image_count(),
-                                           test_image_count=self.training.test_image_count(),
+                                           train_image_count=sets.count('train') + sets.count('val'),
+                                           test_image_count=sets.count('test'),
                                            hyperparameters=self.hyperparameters_for_state_sync,
                                            best_epoch=new_best_model.epoch)
                 await asyncio.sleep(0.1)  # NOTE needed for tests.
@@ -475,7 +477,7 @@ class TrainerLogicGeneric(ABC):
         already_uploaded_formats = self.active_training_io.load_model_upload_progress()
 
         for file_format in [f for f in files if f not in already_uploaded_formats]:
-            _files = files[file_format] + [self._dump_categories_to_json()]
+            _files = files[file_format] + [self._dump_model_json()]
             assert len([f for f in _files if 'model.json' in f]) == 1, "model.json must be included exactly once"
 
             model_uuid = await self.node.data_exchanger.upload_model_get_uuid(context, _files, self.training.training_number, file_format)
@@ -490,10 +492,11 @@ class TrainerLogicGeneric(ABC):
         except FileNotFoundError as e:
             raise CriticalError('The model is uploaded, but the loop never answered with its id') from e
 
-    def _dump_categories_to_json(self) -> str:
-        """Dumps the categories to a json file and returns the path to the file.
+    def _dump_model_json(self) -> str:
+        """Dumps the categories and the trainer's model metadata to a json file and returns the path to the file.
         """
-        content = {'categories': [asdict(c) for c in self._training.categories], } if self._training else None
+        content = {'categories': [asdict(c) for c in self._training.categories],
+                   **self._get_model_metadata()} if self._training else None
         json_path = '/tmp/model.json'
         with open(json_path, 'w') as f:
             json.dump(content, f)
@@ -515,12 +518,17 @@ class TrainerLogicGeneric(ABC):
 
     async def on_shutdown(self) -> None:
         self.shutdown_event.set()
-        await self.stop()
-        await self.stop()
+        await self.abort()
 
-    async def stop(self):
-        """Stop the training. Default is to abort.
-        """
+    async def stop(self) -> None:
+        """Stop a running training but keep its model. In every other state a stop does nothing."""
+        if not self.training_active or self.training.training_state != TrainerState.TrainingRunning:
+            logger.info('Received stop in state %s. This is ignored.', self.training.training_state if self.training_active else None)
+            return
+        await self._stop_running_training()
+
+    async def _stop_running_training(self) -> None:
+        """End the running training so it winds down with its model. This base class cannot, so it aborts."""
         await self.abort()
 
     async def abort(self):
@@ -606,6 +614,13 @@ class TrainerLogicGeneric(ABC):
         """
         raise NotImplementedError
 
+    def _get_model_metadata(self) -> dict[str, Any]:
+        """Called when the files of `_get_latest_model_files` are uploaded.
+        Returns values the trainer determined for this model, merged into its model.json.
+        The keys must be fields of `ModelInformation` (e.g. `conf_threshold`) so detectors can read them.
+        """
+        return {}
+
     @abstractmethod
     async def _clear_training_data(self, training_folder: str) -> None:
         """Called after a training has finished. Deletes all data that is not needed anymore after a training run. 

learning_loop_node-0.27.0/learning_loop_node/trainer/validation_set.py

@@ -0,0 +1,63 @@
+"""Which images a training selects its models on.
+
+A node selects models (best checkpoint, early stopping, confidence threshold) on
+`Training.val_images()` and reads `Training.test_images()` only for the counts it reports. It never
+decides between the two itself: `val_images()` always holds a set to select on, which is the test
+images unless the training has `val` images of its own. A node may notice that both hold the same
+images and spare itself a second pass, but nothing more.
+
+Where the `val` images come from, in this order:
+
+1. The loop sent a dedicated validation set (`set == 'val'`); it is used as it is.
+2. The `compute_validation_set` hyperparameter is true: `assign_validation_set` moves about a tenth of the
+   train images to `val`, chosen by a hash of the image id, so an image stays on its side when new
+   images arrive or a training is continued.
+3. Otherwise there are no `val` images and `val_images()` falls back to the test images, so the
+   model is selected on the test set.
+"""
+
+import hashlib
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from .exceptions import CriticalError
+
+logger = logging.getLogger(__name__)
+
+COMPUTE_VALIDATION_SET = 'compute_validation_set'
+"""The hyperparameter that asks to split a validation set off the train images when the loop sent none."""
+
+VALIDATION_FRACTION = 0.1
+
+
+def compute_validation_set(hyperparameters: Mapping[str, Any]) -> bool:
+    """Whether `compute_validation_set` is set, defaulting to false; anything but a boolean is a CriticalError."""
+    value = hyperparameters.get(COMPUTE_VALIDATION_SET, False)
+    if not isinstance(value, bool):
+        raise CriticalError(f"'{COMPUTE_VALIDATION_SET}' must be true or false, got {value!r}")
+    return value
+
+
+def assign_validation_set(image_data: list[dict], hyperparameters: Mapping[str, Any]) -> None:
+    """Move part of the train images to `val` if `compute_validation_set` asks for it and the loop sent no `val` images.
+    """
+    if any(image['set'] == 'val' for image in image_data):
+        logger.info('Selecting models on the validation set of the loop')
+        return
+    if not compute_validation_set(hyperparameters):
+        return
+
+    train_images = [image for image in image_data if image['set'] == 'train']
+    if len(train_images) < 2:
+        raise CriticalError(f'{len(train_images)} training image(s) are too few to split off a validation set, '
+                            'add more training images')
+    val_images = [image for image in train_images if _position(image) < VALIDATION_FRACTION] \
+        or [min(train_images, key=_position)]
+    for image in val_images:
+        image['set'] = 'val'
+    logger.info('Split %d of %d training images off as validation set', len(val_images), len(train_images))
+
+
+def _position(image: dict) -> float:
+    return int(hashlib.sha256(image['id'].encode()).hexdigest()[:8], 16) / 16**8

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/learning_loop_node.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.26.0
+Version: 0.27.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
@@ -349,3 +349,4 @@ After a short time the converted model should be available as well.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
   - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings
+  - `conf_threshold` (optional): the minimum confidence the trainer chose for this model's detections

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/learning_loop_node.egg-info/SOURCES.txt

@@ -69,5 +69,6 @@ learning_loop_node/trainer/trainer_logic.py
 learning_loop_node/trainer/trainer_logic_generic.py
 learning_loop_node/trainer/trainer_node.py
 learning_loop_node/trainer/training_slot.py
+learning_loop_node/trainer/validation_set.py
 learning_loop_node/trainer/rest/__init__.py
 learning_loop_node/trainer/rest/backdoor_controls.py

{learning_loop_node-0.26.0 → learning_loop_node-0.27.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "learning_loop_node"
-version = "0.26.0"
+version = "0.27.0"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = [{ name = "Zauberzeug GmbH", email = "info@zauberzeug.com" }]
 license = { text = "MIT" }