learning-loop-node 0.27.1__py3-none-any.whl → 0.28.0__py3-none-any.whl

learning_loop_node/data_classes/__init__.py

@@ -30,6 +30,7 @@ from .image_metadata import ImageMetadata, ImagesMetadata
 from .socket_response import SocketResponse
 from .training import (
     Errors,
+    Hyperparameter,
     PretrainedModel,
     Training,
     TrainingError,
@@ -45,6 +46,6 @@ __all__ = [
     'AnnotationNodeStatus', 'Category', 'Context', 'DetectorStatus', 'ErrorConfiguration',
     'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse', 'ImagesMetadata',
     'SocketResponse',
-    'Errors', 'PretrainedModel', 'Training',
+    'Errors', 'Hyperparameter', 'PretrainedModel', 'Training',
     'TrainingError', 'TrainingOut', 'TrainingStateData', 'TrainingStatus',
 ]

learning_loop_node/data_classes/general.py

@@ -12,6 +12,8 @@ from ..enums import CategoryType
 
 KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
 
+DEFAULT_CONF_THRESHOLD = 0.2
+
 
 @dataclass(**KWONLY_SLOTS)
 class Category():
@@ -59,8 +61,9 @@ class ModelInformation():
         default=None, metadata={"description": "The path of the parent directory of the model in the file system."})
     model_size: Optional[str] = field(default=None, metadata={
                                       "description": "The size of the model (i.e. the specification or variant of the model architecture)."})
-    conf_threshold: Optional[float] = field(default=None, metadata={
-        "description": "The minimum confidence the trainer chose for the detections of this model, if it chose one."})
+    conf_threshold: float = field(default=DEFAULT_CONF_THRESHOLD, metadata={
+        "description": "The minimum confidence for the detections of this model: Loaded from the model if available. "
+                       "A detector node's own setting overrides it."})
 
     @property
     def context(self):
@@ -80,6 +83,8 @@ class ModelInformation():
             except Exception as exc:
                 raise Exception(f"could not read model information from file '{model_info_file_path}'") from exc
             try:
+                # a null is a value the writer did not determine, so it reads as if the key were absent
+                content = {key: value for key, value in content.items() if value is not None}
                 model_information = from_dict(data_class=ModelInformation, data=content)
                 model_information.model_root_path = model_root_path
             except Exception as exc:

learning_loop_node/data_classes/training.py

@@ -3,7 +3,7 @@ import sys
 import time
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
 from uuid import uuid4
 
 from ..enums import TrainerState
@@ -22,6 +22,28 @@ class PretrainedModel():
     description: str
 
 
+HyperparameterType = Literal['int', 'float', 'bool', 'str', 'pairs']
+
+
+@dataclass(**KWONLY_SLOTS)
+class Hyperparameter:
+    """One hyperparameter a trainer accepts, reported to the loop so it can offer it in the project settings.
+
+    Plain data only, so it reaches the loop as it is. ``minimum`` and ``maximum`` apply to ``int`` and ``float``,
+    ``choices`` to ``str``. A ``pairs`` value is text of the form ``"key:value,..."``, which ``parse_pairs`` reads.
+    ``notes`` help whoever sets the value. ``expert`` ones are offered behind an expansion.
+    Whether a value can be changed when a training is started is the project's choice, not the trainer's.
+    """
+    name: str
+    type: HyperparameterType
+    default: Any
+    minimum: float | None = None
+    maximum: float | None = None
+    choices: list[str] | None = None
+    notes: str = ''
+    expert: bool = False
+
+
 @dataclass(**KWONLY_SLOTS)
 class TrainingStatus():
     id: str  # NOTE this must not be changed, but tests wont detect a change -> update tests!
@@ -33,6 +55,7 @@ class TrainingStatus():
     progress: Optional[float]
 
     pretrained_models: List[PretrainedModel] = field(default_factory=list)
+    hyperparameters: list[Hyperparameter] = field(default_factory=list)
     architecture: Optional[str] = None
     context: Optional[Context] = None
 

learning_loop_node/detector/detector_logic.py

@@ -10,6 +10,7 @@ class DetectorLogicFactory(Protocol):
 
     The factory controls how the detector is constructed — implementations
     can build synchronously or offload heavy work to a thread pool.
+    `model_info.conf_threshold` is the confidence the detector filters at.
     """
     @property
     def model_format(self) -> str: ...

learning_loop_node/detector/detector_node.py

@@ -7,7 +7,7 @@ import shutil
 import subprocess
 import sys
 from collections.abc import Sequence
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, replace
 from datetime import datetime
 from typing import Dict, List, Literal, Optional, Union
 
@@ -50,9 +50,11 @@ from .rest import upload as rest_upload
 class DetectorNode(Node):
 
     def __init__(self, name: str, detector_factory: DetectorLogicFactory,
-                 uuid: Optional[str] = None, use_backdoor_controls: bool = False) -> None:
+                 uuid: Optional[str] = None, use_backdoor_controls: bool = False,
+                 conf_threshold: Optional[float] = None) -> None:
         super().__init__(name, uuid=uuid, node_type='detector', needs_login=False, needs_sio=False)
         self._detector_factory = detector_factory
+        self._conf_threshold = conf_threshold
         self.node_version: Optional[str] = os.environ.get('NODE_VERSION')
         self._detector: _DetectorState = _Initializing()
         self._exclusive_model_build: bool = os.environ.get('EXCLUSIVE_MODEL_BUILD', '0').lower() in ('1', 'true')
@@ -547,6 +549,10 @@ class DetectorNode(Node):
         if model_info is None:
             logging.warning('No model.json found in %s', model_dir)
             raise Exception('model.json not found')
+        if self._conf_threshold is not None:
+            model_info = replace(model_info, conf_threshold=self._conf_threshold)
+        self.log.info('Set confidence threshold for model %s to %s%s', model_info.version,
+                      model_info.conf_threshold, ' (set on the node)' if self._conf_threshold is not None else '')
 
         if self._exclusive_model_build and isinstance(self._detector, _ActiveDetector):
             async with self.detection_lock:

learning_loop_node/helpers/entrypoint.py

@@ -12,13 +12,14 @@ from argparse import Action, Namespace
 import configargparse
 import uvicorn
 
+from ..data_classes.general import DEFAULT_CONF_THRESHOLD
 from ..trainer.batch_size import VRAM_LIMIT_GB_FLAG, VRAM_LIMIT_GB_HELP
 
 logger = logging.getLogger(__name__)
 
 
 def node_parser(*, description: str, legacy_env_prefix: str = '',
-                vram_limit: bool = False) -> configargparse.ArgumentParser:
+                vram_limit: bool = False, conf_threshold: bool = False) -> configargparse.ArgumentParser:
     """Build the parser for a node, pre-loaded with the settings every node has.
 
     :param legacy_env_prefix: A prefix an earlier version of this node required, e.g.
@@ -26,6 +27,9 @@ def node_parser(*, description: str, legacy_env_prefix: str = '',
         current name (``MY_DETECTOR_NODE_HOST``) and the prefix on the flag it was originally
         applied to (``MY_DETECTOR_HOST``). Leave empty for a node that never used one.
     :param vram_limit: Add :data:`~learning_loop_node.trainer.batch_size.VRAM_LIMIT_GB_FLAG`.
+    :param conf_threshold: Add ``--conf-threshold`` for the ``conf_threshold`` of a
+        :class:`~learning_loop_node.detector.detector_node.DetectorNode`. It has no default:
+        one would override the threshold the trainer chose for the model.
     """
     parser = _NodeArgumentParser(description=description, legacy_env_prefix=legacy_env_prefix)
     parser.add_argument('--host', default='0.0.0.0', env_var='NODE_HOST',
@@ -33,6 +37,10 @@ def node_parser(*, description: str, legacy_env_prefix: str = '',
     parser.add_argument('--port', type=int, default=80, env_var='NODE_PORT', help='Port to bind to')
     if vram_limit:
         parser.add_argument(VRAM_LIMIT_GB_FLAG, type=float, default=0, help=VRAM_LIMIT_GB_HELP)
+    if conf_threshold:
+        parser.add_argument('--conf-threshold', type=float, default=None,
+                            help='Minimum confidence for detections (default: the one the trainer chose for the '
+                                 f'model, else {DEFAULT_CONF_THRESHOLD})')
     return parser
 
 

learning_loop_node/trainer/exceptions.py

@@ -15,6 +15,13 @@ class TrainingRefusedError(Exception):
     """Raised when a trainer cannot take on a training it was asked to begin; nothing was set up."""
 
 
+class HyperparameterMismatchError(Exception):
+    """Raised when a training's hyperparameters do not fit the trainer's declaration; nothing was set up.
+
+    Unlike a refusal, the training would fail on any trainer of the same declaration.
+    """
+
+
 class TrainingSlotTakenError(TrainingRefusedError):
     """Raised when a sibling node holds the training slot."""
 

learning_loop_node/trainer/hyperparameters.py

@@ -0,0 +1,180 @@
+"""Resolving the hyperparameters a training was sent against the ones its trainer declares.
+
+The loop sends only what the project sets, plus what the job overrides. Everything else comes from
+the trainer's declared defaults, so after :func:`resolve` a training holds the full set it runs
+with, and that is what gets reported back to the loop.
+"""
+
+import contextlib
+import logging
+import math
+from collections.abc import Mapping, Sequence
+from typing import Any, get_args
+
+from ..data_classes import Hyperparameter
+from ..data_classes.training import HyperparameterType
+from .batch_size import REQUESTED_BATCH_SIZE
+from .exceptions import CriticalError
+from .validation_set import COMPUTE_VALIDATION_SET
+
+logger = logging.getLogger(__name__)
+
+LIBRARY_NAMES = (REQUESTED_BATCH_SIZE, COMPUTE_VALIDATION_SET)
+"""Hyperparameters the library reads itself, so a trainer that does not declare them still uses them."""
+
+BOOL_STRINGS = {'true': True, '1': True, 'yes': True, 'on': True,
+                'false': False, '0': False, 'no': False, 'off': False}
+
+
+def resolve(declared: Sequence[Hyperparameter], received: Mapping[str, Any]) -> dict[str, Any]:
+    """Coerce and check ``received`` against ``declared`` and fill in the declared defaults.
+
+    ``declared`` is trusted to have passed :func:`check_declaration`. Names the trainer does not declare are kept
+    unchanged: old project configurations still carry them, and a resolved training carries the values its trainer
+    reported. An unset hyperparameter with a default of ``None`` is left out, so the trainer's own fallback applies.
+    Resolving twice gives the same result.
+
+    :raises CriticalError: If a value cannot be coerced to its type, is out of range or not one of its choices.
+    """
+    resolved = dict(received)
+    for hyperparameter in declared:
+        value = received.get(hyperparameter.name)
+        if _is_unset(hyperparameter, value):
+            if hyperparameter.default is None:
+                resolved.pop(hyperparameter.name, None)
+            else:
+                resolved[hyperparameter.name] = _coerce(hyperparameter, hyperparameter.default)
+        else:
+            resolved[hyperparameter.name] = _check(hyperparameter, _coerce(hyperparameter, value))
+    return resolved
+
+
+def check_declaration(declared: Sequence[Hyperparameter]) -> None:
+    """Check that ``declared`` is consistent, so that :func:`resolve` can rely on it.
+
+    A trainer's declaration does not change while it runs, so the node checks it once, when it is built. A default of
+    ``None`` stands for "the trainer decides" and is not checked.
+
+    :raises ValueError: If a name is declared twice, a hyperparameter contradicts itself or its default does not fit.
+    """
+    names = [h.name for h in declared]
+    if duplicates := sorted({name for name in names if names.count(name) > 1}):
+        raise ValueError(f'the trainer declares these hyperparameters twice: {", ".join(duplicates)}')
+    for hyperparameter in declared:
+        _check_one_declaration(hyperparameter)
+
+
+def parse_pairs(text: str) -> list[tuple[str, str]]:
+    """The ``key:value`` items of a ``pairs`` hyperparameter such as ``"uuid:uuid,uuid:uuid"``.
+
+    Empty items and the spaces around keys and values are skipped, so ``''`` holds no pairs.
+
+    :raises ValueError: If an item is not a key and a value separated by a single colon.
+    """
+    pairs = []
+    for item in (i.strip() for i in text.split(',')):
+        if not item:
+            continue
+        key, _, value = (part.strip() for part in item.partition(':'))
+        if not key or not value or ':' in value:
+            raise ValueError(f'"{item}" is not of the form "key:value"')
+        pairs.append((key, value))
+    return pairs
+
+
+def warn_about_undeclared(declared: Sequence[Hyperparameter], received: Mapping[str, Any]) -> None:
+    """Log the names in ``received`` that neither the trainer nor the library knows; they are not rejected."""
+    unknown = sorted(set(received) - {h.name for h in declared} - set(LIBRARY_NAMES))
+    if unknown:
+        logger.warning('these hyperparameters are not declared by this trainer: %s', ', '.join(unknown))
+
+
+def _check_one_declaration(hyperparameter: Hyperparameter) -> None:
+    def contradiction(reason: str) -> ValueError:
+        return ValueError(f"the trainer declares hyperparameter '{hyperparameter.name}' {reason}")
+
+    if hyperparameter.type not in get_args(HyperparameterType):
+        raise contradiction(f'with the unknown type {hyperparameter.type!r}')
+    if hyperparameter.type not in ('int', 'float') and (hyperparameter.minimum is not None or
+                                                        hyperparameter.maximum is not None):
+        raise contradiction(f'as {hyperparameter.type} with a minimum or maximum')
+    if hyperparameter.type != 'str' and hyperparameter.choices:
+        raise contradiction(f'as {hyperparameter.type} with choices')
+    for bound in (hyperparameter.minimum, hyperparameter.maximum):
+        if bound is not None and (isinstance(bound, bool) or not isinstance(bound, (int, float))):
+            raise contradiction(f'with the bound {bound!r}, which is not a number')
+    if (hyperparameter.minimum is not None and hyperparameter.maximum is not None and
+            hyperparameter.minimum > hyperparameter.maximum):
+        raise contradiction('with a minimum above its maximum')
+    if hyperparameter.default is not None:
+        try:
+            _check(hyperparameter, _coerce(hyperparameter, hyperparameter.default))
+        except CriticalError as e:
+            raise contradiction(f'with a default that does not fit: {e}') from None
+
+
+def _is_unset(hyperparameter: Hyperparameter, value: Any) -> bool:
+    """``None``, or the empty value the loop sends for a field left blank, unless it fits as a value of its own."""
+    if value is None:
+        return True
+    if value != '':
+        return False
+    try:
+        _check(hyperparameter, _coerce(hyperparameter, value))
+    except CriticalError:
+        return True
+    return False
+
+
+def _invalid(hyperparameter: Hyperparameter, requirement: str, value: Any) -> CriticalError:
+    return CriticalError(f"hyperparameter '{hyperparameter.name}' {requirement}, got {value!r}")
+
+
+def _coerce(hyperparameter: Hyperparameter, value: Any) -> Any:
+    if hyperparameter.type == 'pairs':
+        if not isinstance(value, str):
+            raise _invalid(hyperparameter, 'must be text of key:value pairs', value)
+        try:
+            return ','.join(f'{key}:{item}' for key, item in parse_pairs(value))
+        except ValueError as e:
+            raise CriticalError(f"hyperparameter '{hyperparameter.name}': {e}") from None
+    if hyperparameter.type == 'str':
+        if isinstance(value, bool) or not isinstance(value, (str, int, float)):
+            raise _invalid(hyperparameter, 'must be text', value)
+        return str(value)
+    if hyperparameter.type == 'bool':
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str) and (word := value.strip().lower()) in BOOL_STRINGS:
+            return BOOL_STRINGS[word]
+        with contextlib.suppress(TypeError, ValueError):
+            number = float(value)
+            if math.isfinite(number):
+                return number != 0
+        raise _invalid(hyperparameter, 'must be true or false', value)
+    if isinstance(value, bool):
+        raise _invalid(hyperparameter, 'must be a number', value)
+    if hyperparameter.type == 'int' and isinstance(value, (int, str)):
+        with contextlib.suppress(ValueError):
+            return int(value)
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        raise _invalid(hyperparameter, 'must be a number', value) from None
+    if not math.isfinite(number):
+        raise _invalid(hyperparameter, 'must be finite', value)
+    if hyperparameter.type == 'float':
+        return number
+    if not number.is_integer():
+        raise _invalid(hyperparameter, 'must be a whole number', value)
+    return int(number)
+
+
+def _check(hyperparameter: Hyperparameter, value: Any) -> Any:
+    if hyperparameter.minimum is not None and value < hyperparameter.minimum:
+        raise _invalid(hyperparameter, f'must be at least {hyperparameter.minimum}', value)
+    if hyperparameter.maximum is not None and value > hyperparameter.maximum:
+        raise _invalid(hyperparameter, f'must be at most {hyperparameter.maximum}', value)
+    if hyperparameter.choices and value not in hyperparameter.choices:
+        raise _invalid(hyperparameter, f"must be one of {', '.join(hyperparameter.choices)}", value)
+    return value

learning_loop_node/trainer/trainer_logic_generic.py

@@ -13,6 +13,7 @@ from fastapi.encoders import jsonable_encoder
 from ..data_classes import (
     Context,
     Errors,
+    Hyperparameter,
     PretrainedModel,
     Training,
     TrainingOut,
@@ -26,7 +27,8 @@ from ..helpers.misc import (
     is_valid_uuid4,
 )
 from .downloader import TrainingsDownloader
-from .exceptions import CriticalError, NodeNeedsRestartError, TrainingRefusedError
+from .exceptions import CriticalError, HyperparameterMismatchError, NodeNeedsRestartError, TrainingRefusedError
+from .hyperparameters import resolve, warn_about_undeclared
 from .io_helpers import ActiveTrainingIO, EnvironmentVars, LastTrainingIO
 from .training_slot import TrainingSlot, training_slot_from_env
 from .validation_set import assign_validation_set
@@ -186,6 +188,12 @@ class TrainerLogicGeneric(ABC):
         """
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def provided_hyperparameters(self) -> list[Hyperparameter]:
+        """Returns the hyperparameters this trainer accepts; an empty list leaves them unchecked."""
+        raise NotImplementedError
+
     # ---------------------------------------- METHODS ----------------------------------------
 
     # NOTE: Trainings are started by the Learning Loop via the begin_training event
@@ -204,6 +212,7 @@ class TrainerLogicGeneric(ABC):
                                 progress=self.general_progress)
 
         status.pretrained_models = self.provided_pretrained_models
+        status.hyperparameters = self.provided_hyperparameters
         status.architecture = self.model_architecture
 
         if self._training:
@@ -224,9 +233,18 @@ class TrainerLogicGeneric(ABC):
 
     def _init_from_last_training(self) -> None:
         """Initializes a new training object from the last training saved on disc via last_training_io.
+
+        The hyperparameters are resolved again, since the training may have been saved before this trainer declared
+        them. If one no longer fits, the training continues with the values it was saved with, since it already runs
+        with them.
         """
         self._training = self.last_training_io.load()
         assert self._training is not None and self._training.training_folder is not None, 'could not restore training folder'
+        try:
+            self._training.hyperparameters = resolve(self.provided_hyperparameters, self._training.hyperparameters)
+        except CriticalError as e:
+            logger.warning('restored training does not fit the declared hyperparameters, continuing with the saved '
+                           'values: %s', e)
         logger.info('restored training: \n%s', self._training)
         self._active_training_io = ActiveTrainingIO(
             self._training.training_folder, self.node.loop_communicator, self._training.context)
@@ -234,13 +252,15 @@ class TrainerLogicGeneric(ABC):
     async def begin_training(self, organization: str, project: str, training_config: Dict) -> None:
         """Called on `begin_training` event from the Learning Loop.
 
-        The training slot is taken before anything is set up, so a refused training leaves the
-        previous training folders in place.
+        The hyperparameters are resolved and the training slot is taken before anything is set up, so a refused
+        training leaves the previous training folders in place.
 
         :raise TrainingRefusedError: if a training is active or a sibling node holds the training slot.
+        :raise HyperparameterMismatchError: if a hyperparameter does not fit its declaration.
         """
         if self.training_active:
             raise TrainingRefusedError(f'training {self.training.id} is still active')
+        training_config = self._with_resolved_hyperparameters(training_config)
         context = Context(organization=organization, project=project)
         self.training_slot.claim(self._slot_claim(context, training_config.get('training_number')))
         try:
@@ -250,6 +270,24 @@ class TrainerLogicGeneric(ABC):
             raise
         self._begin_training_task()
 
+    def _with_resolved_hyperparameters(self, training_config: dict) -> dict:
+        """Returns ``training_config`` with its hyperparameters resolved against the declared ones.
+
+        A config without a hyperparameters dict is returned as it is, for ``Training.generate_training`` to reject.
+
+        :raise HyperparameterMismatchError: if a hyperparameter does not fit its declaration.
+        """
+        declared = self.provided_hyperparameters
+        received = training_config.get('hyperparameters')
+        if not declared or not isinstance(received, dict):
+            return training_config
+        try:
+            resolved = resolve(declared, received)
+        except CriticalError as e:
+            raise HyperparameterMismatchError(str(e)) from e
+        warn_about_undeclared(declared, received)
+        return {**training_config, 'hyperparameters': resolved}
+
     def _begin_training_task(self) -> None:
         # NOTE: Task object is used to potentially cancel the task
         self.training_task = asyncio.get_event_loop().create_task(self._run())
@@ -498,10 +536,11 @@ class TrainerLogicGeneric(ABC):
             raise CriticalError('The model is uploaded, but the loop never answered with its id') from e
 
     def _dump_model_json(self) -> str:
-        """Dumps the categories and the trainer's model metadata to a json file and returns the path to the file.
-        """
-        content = {'categories': [asdict(c) for c in self._training.categories],
-                   **self._get_model_metadata()} if self._training else None
+        """Dump the categories and the trainer's model metadata to a json file and return its path."""
+        content = None
+        if self._training:
+            content = {'categories': [asdict(c) for c in self._training.categories],
+                       **{key: value for key, value in self._get_model_metadata().items() if value is not None}}
         json_path = '/tmp/model.json'
         with open(json_path, 'w') as f:
             json.dump(content, f)
@@ -620,9 +659,11 @@ class TrainerLogicGeneric(ABC):
         raise NotImplementedError
 
     def _get_model_metadata(self) -> dict[str, Any]:
-        """Called when the files of `_get_latest_model_files` are uploaded.
-        Returns values the trainer determined for this model, merged into its model.json.
+        """Return the values the trainer determined for this model, to be merged into its model.json.
+
+        Called when the files of `_get_latest_model_files` are uploaded.
         The keys must be fields of `ModelInformation` (e.g. `conf_threshold`) so detectors can read them.
+        A value of `None` is left out of the file.
         """
         return {}
 

learning_loop_node/trainer/trainer_node.py

@@ -10,7 +10,8 @@ from socketio import AsyncClient, exceptions
 from ..data_classes import SocketResponse
 from ..enums import TrainerState
 from ..node import Node
-from .exceptions import TrainingRefusedError
+from .exceptions import HyperparameterMismatchError, TrainingRefusedError
+from .hyperparameters import check_declaration
 from .io_helpers import LastTrainingIO
 from .rest import backdoor_controls
 from .trainer_logic_generic import TrainerLogicGeneric
@@ -19,6 +20,8 @@ from .trainer_logic_generic import TrainerLogicGeneric
 class TrainerNode(Node):
 
     def __init__(self, name: str, trainer_logic: TrainerLogicGeneric, uuid: Optional[str] = None, use_backdoor_controls: bool = False):
+        """:raise ValueError: if the trainer declares hyperparameters that contradict themselves."""
+        check_declaration(trainer_logic.provided_hyperparameters)
         super().__init__(name, uuid=uuid, node_type='trainer')
         trainer_logic._node = self
         self.trainer_logic = trainer_logic
@@ -72,6 +75,10 @@ class TrainerNode(Node):
             except TrainingRefusedError as e:
                 self.log.warning('refused begin_training: %s', e)
                 return SocketResponse(success=False, error_msg=str(e), payload='refused')
+            except HyperparameterMismatchError as e:
+                # NOTE: not a refusal, which the loop answers by offering the same job again
+                self.log.warning('rejected begin_training: %s', e)
+                return SocketResponse.for_failure(str(e))
             return True
 
         @sio_client.event

learning_loop_node/trainer/validation_set.py

@@ -21,6 +21,7 @@ import logging
 from collections.abc import Mapping
 from typing import Any
 
+from ..data_classes import Hyperparameter
 from .exceptions import CriticalError
 
 logger = logging.getLogger(__name__)
@@ -28,6 +29,12 @@ logger = logging.getLogger(__name__)
 COMPUTE_VALIDATION_SET = 'compute_validation_set'
 """The hyperparameter that asks to split a validation set off the train images when the loop sent none."""
 
+COMPUTE_VALIDATION_SET_HYPERPARAMETER = Hyperparameter(
+    name=COMPUTE_VALIDATION_SET, type='bool', default=False,
+    notes='Split about a tenth of the train images off to select the models on, so the test set is only reported on; '
+          'a validation set sent by the loop is used either way')
+"""The declaration of `compute_validation_set`, for a trainer to list among its `provided_hyperparameters`."""
+
 VALIDATION_FRACTION = 0.1
 
 

{learning_loop_node-0.27.1.dist-info → learning_loop_node-0.28.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.27.1
+Version: 0.28.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
@@ -61,6 +61,7 @@ You can configure connection to our Learning Loop by specifying the following en
 | MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           | 0.3          |
 | MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           | 0.6          |
 | MAX_OUTBOX_SIZE          | -            | max. number of non-priority images queued for upload         | Detector (opt.)           | 1000         |
+| CONF_THRESHOLD           | -            | minimum confidence of a detection, overriding the model's (`--conf-threshold`, detectors built with `node_parser(conf_threshold=True)`) | Detector (opt.) | model's, 0.2 |
 | EXCLUSIVE_MODEL_BUILD    | -            | Reject detections during update to save VRAM (set to 1)      | Detector (opt.)           | 0            |
 | INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            | 10           |
 | VRAM_LIMIT_GB            | -            | GPU memory (GB) a training may use; the batch size is probed against it (`--vram-limit-gb`, trainers built with `node_parser(vram_limit=True)`) | Trainer (opt.) | 0 (whole card) |
@@ -349,4 +350,4 @@ After a short time the converted model should be available as well.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
   - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings
-  - `conf_threshold` (optional): the minimum confidence the trainer chose for this model's detections
+  - `conf_threshold` (optional): the minimum confidence the trainer chose for this model's detections; a detector filters at it unless its `CONF_THRESHOLD` is set, and at 0.2 for a model without one

{learning_loop_node-0.27.1.dist-info → learning_loop_node-0.28.0.dist-info}/RECORD

@@ -8,18 +8,18 @@ learning_loop_node/rest.py,sha256=5X9IVW9kf1gNf8jifGW9g_gI_-9TEeoMMOW16jvwpRE,15
 learning_loop_node/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/annotation/annotator_logic.py,sha256=BTaopkJZkIf1CI5lfsVKsxbxoUIbDJrevavuQUT5e_c,1000
 learning_loop_node/annotation/annotator_node.py,sha256=J5xwSnM5rwTWrTe-TI37J0JHKf_4PlDuABaHvgjYr_Q,4443
-learning_loop_node/data_classes/__init__.py,sha256=_2dxfVkkI9kNM955Y7ZE8RLgfpegZpX5tXkkPP_n6Fo,1354
+learning_loop_node/data_classes/__init__.py,sha256=f9txmNcmpxCfTmgLIwZS0tr1QcJHI45s8Rxgbxk_U7o,1392
 learning_loop_node/data_classes/annotation_data.py,sha256=NfMlTv2_5AfVY_JDM4tbjETFjSN2S2I2LJJPMMcDT50,966
 learning_loop_node/data_classes/annotations.py,sha256=ha7uuWmZqW-LU2vYrkvfnaVpvf5C2xdm826IVVBu0w0,1848
 learning_loop_node/data_classes/detections.py,sha256=hGSKc1elk4Drp4XUjQ3F9HGeRpyLvA0q064b-S9lIug,5335
-learning_loop_node/data_classes/general.py,sha256=LTRtA_sPUKbXkNrka01_o8qwRN6r56tQJCSHPNdg_MI,7522
+learning_loop_node/data_classes/general.py,sha256=0p6RhPOEVr_vBoG-TySTUKhjT2_AS9Jb8jEqSkg5ncc,7829
 learning_loop_node/data_classes/image_metadata.py,sha256=QPLJ4gcVj33xaaCFF1n5e8G6higOJ6iIKy3YhGZMfkw,2459
 learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
-learning_loop_node/data_classes/training.py,sha256=OlbPhm5kYHaZqgpz4_DfWnW4DQ_DK91AurReDhvWpRo,5944
+learning_loop_node/data_classes/training.py,sha256=My56tpDHAaUlraACTh-ZJ3y2V6v23IkD_bkkvil9_S8,6902
 learning_loop_node/detector/__init__.py,sha256=wroy0W1X5MX16C-peA-UHYrcdXet7GhleIlC_Pqa6hI,128
 learning_loop_node/detector/categories.py,sha256=qgquuop3geKXXsNBurO0eC2b64h_ojL37RBYMv0ZzOg,1147
-learning_loop_node/detector/detector_logic.py,sha256=XJaHNiw5PaHoCWnJ-qsxdUELLevvztYx54TQ8KcHl18,1479
-learning_loop_node/detector/detector_node.py,sha256=3e6UU1fl6IVkEgcfJTiOE-W30yebXcd5IpPOBYWV2mM,36950
+learning_loop_node/detector/detector_logic.py,sha256=agPbkFvFuQhZ04qQleWji6iYrbXsKqVGL0fseq4OXXY,1554
+learning_loop_node/detector/detector_node.py,sha256=AB-uXNPGxRbQwjLIN35uyk1Qww6T1bF4fqWSbwV6F1A,37395
 learning_loop_node/detector/exceptions.py,sha256=C6KbNPlSbtfgDrZx2Hbhm7Suk9jVoR3fMRCO0CkrMsQ,196
 learning_loop_node/detector/geometry.py,sha256=bZCF9xuK7qy6Ipv5aQYrTlwSmjaiUz5VEUYnc1zCe-w,1101
 learning_loop_node/detector/outbox.py,sha256=mtMxdcComjRyM6qc-O1fBv9ABCau4-oCCV442hMaryQ,12351
@@ -42,7 +42,7 @@ learning_loop_node/enums/general.py,sha256=OSpcIpdFjuRByON2xBoiIXl1SgDREIqkwnaxk
 learning_loop_node/enums/trainer.py,sha256=VdamD-k6Q8kMm_cwPA8iaEUfZvmTxLfsBHQpEi43Ahk,849
 learning_loop_node/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/helpers/background_tasks.py,sha256=sbbS8JO1ls0ZCbNW68KOT9QLOFyFoJRXwwAq78iOmoA,3256
-learning_loop_node/helpers/entrypoint.py,sha256=U4vHFqgke6mXjDL-E4MND8RO7P1k2VBGwdDOOoJvD9Y,3915
+learning_loop_node/helpers/entrypoint.py,sha256=ZPjbH-mM6quvnZ8-L8GxQaINX8D5qlQ-K77URSFcmc0,4540
 learning_loop_node/helpers/environment_reader.py,sha256=zsoFysxNxBCOJzUwX4Pbm3ftYhxx5zsbOHQfD9VcfvA,1937
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
 learning_loop_node/helpers/log_conf.py,sha256=hqVAa_9NnYEU6N0dcOKmph82p7MpgKqeF_eomTLYzWY,961
@@ -52,20 +52,21 @@ learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 learning_loop_node/trainer/batch_size.py,sha256=LjA-zyY4hdnSMy6ZsdQOhWbmd5H_DI-9LG3yFInv04E,5476
 learning_loop_node/trainer/cuda.py,sha256=ehpfWqPBCVFu06SnGsjmWbSDRJ8sZc-L_nCtemRLUOU,12567
 learning_loop_node/trainer/downloader.py,sha256=AIyEKM4XYtd6VgmXrP3VayV9DpJzdURK1Brx81ePNSM,1470
-learning_loop_node/trainer/exceptions.py,sha256=uCqGzni5CO8R8pHnVjsucuM4S00UxY35qxJaWIaqqjk,1186
+learning_loop_node/trainer/exceptions.py,sha256=7PeJ1wXPvf7NYBrEkPNvZvTTGTBXZ9zIGE5tE-tWPu8,1435
 learning_loop_node/trainer/executor.py,sha256=XQ1TwgOF2pQv7bgzSaljDWG4I1ySjYXL8r9c_p9MX1Q,3947
+learning_loop_node/trainer/hyperparameters.py,sha256=WtVh3PThIZs0Oe6IGQF73-WfdtHpf6wL6tsc0r9iUAY,8623
 learning_loop_node/trainer/io_helpers.py,sha256=fmSh2-9sEfBavYxfa_zT4FfZMByT3KxDbA3O57I8iBY,7834
 learning_loop_node/trainer/metrics.py,sha256=H1jQh5fHN3-cTdMTuH8mF_uODV0T9DBdqcPUD-ZqFAI,1439
 learning_loop_node/trainer/subprocess.py,sha256=WxuTly0zhpClEjTd9wqoI_-7YTNMaiZpRHxu00vttKw,2647
 learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
 learning_loop_node/trainer/trainer_logic.py,sha256=x15bEy7uTfaGGHGYRGUl2olMeL0x0QW0vlu8nXZOaGg,9769
-learning_loop_node/trainer/trainer_logic_generic.py,sha256=mDTcGXXHqoXkqJNsYU8yNixBxIq9piBqHDvE9KlTC8U,31483
-learning_loop_node/trainer/trainer_node.py,sha256=GU4LyFA-JC9s4d-PbyKhKWyGMMIzTMqKJFNxM6tDJE4,5316
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=_vHtOfgZInGcEwUHZFTcZhiAp7rui4DwTd8GxkH1Wf8,33682
+learning_loop_node/trainer/trainer_node.py,sha256=viYXQvUnwWfFjMKsh5Y-uWPCY2I2HwQh92w7bnBa0XY,5830
 learning_loop_node/trainer/training_slot.py,sha256=Njq6Z0T2Lqlr24ftoIp11Pw2OxCo9Z2RGn1tuFhHqLY,6737
-learning_loop_node/trainer/validation_set.py,sha256=3gyqw88jadYRYDTUZVsaE0o5LGShmqaJykjr0sdNLJo,2907
+learning_loop_node/trainer/validation_set.py,sha256=j0KAUP8CN4NpudL180xtT1xUQV0bCfxw1tMlPdb9l4Y,3363
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/rest/backdoor_controls.py,sha256=bCjrQvqtgd2DtyYNgftkcDe9AFHExFLtfdW_jISLwuk,5026
-learning_loop_node-0.27.1.dist-info/METADATA,sha256=bv0ofwadIpn6gVRH8K6zFuPh64IokrH-Cxi-UvcePwE,20317
-learning_loop_node-0.27.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-learning_loop_node-0.27.1.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
-learning_loop_node-0.27.1.dist-info/RECORD,,
+learning_loop_node-0.28.0.dist-info/METADATA,sha256=9T_Gxp-5PA9CCPsmbBPWy5jBjmDElOaUbDhY1Qfeors,20629
+learning_loop_node-0.28.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+learning_loop_node-0.28.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
+learning_loop_node-0.28.0.dist-info/RECORD,,