learning-loop-node 0.23.0__py3-none-any.whl → 0.24.0__py3-none-any.whl

learning_loop_node/detector/outbox.py

@@ -37,7 +37,7 @@ class Outbox():
     It handles upload failures by splitting the upload into two smaller batches until the problematic image is identified - and removed.
     Any image can be saved to the normal or the priority queue.
     Images in the priority queue are uploaded first.
-    The total queue length is limited to 1000 images.
+    The normal queue is limited to MAX_OUTBOX_SIZE images (default 1000); the oldest are dropped first.
     """
 
     def __init__(self) -> None:
@@ -59,7 +59,9 @@ class Outbox():
         self.log.info('Outbox initialized with target_uri: %s', self.target_uri)
 
         self.BATCH_SIZE = 20
-        self.MAX_UPLOAD_LENGTH = 1000  # only affects the `upload_folders` list
+        self.MAX_UPLOAD_LENGTH = int(os.environ.get('MAX_OUTBOX_SIZE', '1000'))  # only limits `upload_folders`
+        if self.MAX_UPLOAD_LENGTH <= 0:
+            raise ValueError(f'MAX_OUTBOX_SIZE must be positive, got {self.MAX_UPLOAD_LENGTH}')
         self.UPLOAD_INTERVAL_S = 5
         self.UPLOAD_TIMEOUT_S = 30
 

learning_loop_node/helpers/entrypoint.py

@@ -12,21 +12,27 @@ from argparse import Action, Namespace
 import configargparse
 import uvicorn
 
+from ..trainer.batch_size import VRAM_LIMIT_GB_FLAG, VRAM_LIMIT_GB_HELP
+
 logger = logging.getLogger(__name__)
 
 
-def node_parser(*, description: str, legacy_env_prefix: str = '') -> configargparse.ArgumentParser:
+def node_parser(*, description: str, legacy_env_prefix: str = '',
+                vram_limit: bool = False) -> configargparse.ArgumentParser:
     """Build the parser for a node, pre-loaded with the settings every node has.
 
     :param legacy_env_prefix: A prefix an earlier version of this node required, e.g.
         ``'MY_DETECTOR_'``. Both spellings keep working, with a warning: the prefix on the
         current name (``MY_DETECTOR_NODE_HOST``) and the prefix on the flag it was originally
         applied to (``MY_DETECTOR_HOST``). Leave empty for a node that never used one.
+    :param vram_limit: Add :data:`~learning_loop_node.trainer.batch_size.VRAM_LIMIT_GB_FLAG`.
     """
     parser = _NodeArgumentParser(description=description, legacy_env_prefix=legacy_env_prefix)
     parser.add_argument('--host', default='0.0.0.0', env_var='NODE_HOST',
                         help='Host interface to bind to')
     parser.add_argument('--port', type=int, default=80, env_var='NODE_PORT', help='Port to bind to')
+    if vram_limit:
+        parser.add_argument(VRAM_LIMIT_GB_FLAG, type=float, default=0, help=VRAM_LIMIT_GB_HELP)
     return parser
 
 

learning_loop_node/trainer/batch_size.py

@@ -10,12 +10,29 @@ https://github.com/Lightning-AI/pytorch-lightning
 """
 
 import logging
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+from typing import Any
 
 from .exceptions import InsufficientMemoryError
 
 logger = logging.getLogger(__name__)
 
+REQUESTED_BATCH_SIZE = 'max_batch_size'
+"""The hyperparameter every node reads its `measure_batch_size` argument out of.
+
+0 or absent means the card decides. It is an input only: a trainer reports the size it settled on
+under a different key, conventionally ``batch_size``, and never writes it back here.
+"""
+
+VRAM_LIMIT_GB_FLAG = '--vram-limit-gb'
+"""The flag a trainer takes its GPU budget from; ``VRAM_LIMIT_GB`` follows from the name."""
+
+VRAM_LIMIT_GB_HELP = ('Gigabytes of GPU memory a training may use. The batch size is probed against this limit '
+                      'instead of the whole card, so a lower limit yields a smaller batch size rather than an '
+                      'out-of-memory error. Use it to share a GPU or to keep headroom against fragmentation. '
+                      "The limit is relative to the card's total memory, not to what is currently free. "
+                      '0 (default) means no limit.')
+
 MAX_BATCH_SIZE = 1024
 """Where a search stops when its caller sets no bound of its own."""
 
@@ -26,23 +43,51 @@ MIN_TRAIN_STEPS_PER_EPOCH = 8
 """Fewest optimizer steps an epoch must have; :func:`dataset_limit` is derived from it."""
 
 
-def find_batch_size(fits: Callable[[int], bool], *, limit: int) -> int:
-    """Return the largest power-of-two batch size that fits, never exceeding ``limit``.
+def find_batch_size(fits: Callable[[int], bool], *, limit: int, minimum: int = 1,
+                    candidate: int = 0) -> int:
+    """Return the largest batch size that fits, never exceeding ``limit``.
+
+    Powers of two, plus ``minimum`` and ``candidate``, the two sizes outside that set this will
+    return, and only because somebody named them and they then measured.
 
     :param fits: Runs a representative probe; ``False`` on out-of-memory.
-    :raises InsufficientMemoryError: If not even a batch size of 1 fits.
+    :param limit: Upper bound; the doubling stops at the largest power of two within it.
+    :param minimum: Smallest size to try, as named; the doubling continues from the next power of
+        two above it. Raise it above one for a step that cannot run on a single sample at all:
+        BatchNorm over a 1x1 feature map, a validation pass that halves the batch.
+    :param candidate: An exact size, tried once the doubling has reached its ceiling, so a size
+        that was asked for is used as asked for rather than rounded down. Ignored unless it lies
+        between that ceiling and ``limit``; a bound nobody named — one derived from the dataset,
+        say — must not be passed here.
+    :raises InsufficientMemoryError: If not even ``minimum`` fits.
     """
-    limit = smaller_pot(limit)
-    if not fits(1):
-        raise InsufficientMemoryError('batch size 1 does not fit in memory')
+    minimum = max(1, minimum)
+    bound = max(limit, minimum)
+    ceiling = max(smaller_pot(bound), minimum)
+
+    if not fits(minimum):
+        raise InsufficientMemoryError(f'batch size {minimum} does not fit in memory')
 
-    size = 1
-    while size < limit and fits(size * 2):
-        size *= 2
+    size = minimum
+    while size < ceiling and fits(smaller_pot(size) * 2):
+        size = smaller_pot(size) * 2
+
+    if size == ceiling and ceiling < candidate <= bound and fits(candidate):
+        size = candidate
 
     return size
 
 
+def requested_batch_size(hyperparameters: Mapping[str, Any]) -> int:
+    """The bound a training asked for, read out of the hyperparameters the loop sent.
+
+    An unfilled field arrives as absent, ``None`` or ``''``, and all three mean no bound.
+
+    :raises ValueError: If the value is there but is not a number.
+    """
+    return int(hyperparameters.get(REQUESTED_BATCH_SIZE, 0) or 0)
+
+
 def dataset_limit(sample_count: int) -> int:
     """The batch size ceiling that still leaves ``MIN_TRAIN_STEPS_PER_EPOCH`` steps per epoch."""
     if sample_count < 1:
@@ -50,9 +95,9 @@ def dataset_limit(sample_count: int) -> int:
     return max(1, sample_count // MIN_TRAIN_STEPS_PER_EPOCH)
 
 
-def no_gpu_batch_size(limit: int, probe: str) -> int:
-    """The batch size to fall back on when there is no GPU to probe."""
-    batch_size = min(smaller_pot(limit), NO_GPU_BATCH_SIZE)
+def no_gpu_batch_size(limit: int, probe: str, minimum: int = 1) -> int:
+    """The batch size to fall back on when there is no GPU to probe; never below ``minimum``."""
+    batch_size = max(min(smaller_pot(limit), NO_GPU_BATCH_SIZE), minimum)
     logger.warning('%s: CUDA is unavailable; using batch size %d without probing', probe, batch_size)
     return batch_size
 

learning_loop_node/trainer/cuda.py

@@ -11,47 +11,185 @@ from __future__ import annotations
 
 import gc
 import logging
+from abc import ABC, abstractmethod
+from argparse import ArgumentParser
 from collections.abc import Callable
 
 import torch
 
-from .batch_size import MAX_BATCH_SIZE, find_batch_size, is_out_of_memory, no_gpu_batch_size, smaller_pot
+from .batch_size import (
+    MAX_BATCH_SIZE,
+    REQUESTED_BATCH_SIZE,
+    VRAM_LIMIT_GB_FLAG,
+    VRAM_LIMIT_GB_HELP,
+    dataset_limit,
+    find_batch_size,
+    is_out_of_memory,
+    no_gpu_batch_size,
+)
 
 logger = logging.getLogger(__name__)
 
 SAFETY_MARGIN = 0.05
 """Share of the budget held back while probing, against allocator fragmentation later on."""
 
+DEFAULT_STEPS_PER_TRIAL = 4
+"""Training steps a batch size must survive before it counts as fitting.
+
+Some trainings reach their peak only after a few steps: an optimizer that allocates its state on
+the first one, gradients that accumulate, a scaler that skips the first updates.
+"""
+
+_budget_gb: float = 0
+"""What :func:`limit_cuda_memory` capped this process to; 0 means the whole card."""
+
+
+class ProbeStep(ABC):
+    """A training cycle built for one batch-size probe and released once the probe is done.
+
+    :func:`measure_batch_size` builds it through a factory, after the safety margin is reserved,
+    and holds the only reference to it, so nothing built for the probe stays on the card for the
+    training that follows. A trial at one batch size is ``steps_per_trial`` calls of
+    :meth:`train_step` followed by one of :meth:`val_step`; a subclass implements both, the other
+    two are optional.
+    """
+
+    @abstractmethod
+    def train_step(self, batch_size: int) -> str | None:
+        """Run one training step at ``batch_size``; may return a detail to append to the log line."""
+
+    @abstractmethod
+    def val_step(self, batch_size: int) -> str | None:
+        """Run the validation a training at ``batch_size`` runs between epochs.
+
+        Validate at the batch size the training validates with, which need not be ``batch_size``
+        (YOLOv5 halves it). A training that does not validate returns ``None`` without running
+        anything.
+        """
+
+    def on_out_of_memory(self) -> None:  # noqa: B027
+        """Drop what a trial that ran out of memory left behind (an optimizer's gradients, say)."""
+
+    def release(self) -> None:  # noqa: B027
+        """Undo what building the step changed outside it, such as moving the real model off the card."""
+
+
+def measure_batch_size(build_step: Callable[[], ProbeStep], *, max_batch_size: int = 0,
+                       sample_count: int | None = None, probe: str = 'batch-size probe',
+                       minimum: int = 1, steps_per_trial: int = DEFAULT_STEPS_PER_TRIAL) -> int:
+    """Settle a training's batch size against what it asked for and what the card allows.
+
+    Every training enters here, whatever shape its hyperparameters have; :func:`probe_batch_size` is
+    for a probe with no requested size to honour, such as a detection pass bounded only by how many
+    images there are. The step is built only once there is a card to probe.
+
+    :param build_step: Builds the step the trials run; called once, after the margin is reserved.
+    :param max_batch_size: What the training asked for, as carried in the
+        :data:`~.batch_size.REQUESTED_BATCH_SIZE` hyperparameter: the largest batch it may use,
+        measured rather than trusted. A size that fits is used as asked for, whether or not it is a
+        power of two; one that does not becomes the largest power of two below it that does. 0 means
+        the card decides alone. A request below ``minimum`` is raised to it, with a warning.
+    :param sample_count: Samples in the training split, when the caller knows it. The search is
+        then bounded by :func:`~.batch_size.dataset_limit`; that bound is never used as the exact
+        candidate.
+    :param probe: Names this probe in the log, so a node running several stays readable.
+    :param minimum: Smallest size to try, taking precedence over ``max_batch_size``; see
+        :func:`~.batch_size.find_batch_size`.
+    :param steps_per_trial: Training steps a size must survive, each trial then ending in one
+        validation step.
+    :raises InsufficientMemoryError: If not even ``minimum`` fits.
+    :raises ValueError: If the training asked for a negative batch size, or ``steps_per_trial`` is
+        below one.
+    """
+    if max_batch_size < 0:
+        raise ValueError(f'{REQUESTED_BATCH_SIZE} must be >= 0, got {max_batch_size}')
+    if steps_per_trial < 1:
+        raise ValueError(f'steps_per_trial must be >= 1, got {steps_per_trial}')
+    if 0 < max_batch_size < minimum:
+        logger.warning('%s: requested %s=%d is below the trainer minimum of %d; using %d', probe,
+                       REQUESTED_BATCH_SIZE, max_batch_size, minimum, minimum)
+
+    limit = max_batch_size
+    if sample_count is not None:
+        limit = min(limit or MAX_BATCH_SIZE, dataset_limit(sample_count))
+        logger.info('%s: %d training samples allow at most %d per batch', probe, sample_count,
+                    dataset_limit(sample_count))
+
+    return _probe(build_step, probe=probe, limit=limit, steps_per_trial=steps_per_trial,
+                  candidate=max_batch_size, minimum=minimum)
 
-def probe_batch_size(run_batch: Callable[[int], str | None], *, probe: str = 'batch-size probe',
-                     limit: int = 0, vram_limit_gb: float = 0) -> int:
-    """Find the largest power-of-two batch size ``run_batch`` fits into.
 
-    For a probe whose measurement is one call. A probe that has to build a throwaway model first
-    composes :func:`reserve_margin`, :func:`measured_fits` and ``find_batch_size`` itself.
+def probe_batch_size(run_batch: Callable[[int], str | None], *, probe: str = 'batch-size probe',
+                     limit: int = 0) -> int:
+    """Find the largest batch that fits for a pass with no requested size, such as a detection pass.
 
     :param run_batch: Runs the batch; may return a detail to append to the log line.
     :param probe: Names this probe in the log, so a node running several stays readable.
-    :param limit: Caps the search, rounded down to a power of two; 0 means
+    :param limit: Caps the search; 0 means
         :data:`~learning_loop_node.trainer.batch_size.MAX_BATCH_SIZE`.
-    :param vram_limit_gb: The budget the safety margin is a share of; 0 means the whole card.
-    :raises InsufficientMemoryError: If not even a batch size of 1 fits.
+    :raises InsufficientMemoryError: If not even a batch of one fits.
     """
-    limit = smaller_pot(limit or MAX_BATCH_SIZE)
+    return _probe(lambda: _PassStep(run_batch), probe=probe, limit=limit, steps_per_trial=1)
+
+
+class _PassStep(ProbeStep):
+    """A plain callable as a step; there is nothing of its own to build, validate or release."""
+
+    def __init__(self, run_batch: Callable[[int], str | None]) -> None:
+        self._run_batch = run_batch
+
+    def train_step(self, batch_size: int) -> str | None:
+        return self._run_batch(batch_size)
+
+    def val_step(self, batch_size: int) -> None:
+        return None
+
+
+def _probe(build_step: Callable[[], ProbeStep], *, probe: str, limit: int, steps_per_trial: int,
+           candidate: int = 0, minimum: int = 1) -> int:
+    """Everything of a probe but the step: the GPU check, the margin, the search, the release.
+
+    The margin is a share of the budget :func:`limit_cuda_memory` set in this process.
+    """
+    bound = max(limit or MAX_BATCH_SIZE, minimum)
 
     if not torch.cuda.is_available():
-        return no_gpu_batch_size(limit, probe)
+        return no_gpu_batch_size(bound, probe, minimum)
 
-    margin = reserve_margin(vram_limit_gb, probe=probe)
+    margin = reserve_margin(probe=probe)
     try:
-        chosen = find_batch_size(measured_fits(run_batch, probe=probe), limit=limit)
+        chosen = _search(build_step, probe=probe, bound=bound, steps_per_trial=steps_per_trial,
+                         minimum=minimum, candidate=candidate)
     finally:
         del margin
         free_cuda_memory()
-    logger.info('%s: selected batch size %d (upper bound %d)', probe, chosen, limit)
+    logger.info('%s: selected batch size %d (upper bound %d)', probe, chosen, bound)
     return chosen
 
 
+def _search(build_step: Callable[[], ProbeStep], *, probe: str, bound: int, steps_per_trial: int,
+            minimum: int, candidate: int) -> int:
+    """Build the step, search with it and release it; once this returns, nothing references it."""
+    step = build_step()
+    try:
+        fits = measured_fits(_trial(step, steps_per_trial), probe=probe, on_out_of_memory=step.on_out_of_memory)
+        return find_batch_size(fits, limit=bound, minimum=minimum, candidate=candidate)
+    finally:
+        step.release()
+
+
+def _trial(step: ProbeStep, steps_per_trial: int) -> Callable[[int], str | None]:
+    """One trial of ``step``: its training steps, then its validation."""
+    def run_trial(batch_size: int) -> str | None:
+        train_detail = None
+        for _ in range(steps_per_trial):
+            train_detail = step.train_step(batch_size)
+        val_detail = step.val_step(batch_size)
+        return '; '.join(detail for detail in (train_detail, val_detail) if detail) or None
+
+    return run_trial
+
+
 def measured_fits(run_batch: Callable[[int], str | None], *, probe: str,
                   on_out_of_memory: Callable[[], None] | None = None) -> Callable[[int], bool]:
     """Wrap ``run_batch`` into the ``fits`` predicate ``find_batch_size`` searches with.
@@ -84,38 +222,41 @@ def measured_fits(run_batch: Callable[[int], str | None], *, probe: str,
     return fits
 
 
-def reserve_margin(vram_limit_gb: float, *, probe: str) -> torch.Tensor:
+def reserve_margin(*, probe: str) -> torch.Tensor:
     """Claim :data:`SAFETY_MARGIN` of the budget, so a trial competes against a smaller card.
 
     Keep the returned tensor alive for as long as the probe runs: releasing it hands the margin
     back, and the chosen size is no longer the size that was measured.
-
-    :param vram_limit_gb: The budget the margin is a share of; 0 means the whole card.
     """
-    margin_bytes = int(usable_memory_bytes(vram_limit_gb) * SAFETY_MARGIN)
+    margin_bytes = int(usable_memory_bytes() * SAFETY_MARGIN)
     logger.info('%s: keeping %.0f MB free as a safety margin', probe, margin_bytes / 1024**2)
     return torch.empty(margin_bytes, dtype=torch.uint8, device='cuda')
 
 
-def usable_memory_bytes(vram_limit_gb: float) -> int:
-    """How much GPU memory this process may allocate, honouring :func:`limit_cuda_memory`.
+def usable_memory_bytes() -> int:
+    """How much GPU memory this process may allocate, honouring :func:`limit_cuda_memory`."""
+    if _budget_gb > 0:
+        return int(_budget_gb * 1024**3)
+    return torch.cuda.get_device_properties(None).total_memory
+
 
-    :param vram_limit_gb: 0 or less means the whole card.
+def add_vram_limit_argument(parser: ArgumentParser) -> None:
+    """Give a spawned training script the same GPU budget flag its node has.
+
+    The spawned process still has to call :func:`limit_cuda_memory` with it.
     """
-    total_bytes = torch.cuda.get_device_properties(None).total_memory
-    if vram_limit_gb <= 0:
-        return total_bytes
-    return min(total_bytes, int(vram_limit_gb * 1024**3))
+    parser.add_argument(VRAM_LIMIT_GB_FLAG, type=float, default=0, help=VRAM_LIMIT_GB_HELP)
 
 
 def limit_cuda_memory(vram_limit_gb: float) -> None:
     """Cap how much of the GPU this process may allocate, to ``vram_limit_gb`` gigabytes.
 
     Call this once per process that touches the GPU, a spawned training process included: the
-    cap does not survive the spawn.
+    cap does not survive the spawn. A probe in this process then measures against the same budget.
 
     :param vram_limit_gb: 0 or less means no cap.
     """
+    global _budget_gb  # pylint: disable=global-statement
     if vram_limit_gb <= 0 or not torch.cuda.is_available():
         return
 
@@ -129,6 +270,7 @@ def limit_cuda_memory(vram_limit_gb: float) -> None:
         return
 
     torch.cuda.set_per_process_memory_fraction(fraction, None)
+    _budget_gb = vram_limit_gb
     logger.info('Limiting VRAM usage to %.1f GB of %.1f GB (%.0f%%)', vram_limit_gb, total_gb, fraction * 100)
 
 

{learning_loop_node-0.23.0.dist-info → learning_loop_node-0.24.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.23.0
+Version: 0.24.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
@@ -60,8 +60,10 @@ You can configure connection to our Learning Loop by specifying the following en
 | LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector                  | -            |
 | MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           | 0.3          |
 | MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           | 0.6          |
+| MAX_OUTBOX_SIZE          | -            | max. number of non-priority images queued for upload         | Detector (opt.)           | 1000         |
 | EXCLUSIVE_MODEL_BUILD    | -            | Reject detections during update to save VRAM (set to 1)      | Detector (opt.)           | 0            |
 | INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            | 10           |
+| VRAM_LIMIT_GB            | -            | GPU memory (GB) a training may use; the batch size is probed against it (`--vram-limit-gb`, trainers built with `node_parser(vram_limit=True)`) | Trainer (opt.) | 0 (whole card) |
 | RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            | 0            |
 | KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            | 0            |
 | TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            | 0 (disabled) |
@@ -73,6 +75,16 @@ Where a name has an alias, either spelling works. If both are set to **different
 prefixed name wins and a warning names the value used — the variable is never treated as unset,
 which would otherwise let `LOOP_HOST` fall back to its default of `learning-loop.ai`.
 
+#### Debug logging
+
+Every node type (Detector, Trainer, Annotator, Converter) has a REST endpoint to switch its own log level between `INFO` and `DEBUG` at runtime. Send the plain text `on` or `off` with a PUT request:
+
+- Turn debug logging on: `curl -X PUT -d "on" http://localhost/debug_logging`
+- Turn debug logging off: `curl -X PUT -d "off" http://localhost/debug_logging`
+
+The response repeats the new state. Any other body returns status code 400.
+Only the node's own logger is affected; third-party loggers such as `httpx` keep their level. The setting is not persistent, a restart brings the log level back to `INFO`.
+
 #### Testing
 
 We use github actions for CI. Tests can also be executed locally by running

{learning_loop_node-0.23.0.dist-info → learning_loop_node-0.24.0.dist-info}/RECORD

@@ -22,7 +22,7 @@ learning_loop_node/detector/detector_logic.py,sha256=XJaHNiw5PaHoCWnJ-qsxdUELLev
 learning_loop_node/detector/detector_node.py,sha256=GC5CaGcW_qVxyolDEz9U5W7y0ejZGEu9FZ__ckLxd7g,35728
 learning_loop_node/detector/exceptions.py,sha256=C6KbNPlSbtfgDrZx2Hbhm7Suk9jVoR3fMRCO0CkrMsQ,196
 learning_loop_node/detector/geometry.py,sha256=bZCF9xuK7qy6Ipv5aQYrTlwSmjaiUz5VEUYnc1zCe-w,1101
-learning_loop_node/detector/outbox.py,sha256=6F0MjFM-zNX-xWP_YGbmU0UYuyb0odsX_OcAxMPSzS0,12288
+learning_loop_node/detector/outbox.py,sha256=HT0GHv6rsQzdbCD8DAT23m1-Q1XEvA_KgwAJLONyX_0,12506
 learning_loop_node/detector/postprocess.py,sha256=fPBbgE6Obkej709zoFPGqA2gnbJNbpbMuZogilKy4pI,7274
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=slPGm87TbBCfDUdpRTAM5yOz1An8Br5X_2HFJOGSrFg,3540
@@ -42,15 +42,15 @@ learning_loop_node/enums/general.py,sha256=OSpcIpdFjuRByON2xBoiIXl1SgDREIqkwnaxk
 learning_loop_node/enums/trainer.py,sha256=VaD63guLO4aKgVfXT0EryPlXKQGegSET3Cp4R0uC_Ew,746
 learning_loop_node/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/helpers/background_tasks.py,sha256=sbbS8JO1ls0ZCbNW68KOT9QLOFyFoJRXwwAq78iOmoA,3256
-learning_loop_node/helpers/entrypoint.py,sha256=EtRMXZKzNapwbIylf4hg8Obw-2EydrLGKUjbIn5H-94,3591
+learning_loop_node/helpers/entrypoint.py,sha256=U4vHFqgke6mXjDL-E4MND8RO7P1k2VBGwdDOOoJvD9Y,3915
 learning_loop_node/helpers/environment_reader.py,sha256=zsoFysxNxBCOJzUwX4Pbm3ftYhxx5zsbOHQfD9VcfvA,1937
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
 learning_loop_node/helpers/log_conf.py,sha256=hqVAa_9NnYEU6N0dcOKmph82p7MpgKqeF_eomTLYzWY,961
 learning_loop_node/helpers/misc.py,sha256=qdN18Ukbw6eR6sFL4BxNq0EZzg2tVm2Cm8cTbFyy8QU,8931
 learning_loop_node/helpers/run.py,sha256=_uox-j3_K_bL3yCAwy3JYSOiIxrnhzVxyxWpCe8_J9U,876
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/trainer/batch_size.py,sha256=Ma_nH3PHAk2gzqdkNXstNFqwXUUTt7pVe4aqWRrkHX4,2914
-learning_loop_node/trainer/cuda.py,sha256=LkXJaJ404S64XGRTMqqULB9e6g85dJfjLIAn12lnpTY,5968
+learning_loop_node/trainer/batch_size.py,sha256=LjA-zyY4hdnSMy6ZsdQOhWbmd5H_DI-9LG3yFInv04E,5476
+learning_loop_node/trainer/cuda.py,sha256=ehpfWqPBCVFu06SnGsjmWbSDRJ8sZc-L_nCtemRLUOU,12567
 learning_loop_node/trainer/downloader.py,sha256=AIyEKM4XYtd6VgmXrP3VayV9DpJzdURK1Brx81ePNSM,1470
 learning_loop_node/trainer/exceptions.py,sha256=IRT7qvn7-RjXaqL8mF21ySm7JWhQtqm1Cl0Gi3DChVY,540
 learning_loop_node/trainer/executor.py,sha256=XQ1TwgOF2pQv7bgzSaljDWG4I1ySjYXL8r9c_p9MX1Q,3947
@@ -63,7 +63,7 @@ learning_loop_node/trainer/trainer_logic_generic.py,sha256=IRP68ZGpnOX4OS03OwB7H
 learning_loop_node/trainer/trainer_node.py,sha256=ZnydfHJ2ehAhb821Xke_09qthCE3vDf7iqeYUrliSSo,4938
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/rest/backdoor_controls.py,sha256=ZnK8ypY5r_q0-YZbtaOxhQThzuZvMsQHM5gJGESd_dE,5131
-learning_loop_node-0.23.0.dist-info/METADATA,sha256=L8ASHsKUg7Z_JGji4tqDJEbsCf44tZoKiPsZ4YeU6oo,17607
-learning_loop_node-0.23.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-learning_loop_node-0.23.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
-learning_loop_node-0.23.0.dist-info/RECORD,,
+learning_loop_node-0.24.0.dist-info/METADATA,sha256=rQNtas26mLdkkjI37mJ_jteRe7RtUO_3DpReanEUC2U,18609
+learning_loop_node-0.24.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+learning_loop_node-0.24.0.dist-info/top_level.txt,sha256=nS5qiFc4stVF_VXVCVJxZnrqVOv3IGX5-_E3sLby6OE,19
+learning_loop_node-0.24.0.dist-info/RECORD,,