learning-loop-node 0.10.14__py3-none-any.whl → 0.10.15rc1__py3-none-any.whl

learning_loop_node/loop_communication.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+import time
 from typing import Awaitable, Callable, List, Optional
 
 import httpx
@@ -33,7 +34,6 @@ class LoopCommunicator():
                 base_url=self.base_url, timeout=Timeout(60.0), verify=self.ssl_cert_path)
         else:
             self.async_client = httpx.AsyncClient(base_url=self.base_url, timeout=Timeout(60.0))
-        self.async_client.cookies.clear()
 
         logging.info(f'Loop interface initialized with base_url: {self.base_url} / user: {self.username}')
 
@@ -68,16 +68,19 @@ class LoopCommunicator():
         if self.async_client is not None and not self.async_client.is_closed:
             await self.async_client.aclose()
 
-    async def backend_ready(self) -> bool:
+    async def backend_ready(self, timeout: Optional[int] = None) -> bool:
         """Wait until the backend is ready"""
+        start_time = time.time()
         while True:
             try:
                 logging.info('Checking if backend is ready')
                 response = await self.get('/status', requires_login=False)
                 if response.status_code == 200:
                     return True
-            except Exception as e:
-                logging.info(f'backend not ready: {e}')
+            except Exception:
+                logging.info('backend not ready yet.')
+            if timeout is not None and time.time() + 10 - start_time > timeout:
+                raise TimeoutError('Backend not ready within timeout')
             await asyncio.sleep(10)
 
     async def retry_on_401(self, func: Callable[..., Awaitable[httpx.Response]], *args, **kwargs) -> httpx.Response:

learning_loop_node/node.py

@@ -126,6 +126,8 @@ class Node(FastAPI):
                     await self.on_repeat()
             except asyncio.CancelledError:
                 return
+            except TimeoutError:
+                self.log.debug('Backend not ready within timeout, skipping repeat loop')
             except Exception:
                 self.log.exception('error in repeat loop')
 
@@ -140,6 +142,7 @@ class Node(FastAPI):
     async def reconnect_to_loop(self):
         """Initialize the loop communicator, log in if needed and reconnect to the loop via socket.io."""
         self.init_loop_communicator()
+        await self.loop_communicator.backend_ready(timeout=5)
         if self.needs_login:
             await self.loop_communicator.ensure_login(relogin=True)
         try:
@@ -162,7 +165,8 @@ class Node(FastAPI):
         The current client is disconnected and deleted if it already exists."""
 
         self.log.debug('-------------- Connecting to loop via socket.io -------------------')
-        self.log.debug('HTTP Cookies: %s\n', self.loop_communicator.get_cookies())
+        cookies = self.loop_communicator.get_cookies()
+        self.log.debug('HTTP Cookies: %s\n', cookies)
 
         if self._sio_client is not None:
             try:
@@ -185,8 +189,12 @@ class Node(FastAPI):
             ssl_context.verify_mode = ssl.CERT_REQUIRED
             connector = TCPConnector(ssl=ssl_context)
 
-        self._sio_client = AsyncClient(request_timeout=20, http_session=aiohttp.ClientSession(
-            cookies=self.loop_communicator.get_cookies(), connector=connector))
+        if self.needs_login:
+            self._sio_client = AsyncClient(request_timeout=20, http_session=aiohttp.ClientSession(
+                cookies=cookies, connector=connector))
+        else:
+            self._sio_client = AsyncClient(request_timeout=20, http_session=aiohttp.ClientSession(
+                connector=connector))
 
         # pylint: disable=protected-access
         self._sio_client._trigger_event = ensure_socket_response(self._sio_client._trigger_event)

learning_loop_node/trainer/exceptions.py

@@ -1,2 +1,12 @@
 class CriticalError(Exception):
-    pass
+    '''
+    CriticalError is raised when the training cannot be continued.
+    In this case the trainer jumps to the TrainerState.ReadyForCleanup and tries to upload the latest model.
+    '''
+
+
+class NodeNeedsRestartError(Exception):
+    '''
+    NodeNeedsRestartError is raised when the node needs to be restarted.
+    This is e.g. the case when the GPU is not available anymore.
+    '''

learning_loop_node/trainer/trainer_logic_generic.py

@@ -14,7 +14,7 @@ from ..data_classes import (Context, Errors, Hyperparameter, PretrainedModel, Tr
                             TrainingOut, TrainingStateData)
 from ..helpers.misc import create_project_folder, delete_all_training_folders, generate_training, is_valid_uuid4
 from .downloader import TrainingsDownloader
-from .exceptions import CriticalError
+from .exceptions import CriticalError, NodeNeedsRestartError
 from .io_helpers import ActiveTrainingIO, EnvironmentVars, LastTrainingIO
 
 if TYPE_CHECKING:
@@ -294,6 +294,9 @@ class TrainerLogicGeneric(ABC):
             logger.error('CriticalError in %s - Exception: %s', state_during, e)
             self.errors.set(error_key, str(e))
             self.training.training_state = TrainerState.ReadyForCleanup
+        except NodeNeedsRestartError:
+            logger.error('Node Restart Requested')
+            sys.exit(0)
         except Exception as e:
             self.errors.set(error_key, str(e))
             logger.exception('Error in %s - Exception: %s', state_during, e)

{learning_loop_node-0.10.14.dist-info → learning_loop_node-0.10.15rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.14
+Version: 0.10.15rc1
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.10.14.dist-info → learning_loop_node-0.10.15rc1.dist-info}/RECORD

@@ -31,8 +31,8 @@ learning_loop_node/helpers/environment_reader.py,sha256=OtCTDc0KT9r-SMygkZB_Mw-Z
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
 learning_loop_node/helpers/log_conf.py,sha256=z_0PHh7U7DkJbSbKoSPyUfS7NhBHtRxXHdNcj67Hpbc,951
 learning_loop_node/helpers/misc.py,sha256=j4is8Rv0ttnCqF-R-wP3xwEi67OI6IBJav5Woo5lyDk,7701
-learning_loop_node/loop_communication.py,sha256=rG5MdavSTaREZ6OWfAUIT_qkkYPw3is2_FujLmHQeIc,6576
-learning_loop_node/node.py,sha256=7Xc6VM_Rk1yDKDjKoUjoY4gSsC1zokjt5YQJu0PP5ao,9931
+learning_loop_node/loop_communication.py,sha256=xkoZtHRgxq1arusHQtC_lEBculFBLeCijyfVSYIEchY,6755
+learning_loop_node/node.py,sha256=vbMR_6QsruB2IYYKUWx4--9Ywjf_vuBQb4jyzLRqpRQ,10300
 learning_loop_node/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/rest.py,sha256=o1dl4Mtznd5duyEQtCYSGlK04l1Y-p_YRjG40Q4l31c,1491
 learning_loop_node/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -81,15 +81,15 @@ learning_loop_node/tests/trainer/test_trainer_states.py,sha256=djYCs5ieajQHRjk8Q
 learning_loop_node/tests/trainer/testing_trainer_logic.py,sha256=KslqDJDntkgH4Yd_z-guiVPvzi5Q-l-Bqc3fUjT5N7U,3883
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/downloader.py,sha256=Qk-oBcrGCVuWTVs3hvAJzQSqCIHPGZ7NXLJ_fAqvCoY,1469
-learning_loop_node/trainer/exceptions.py,sha256=hLLDGncC6PLZjKg4lZBpu-QA8itQIxiuxExz1uptgnw,40
+learning_loop_node/trainer/exceptions.py,sha256=vbuoE6kssLQuA8zd3LiDHmZglP6E2IJJwEi5AZtWXxY,420
 learning_loop_node/trainer/executor.py,sha256=-0BxDqmAI1NCiISi7Rw8McJQfgxxVy1gSa1epYuL3U0,3942
 learning_loop_node/trainer/io_helpers.py,sha256=hGEtNAQBSBbVB56U1ndwfP8qK5K4YIwMQrjCDcaMy9I,7218
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/rest/backdoor_controls.py,sha256=-pU4iHheBWf0SW2QzBVBsLiCMZBRz9CDdVZv6414Ts8,5134
 learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
 learning_loop_node/trainer/trainer_logic.py,sha256=PlYExIskU9pWJO0e9m_0KJnUdOI10GtW0oDOevYmg1o,8461
-learning_loop_node/trainer/trainer_logic_generic.py,sha256=dVsvl1LC1w3svl4O8HCW49R7vSpVpC9Ermu4C20sFoA,25821
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=ERfuGhHGNvIPRyd_QOGavylPDXTCC8qCOO1eJXAwEO8,25957
 learning_loop_node/trainer/trainer_node.py,sha256=8ANS9iy-swdTLvt9wEFixE6YlmqvqBl17A-R4tVYD-I,5384
-learning_loop_node-0.10.14.dist-info/METADATA,sha256=RHHVhHfTIPjxY5lO3UsR6L7BbHzwXs9wch4WJzePXJQ,11907
-learning_loop_node-0.10.14.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.10.14.dist-info/RECORD,,
+learning_loop_node-0.10.15rc1.dist-info/METADATA,sha256=uPPK1CQexq-mVs_1oXnzd3CrGTvieHyApPYipMsOud8,11910
+learning_loop_node-0.10.15rc1.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.10.15rc1.dist-info/RECORD,,