imt-ring 1.6.34__tar.gz → 1.6.36__tar.gz

{imt_ring-1.6.34 → imt_ring-1.6.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imt-ring
-Version: 1.6.34
+Version: 1.6.36
 Summary: RING: Recurrent Inertial Graph-based Estimator
 Author-email: Simon Bachhuber <simon.bachhuber@fau.de>
 Project-URL: Homepage, https://github.com/SimiPixel/ring

{imt_ring-1.6.34 → imt_ring-1.6.36}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "imt-ring"
-version = "1.6.34"
+version = "1.6.36"
 authors = [
   { name="Simon Bachhuber", email="simon.bachhuber@fau.de" },
 ]

{imt_ring-1.6.34 → imt_ring-1.6.36}/src/imt_ring.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imt-ring
-Version: 1.6.34
+Version: 1.6.36
 Summary: RING: Recurrent Inertial Graph-based Estimator
 Author-email: Simon Bachhuber <simon.bachhuber@fau.de>
 Project-URL: Homepage, https://github.com/SimiPixel/ring

{imt_ring-1.6.34 → imt_ring-1.6.36}/src/ring/ml/callbacks.py

@@ -388,6 +388,14 @@ class TimingKillRunCallback(training_loop.TrainingLoopCallback):
 
 
 class CheckpointCallback(training_loop.TrainingLoopCallback):
+    def __init__(
+        self,
+        checkpoint_every: Optional[int] = None,
+        checkpoint_folder: str = "~/.ring_checkpoints",
+    ):
+        self.checkpoint_every = checkpoint_every
+        self.checkpoint_folder = checkpoint_folder
+
     def after_training_step(
         self,
         i_episode: int,
@@ -401,18 +409,26 @@ class CheckpointCallback(training_loop.TrainingLoopCallback):
         self.params = params
         self.opt_state = opt_state
 
+        if self.checkpoint_every is not None and (
+            (i_episode % self.checkpoint_every) == 0
+        ):
+            self._create_checkpoint()
+
+    def _create_checkpoint(self):
+        path = parse_path(
+            self.checkpoint_folder, ml_utils.unique_id(), extension="pickle"
+        )
+        data = {"params": self.params, "opt_state": self.opt_state}
+        pickle_save(
+            obj=jax.device_get(data),
+            path=path,
+            overwrite=True,
+        )
+
     def close(self):
         # only checkpoint if run has been killed
         if training_loop.recv_kill_run_signal():
-            path = parse_path(
-                "~/.ring_checkpoints", ml_utils.unique_id(), extension="pickle"
-            )
-            data = {"params": self.params, "opt_state": self.opt_state}
-            pickle_save(
-                obj=jax.device_get(data),
-                path=path,
-                overwrite=True,
-            )
+            self._create_checkpoint()
 
 
 class WandbKillRun(training_loop.TrainingLoopCallback):

{imt_ring-1.6.34 → imt_ring-1.6.36}/src/ring/ml/training_loop.py

@@ -2,11 +2,12 @@ import random
 from typing import Optional
 
 import jax
-from ring.algorithms import Generator
-from ring.ml import ml_utils
 import tqdm
 import tree_utils
 
+from ring.algorithms import Generator
+from ring.ml import ml_utils
+
 _KILL_RUN = False
 
 
@@ -83,14 +84,20 @@ class TrainingLoop:
         # reset the kill_run flag from previous runs
         send_kill_run_signal(value=False)
 
-        for _ in tqdm.tqdm(range(n_episodes)):
-            self.step()
-
-            if recv_kill_run_signal():
-                break
+        try:
+            for _ in tqdm.tqdm(range(n_episodes)):
+                self.step()
 
-        if close_afterwards:
-            self.close()
+                if recv_kill_run_signal():
+                    break
+        except Exception as e:
+            print(
+                "Exception occured, attemping to .close() all callbacks before raising."
+            )
+            raise e
+        finally:
+            if close_afterwards:
+                self.close()
 
         return recv_kill_run_signal()