imt-ring 1.6.17__tar.gz → 1.6.19__tar.gz

{imt_ring-1.6.17 → imt_ring-1.6.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imt-ring
-Version: 1.6.17
+Version: 1.6.19
 Summary: RING: Recurrent Inertial Graph-based Estimator
 Author-email: Simon Bachhuber <simon.bachhuber@fau.de>
 Project-URL: Homepage, https://github.com/SimiPixel/ring

{imt_ring-1.6.17 → imt_ring-1.6.19}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "imt-ring"
-version = "1.6.17"
+version = "1.6.19"
 authors = [
   { name="Simon Bachhuber", email="simon.bachhuber@fau.de" },
 ]

{imt_ring-1.6.17 → imt_ring-1.6.19}/src/imt_ring.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imt-ring
-Version: 1.6.17
+Version: 1.6.19
 Summary: RING: Recurrent Inertial Graph-based Estimator
 Author-email: Simon Bachhuber <simon.bachhuber@fau.de>
 Project-URL: Homepage, https://github.com/SimiPixel/ring

{imt_ring-1.6.17 → imt_ring-1.6.19}/src/imt_ring.egg-info/SOURCES.txt

@@ -86,6 +86,7 @@ src/ring/utils/backend.py
 src/ring/utils/batchsize.py
 src/ring/utils/colab.py
 src/ring/utils/dataloader.py
+src/ring/utils/dataloader_torch.py
 src/ring/utils/hdf5.py
 src/ring/utils/normalizer.py
 src/ring/utils/path.py

{imt_ring-1.6.17 → imt_ring-1.6.19}/src/ring/ml/ml_utils.py

@@ -3,6 +3,7 @@ from functools import partial
 import os
 from pathlib import Path
 import pickle
+import shutil
 import time
 from typing import Optional, Protocol
 import warnings
@@ -107,7 +108,7 @@ class WandbLogger(MixinLogger):
         wandb.log(data)
 
     def log_params(self, path: str):
-        wandb.save(path, policy="now")
+        self.wandb_save(path)
 
     def log_video(
         self,
@@ -117,7 +118,7 @@ class WandbLogger(MixinLogger):
         step: Optional[int] = None,
     ):
         # TODO >>>
-        wandb.save(path, policy="now")
+        self.wandb_save(path)
         return
         # <<<
         data = {"video": wandb.Video(path, caption=caption, fps=fps)}
@@ -127,10 +128,10 @@ class WandbLogger(MixinLogger):
 
     def log_image(self, path: str, caption: Optional[str] = None):
         # wandb.log({"image": wandb.Image(path, caption=caption)})
-        wandb.save(path, policy="now")
+        self.wandb_save(path)
 
     def log_txt(self, path: str, wait: bool = True):
-        wandb.save(path, policy="now")
+        self.wandb_save(path)
         # TODO: `wandb` is not async at all?
         if wait:
             time.sleep(3)
@@ -138,6 +139,23 @@ class WandbLogger(MixinLogger):
     def close(self):
         wandb.run.finish()
 
+    @staticmethod
+    def wandb_save(path):
+        if wandb.run is not None and wandb.run.settings._offline:
+            # Create a dedicated directory in the WandB run directory to store copies
+            # of files
+            destination_dir = os.path.join(wandb.run.dir, "copied_files")
+            os.makedirs(destination_dir, exist_ok=True)
+
+            # Copy the file to this new location
+            copied_file_path = os.path.join(destination_dir, os.path.basename(path))
+            shutil.copy2(path, copied_file_path)
+
+            # Use wandb.save to save the copied file (now a true copy)
+            wandb.save(copied_file_path)
+        else:
+            wandb.save(path, policy="now")
+
 
 def _flatten_convert_filter_nested_dict(
     metrices: NestedDict, filter_nan_inf: bool = True

imt_ring-1.6.19/src/ring/utils/dataloader_torch.py

@@ -0,0 +1,62 @@
+import os
+
+import jax
+import torch
+from torch.utils.data import DataLoader
+from torch.utils.data import Dataset
+from tree_utils import PyTree
+
+from ring.utils import parse_path
+from ring.utils import pickle_load
+
+
+class FolderOfPickleFilesDataset(Dataset):
+    def __init__(self, path, transform=None):
+        self.files = self.listdir(path)
+        self.transform = transform
+        self.N = len(self.files)
+
+    def __len__(self):
+        return self.N
+
+    def __getitem__(self, idx: int):
+        element = pickle_load(self.files[idx])
+        if self.transform is not None:
+            element = self.transform(element)
+        return element
+
+    @staticmethod
+    def listdir(path: str) -> list:
+        return [parse_path(path, file) for file in os.listdir(path)]
+
+
+def dataset_to_generator(
+    dataset: Dataset,
+    batch_size: int,
+    shuffle=True,
+    seed: int = 1,
+    **kwargs,
+):
+    torch.manual_seed(seed)
+
+    dl = DataLoader(
+        dataset,
+        batch_size=batch_size,
+        shuffle=shuffle,
+        multiprocessing_context="spawn" if kwargs.get("num_workers", 0) > 0 else None,
+        **kwargs,
+    )
+    dl_iter = iter(dl)
+
+    def to_numpy(tree: PyTree[torch.Tensor]):
+        return jax.tree_map(lambda tensor: tensor.numpy(), tree)
+
+    def generator(_):
+        nonlocal dl, dl_iter
+        try:
+            return to_numpy(next(dl_iter))
+        except StopIteration:
+            dl_iter = iter(dl)
+            return to_numpy(next(dl_iter))
+
+    return generator