imt-ring 1.6.32__tar.gz → 1.6.33__tar.gz

{imt_ring-1.6.32 → imt_ring-1.6.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imt-ring
-Version: 1.6.32
+Version: 1.6.33
 Summary: RING: Recurrent Inertial Graph-based Estimator
 Author-email: Simon Bachhuber <simon.bachhuber@fau.de>
 Project-URL: Homepage, https://github.com/SimiPixel/ring

{imt_ring-1.6.32 → imt_ring-1.6.33}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "imt-ring"
-version = "1.6.32"
+version = "1.6.33"
 authors = [
   { name="Simon Bachhuber", email="simon.bachhuber@fau.de" },
 ]

{imt_ring-1.6.32 → imt_ring-1.6.33}/src/imt_ring.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imt-ring
-Version: 1.6.32
+Version: 1.6.33
 Summary: RING: Recurrent Inertial Graph-based Estimator
 Author-email: Simon Bachhuber <simon.bachhuber@fau.de>
 Project-URL: Homepage, https://github.com/SimiPixel/ring

{imt_ring-1.6.32 → imt_ring-1.6.33}/src/ring/algorithms/generator/base.py

@@ -213,6 +213,8 @@ class RCMG:
                 )
                 save_fn(d, file)
                 i += 1
+            # cleanup
+            del data
 
         gens, n_calls = self._generators_ncalls(sizes)
         batch.generators_eager(gens, n_calls, callback, seed, self._disable_tqdm)

{imt_ring-1.6.32 → imt_ring-1.6.33}/src/ring/algorithms/generator/batch.py

@@ -1,3 +1,4 @@
+import gc
 from typing import Callable
 
 import jax
@@ -83,4 +84,8 @@ def generators_eager(
 
             sample_flat, _ = jax.tree_util.tree_flatten(sample)
             size = 1 if len(sample_flat) == 0 else sample_flat[0].shape[0]
-            callback([jax.tree_map(lambda a: a[i], sample) for i in range(size)])
+            callback([jax.tree_map(lambda a: a[i].copy(), sample) for i in range(size)])
+
+            # cleanup
+            del sample, sample_flat
+            gc.collect()

{imt_ring-1.6.32 → imt_ring-1.6.33}/src/ring/utils/dataloader_torch.py

@@ -1,8 +1,9 @@
 import os
-from typing import Optional
+from typing import Any, Optional
 import warnings
 
 import jax
+import numpy as np
 import torch
 from torch.utils.data import DataLoader
 from torch.utils.data import Dataset
@@ -12,7 +13,7 @@ from ring.utils import parse_path
 from ring.utils import pickle_load
 
 
-class FolderOfPickleFilesDataset(Dataset):
+class FolderOfFilesDataset(Dataset):
     def __init__(self, path, transform=None):
         self.files = self.listdir(path)
         self.transform = transform
@@ -22,7 +23,7 @@ class FolderOfPickleFilesDataset(Dataset):
         return self.N
 
     def __getitem__(self, idx: int):
-        element = pickle_load(self.files[idx])
+        element = self._load_file(self.files[idx])
         if self.transform is not None:
             element = self.transform(element)
         return element
@@ -31,6 +32,10 @@ class FolderOfPickleFilesDataset(Dataset):
     def listdir(path: str) -> list:
         return [parse_path(path, file) for file in os.listdir(path)]
 
+    @staticmethod
+    def _load_file(file_path: str) -> Any:
+        return pickle_load(file_path)
+
 
 def dataset_to_generator(
     dataset: Dataset,
@@ -84,3 +89,60 @@ def _get_number_of_logical_cores() -> int:
         )
         N = 0
     return N
+
+
+class MultiDataset(Dataset):
+    def __init__(self, datasets, transform=None):
+        """
+        Args:
+            datasets: A list of datasets to sample from.
+            transform: A function that takes N items (one from each dataset) and combines them.
+        """  # noqa: E501
+        self.datasets = datasets
+        self.transform = transform
+
+    def __len__(self):
+        # Length is defined by the smallest dataset in the list
+        return min(len(ds) for ds in self.datasets)
+
+    def __getitem__(self, idx):
+        sampled_items = [ds[idx] for ds in self.datasets]
+
+        if self.transform:
+            # Apply the transformation to all sampled items
+            return self.transform(*sampled_items)
+
+        return tuple(sampled_items)
+
+
+class ShuffledDataset(Dataset):
+    def __init__(self, dataset):
+        """
+        Wrapper that shuffles the dataset indices once.
+
+        Args:
+            dataset (Dataset): The original dataset to shuffle.
+        """
+        self.dataset = dataset
+        self.shuffled_indices = np.random.permutation(
+            len(dataset)
+        )  # Shuffle indices once
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getitem__(self, idx):
+        """
+        Returns the data at the shuffled index.
+
+        Args:
+            idx (int): Index in the shuffled dataset.
+        """
+        original_idx = self.shuffled_indices[idx]
+        return self.dataset[original_idx]
+
+
+def dataset_to_Xy(ds: Dataset):
+    return dataset_to_generator(ds, batch_size=len(ds), shuffle=False, num_workers=0)(
+        None
+    )