imt-ring 1.5.0__py3-none-any.whl → 1.5.2__py3-none-any.whl

{imt_ring-1.5.0.dist-info → imt_ring-1.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imt-ring
-Version: 1.5.0
+Version: 1.5.2
 Summary: RING: Recurrent Inertial Graph-based Estimator
 Author-email: Simon Bachhuber <simon.bachhuber@fau.de>
 Project-URL: Homepage, https://github.com/SimiPixel/ring

{imt_ring-1.5.0.dist-info → imt_ring-1.5.2.dist-info}/RECORD

@@ -4,20 +4,20 @@ ring/base.py,sha256=YFPrUWelWswEhq8x8Byv-5pK64mipiGW6x5IlMr4we4,33803
 ring/maths.py,sha256=jJr_kr78-XDce8B4tXQ2Li-jBntVQhaS8csxglCsj8A,12193
 ring/spatial.py,sha256=nmZ-UhRanhyM34bez8uCS4wMwaKqLkuEbgKGP5XNH60,2351
 ring/algorithms/__init__.py,sha256=IiK9EN5Xgs3dB075-A-H-Yad0Z7vzvKIJF2g6X_-C_8,1224
-ring/algorithms/_random.py,sha256=M9JQSMXSUARWuzlRLP3Wmkuntrk9LZpP30p4_IPgDB4,13805
+ring/algorithms/_random.py,sha256=fc26yEQjSjtf0NluZ41CyeGIRci0ldrRlThueHR9H7U,14007
 ring/algorithms/dynamics.py,sha256=_TwclBXe6vi5C5iJWAIeUIJEIMHQ_1QTmnHvCEpVO0M,10867
-ring/algorithms/jcalc.py,sha256=seis_VQwSvyrZBtmgKwAgSfT_fhXT4Gcyufp0KCUBME,28094
+ring/algorithms/jcalc.py,sha256=bM8VARgqEiVPy7632geKYGk4MZddZfI8XHdW5kXF3HI,28594
 ring/algorithms/kinematics.py,sha256=DOboHI517Vx0pRJUFZtZPmK_qFaiKiQe-37B-M0aC-c,7422
 ring/algorithms/sensors.py,sha256=MICO9Sn0AfoqRx_9KWR3hufsIID-K6SOIg3oPDgsYMU,17869
 ring/algorithms/custom_joints/__init__.py,sha256=fzeE7TdUhmGgbbFAyis1tKcyQ4Fo8LigDwD3hUVnH_w,316
-ring/algorithms/custom_joints/rr_imp_joint.py,sha256=a3JT0w7pB94kZ95eBR8ZO853eSeyjFoiXmhYlaXoHDE,2392
+ring/algorithms/custom_joints/rr_imp_joint.py,sha256=_YJK0p8_0MHFtr1NuGnNZoxTbwaMQyUjYv7EtsPiU3A,2402
 ring/algorithms/custom_joints/rr_joint.py,sha256=jnRtjtOCALMaq2_0bcu2d7qgfQ6etXpoh43MioRaDmY,1000
-ring/algorithms/custom_joints/suntay.py,sha256=7-kym1kMDwqYD_2um1roGcBeB8BlTCPe1wljuNGNARA,16676
+ring/algorithms/custom_joints/suntay.py,sha256=tOEGM304XciHO4pmvxr4faA4xXVO4N2HlPdFmXKbcrw,16726
 ring/algorithms/generator/__init__.py,sha256=bF-CW3x2x-o6KWESKy-DuxzZPh3UNSjJb_MaAcSHGsQ,277
-ring/algorithms/generator/base.py,sha256=1rzClXZ0WMJ_IZroTO9i1aWiHBy7whsCrJLIMY4zC3c,13280
-ring/algorithms/generator/batch.py,sha256=Hwh5jYZQEmkx73YaXjWd6sZdikmj43spE7DCzGDHXtE,6637
-ring/algorithms/generator/finalize_fns.py,sha256=0fbtwQw89_w0ytQ_aJ877CZGY5fbtb8sbsRO0O8pT34,9081
-ring/algorithms/generator/motion_artifacts.py,sha256=vzBLlG60KCAa7Zj1RdUiRkoOx_3inA_2M1mBKl3lTKs,8834
+ring/algorithms/generator/base.py,sha256=KQSg9uhhR-rC563busVFx4gJrqOx3BXdaChozO9gwTA,14224
+ring/algorithms/generator/batch.py,sha256=ylootnXmj-JyuB_f5OCknHst9wFKO3gkjQbMrFNXY2g,2513
+ring/algorithms/generator/finalize_fns.py,sha256=L_5wIVA7g0P4P2U6EmgcvsoI-YuF3TOaHBwk5_oEaUU,9077
+ring/algorithms/generator/motion_artifacts.py,sha256=2VJbldVDbI3PSyboshIbtYvSAKzBBwGV7cQfYjqvluM,9167
 ring/algorithms/generator/pd_control.py,sha256=XJ_Gd5AkIRh-jBrMfQyMXjVwhx2gCNHznjzFbmAwhZs,5767
 ring/algorithms/generator/setup_fns.py,sha256=MFz3czHBeWs1Zk1A8O02CyQpQ-NCyW9PMpbqmKit6es,1455
 ring/algorithms/generator/types.py,sha256=HjNyATFSLfHkXlzdJhvUkiqnhzpXFDDXmWS3LYBlOtU,721
@@ -53,7 +53,7 @@ ring/io/xml/to_xml.py,sha256=fohb-jWMf2cxVdT5dmknsGyrNMseICSbKEz_urbaWbQ,3407
 ring/ml/__init__.py,sha256=8SZTCs9rJ1kzR0Psh7lUzFhIMhKRPIK41mVfxJAGyMo,1471
 ring/ml/base.py,sha256=-3JQ27zMFESNn5zeNer14GJU2yQgiqDcJUaULOeSyp8,9799
 ring/ml/callbacks.py,sha256=W19QF6_uvaNCjs8ObsjNXD7mv9gFgJBixdRSbB_BynE,13301
-ring/ml/ml_utils.py,sha256=hQEmeZoahdJyFrz0NZXYi1Yijl7GvPBdqwzZBzlUIUM,7638
+ring/ml/ml_utils.py,sha256=GooyH5uxA6cJM7ZcWDUfSkSKq6dg7kCIbhkbjJs_rLw,6674
 ring/ml/optimizer.py,sha256=fWyF__ezUltrA16SLfOC1jvS3zBh9NJsMYa6-V0frhs,4709
 ring/ml/ringnet.py,sha256=rgje5AKUKpT8K-vbE9_SgZ3IijR8TJEHnaqxsE57Mhc,8617
 ring/ml/rnno_v1.py,sha256=T4SKG7iypqn2HBQLKhDmJ2Slj2Z5jtUBHvX_6aL8pyM,1103
@@ -72,7 +72,7 @@ ring/sys_composer/__init__.py,sha256=5J_JJJIHfTPcpxh0v4FqiOs81V1REPUd7pgiw2nAN5E
 ring/sys_composer/delete_sys.py,sha256=cIM9KbyLfg7B9121g7yjzuFbjeNu9cil1dPavAYEgzk,3408
 ring/sys_composer/inject_sys.py,sha256=Mj-q-mUjXKwkg-ol6IQAjf9IJfk7pGhez0_WoTKTgm0,3503
 ring/sys_composer/morph_sys.py,sha256=2GpPtS5hT0eZMptdGpt30Hc97OykJNE67lEVRf7sHrc,12700
-ring/utils/__init__.py,sha256=9ZEooVyri0IWXHA5T-L03vP7aWX0zo8qvfNioGnIAkc,696
+ring/utils/__init__.py,sha256=M9bR1-SYtmF9c4mTRIrGuIQws3K2aKUQxbpltIDkgZQ,739
 ring/utils/backend.py,sha256=cKSi9sB59texqKzNVASTDczGKLCBL8VVDiP7TNdj41k,1294
 ring/utils/batchsize.py,sha256=FbOii7MDP4oPZd9GJOKehFatfnb6WZ0b9z349iZYs1A,1786
 ring/utils/colab.py,sha256=ZLHwP0jNQUsmZJU4l68a5djULPi6T-jYNNHevjIoMn8,1631
@@ -80,8 +80,8 @@ ring/utils/hdf5.py,sha256=BzXwVypZmEZeHVgeGZ78YYdi10NEQtnPhdrb8dQAXo0,5856
 ring/utils/normalizer.py,sha256=67L2BU1MRsMT4pD41ta3JJMppLN0ozFmnwrmXDtnqrQ,1698
 ring/utils/path.py,sha256=hAfSlqRi-ew536RnjDDM7IKapdMJc-EvhrR0Y-BCFWc,1265
 ring/utils/randomize_sys.py,sha256=G_vBIo0OwQkXL2u0djwbaoaeb02C4LQCTNNloOYIU2M,3699
-ring/utils/utils.py,sha256=VkB0Gvmlaz2MZdntgjWA0rOpRkvIRpLWRFgIofoY7hs,5441
-imt_ring-1.5.0.dist-info/METADATA,sha256=ZjJMt4357zV4eK-ZKH_d4Q7nhc8dJ6RG_AnocvzDNzU,3104
-imt_ring-1.5.0.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-imt_ring-1.5.0.dist-info/top_level.txt,sha256=EiT790-lAyi8iwTzJArH3f2k77rwhDn00q-4PlmvDQo,5
-imt_ring-1.5.0.dist-info/RECORD,,
+ring/utils/utils.py,sha256=Y8B2V647JMM57S3GmCwAjCM4XuN5RwMLhcDfjReP3kQ,6526
+imt_ring-1.5.2.dist-info/METADATA,sha256=YhkKO-ToWNUrygQCGNFqn6Ugph4_ZVHdLK8W7LnL2n0,3104
+imt_ring-1.5.2.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+imt_ring-1.5.2.dist-info/top_level.txt,sha256=EiT790-lAyi8iwTzJArH3f2k77rwhDn00q-4PlmvDQo,5
+imt_ring-1.5.2.dist-info/RECORD,,

ring/algorithms/_random.py

@@ -29,7 +29,8 @@ def random_angle_over_time(
     t_min: float,
     t_max: float | TimeDependentFloat,
     T: float,
-    Ts: float,
+    Ts: float | jax.Array,
+    N: Optional[int] = None,
     max_iter: int = 5,
     randomized_interpolation: bool = False,
     range_of_motion: bool = False,
@@ -84,7 +85,10 @@ def random_angle_over_time(
     )
 
     # resample
-    t = jnp.arange(T, step=Ts)
+    if N is None:
+        t = jnp.arange(T, step=Ts)
+    else:
+        t = jnp.arange(N) * Ts
     if randomized_interpolation:
         q = interpolate(cdf_bins_min, cdf_bins_max, method=interpolation_method)(
             t, ANG[:, 0], ANG[:, 1], consume
@@ -117,7 +121,8 @@ def random_position_over_time(
     t_max: float | TimeDependentFloat,
     T: float,
     Ts: float,
-    max_it: int,
+    N: Optional[int] = None,
+    max_it: int = 100,
     randomized_interpolation: bool = False,
     cdf_bins_min: int = 5,
     cdf_bins_max: Optional[int] = None,
@@ -203,7 +208,10 @@ def random_position_over_time(
     )
 
     # resample
-    t = jnp.arange(T, step=Ts)
+    if N is None:
+        t = jnp.arange(T, step=Ts)
+    else:
+        t = jnp.arange(N) * Ts
     if randomized_interpolation:
         r = interpolate(cdf_bins_min, cdf_bins_max, method=interpolation_method)(
             t, POS[:, 0], POS[:, 1], consume

ring/algorithms/custom_joints/rr_imp_joint.py

@@ -2,6 +2,7 @@ from dataclasses import replace
 
 import jax
 import jax.numpy as jnp
+
 import ring
 from ring import maths
 from ring.algorithms.jcalc import _draw_rxyz
@@ -21,12 +22,12 @@ def register_rr_imp_joint(
         rot = ring.maths.quat_mul(rot_res, rot_pri)
         return ring.Transform.create(rot=rot)
 
-    def _draw_rr_imp(config, key_t, key_value, dt, _):
+    def _draw_rr_imp(config, key_t, key_value, dt, N, _):
         key_t1, key_t2 = jax.random.split(key_t)
         key_value1, key_value2 = jax.random.split(key_value)
-        q_traj_pri = _draw_rxyz(config, key_t1, key_value1, dt, _)
+        q_traj_pri = _draw_rxyz(config, key_t1, key_value1, dt, N, _)
         q_traj_res = _draw_rxyz(
-            replace(config_res, T=config.T), key_t2, key_value2, dt, _
+            replace(config_res, T=config.T), key_t2, key_value2, dt, N, _
         )
         # scale to be within bounds
         q_traj_res = q_traj_res * (jnp.deg2rad(ang_max_deg) / jnp.pi)

ring/algorithms/custom_joints/suntay.py

@@ -225,7 +225,8 @@ def register_suntay(sconfig: SuntayConfig, name: str = "suntay"):
         mconfig: ring.MotionConfig,
         key_t: jax.random.PRNGKey,
         key_value: jax.random.PRNGKey,
-        dt: float,
+        dt: float | jax.Array,
+        N: int | None,
         _: jax.Array,
     ) -> jax.Array:
         key_value, consume = jax.random.split(key_value)
@@ -251,6 +252,7 @@ def register_suntay(sconfig: SuntayConfig, name: str = "suntay"):
             mconfig.t_max,
             mconfig.T,
             dt,
+            N,
             5,
             mconfig.randomized_interpolation_angle,
             mconfig.range_of_motion_hinge,

ring/algorithms/generator/base.py

@@ -1,3 +1,4 @@
+import random
 from typing import Callable, Optional
 import warnings
 
@@ -33,8 +34,10 @@ class RCMG:
         randomize_positions: bool = False,
         randomize_motion_artifacts: bool = False,
         randomize_joint_params: bool = False,
+        randomize_hz: bool = False,
+        randomize_hz_kwargs: dict = dict(),
         imu_motion_artifacts: bool = False,
-        imu_motion_artifacts_kwargs: dict = dict(hide_injected_bodies=True),
+        imu_motion_artifacts_kwargs: dict = dict(),
         dynamic_simulation: bool = False,
         dynamic_simulation_kwargs: dict = dict(),
         output_transform: Optional[Callable] = None,
@@ -50,9 +53,6 @@ class RCMG:
         for c in config:
             assert c.is_feasible()
 
-        if cor:
-            sys = [s._replace_free_with_cor() for s in sys]
-
         self.gens = []
         for _sys in sys:
             self.gens.append(
@@ -71,6 +71,8 @@ class RCMG:
                     randomize_positions=randomize_positions,
                     randomize_motion_artifacts=randomize_motion_artifacts,
                     randomize_joint_params=randomize_joint_params,
+                    randomize_hz=randomize_hz,
+                    randomize_hz_kwargs=randomize_hz_kwargs,
                     imu_motion_artifacts=imu_motion_artifacts,
                     imu_motion_artifacts_kwargs=imu_motion_artifacts_kwargs,
                     dynamic_simulation=dynamic_simulation,
@@ -78,6 +80,7 @@ class RCMG:
                     output_transform=output_transform,
                     keep_output_extras=keep_output_extras,
                     use_link_number_in_Xy=use_link_number_in_Xy,
+                    cor=cor,
                 )
             )
 
@@ -174,35 +177,37 @@ class RCMG:
         sizes: int | list[int] = 1,
         seed: int = 1,
         shuffle: bool = True,
+        transform=None,
     ) -> types.BatchedGenerator:
         data = self.to_list(sizes, seed)
         assert len(data) >= batchsize
-
-        def data_fn(indices: list[int]):
-            return tree_utils.tree_batch([data[i] for i in indices])
-
-        return batch.generator_from_data_fn(
-            data_fn, list(range(len(data))), shuffle, batchsize
-        )
+        return self.eager_gen_from_list(data, batchsize, shuffle, transform)
 
     @staticmethod
-    def eager_gen_from_paths(
-        paths: str | list[str],
+    def eager_gen_from_list(
+        data: list[tree_utils.PyTree],
         batchsize: int,
-        include_samples: Optional[list[int]] = None,
         shuffle: bool = True,
-        load_all_into_memory: bool = False,
-        tree_transform=None,
-    ) -> tuple[types.BatchedGenerator, int]:
-        paths = utils.to_list(paths)
-        return batch.generator_from_paths(
-            paths,
-            batchsize,
-            include_samples,
-            shuffle,
-            load_all_into_memory=load_all_into_memory,
-            tree_transform=tree_transform,
-        )
+        transform=None,
+    ) -> types.BatchedGenerator:
+        data = data.copy()
+        n_batches, i = len(data) // batchsize, 0
+
+        def generator(key: jax.Array):
+            nonlocal i
+            if shuffle and i == 0:
+                random.shuffle(data)
+
+            start, stop = i * batchsize, (i + 1) * batchsize
+            batch = tree_utils.tree_batch(data[start:stop], backend="numpy")
+            batch = utils.pytree_deepcopy(batch)
+            if transform is not None:
+                batch = transform(batch)
+
+            i = (i + 1) % n_batches
+            return batch
+
+        return generator
 
 
 def _copy_dicts(f) -> dict:
@@ -231,6 +236,8 @@ def _build_mconfig_batched_generator(
     randomize_positions: bool,
     randomize_motion_artifacts: bool,
     randomize_joint_params: bool,
+    randomize_hz: bool,
+    randomize_hz_kwargs: dict,
     imu_motion_artifacts: bool,
     imu_motion_artifacts_kwargs: dict,
     dynamic_simulation: bool,
@@ -238,6 +245,7 @@ def _build_mconfig_batched_generator(
     output_transform: Callable | None,
     keep_output_extras: bool,
     use_link_number_in_Xy: bool,
+    cor: bool,
 ) -> types.BatchedGenerator:
 
     if add_X_jointaxes or add_y_relpose or add_y_rootincl:
@@ -284,13 +292,17 @@ def _build_mconfig_batched_generator(
         for f in pipe:
             key, consume = jax.random.split(key)
             sys = f(consume, sys)
+        if cor:
+            sys = sys._replace_free_with_cor()
         return sys
 
     def _finalize_fn(Xy: types.Xy, extras: types.OutputExtras):
         pipe = []
         if dynamic_simulation:
             pipe.append(finalize_fns.DynamicalSimulation(**dynamic_simulation_kwargs))
-        if imu_motion_artifacts and imu_motion_artifacts_kwargs["hide_injected_bodies"]:
+        if imu_motion_artifacts and imu_motion_artifacts_kwargs.get(
+            "hide_injected_bodies", True
+        ):
             pipe.append(motion_artifacts.HideInjectedBodies())
         if finalize_fn is not None:
             pipe.append(finalize_fns.FinalizeFn(finalize_fn))
@@ -312,19 +324,32 @@ def _build_mconfig_batched_generator(
         return Xy, extras
 
     def _gen(key: types.PRNGKey):
+        key, *consume = jax.random.split(key, len(config) + 1)
+        syss = jax.vmap(_setup_fn, (0, None))(jnp.array(consume), sys)
+
+        if randomize_hz:
+            assert "sampling_rates" in randomize_hz_kwargs
+            hzs = randomize_hz_kwargs["sampling_rates"]
+            assert len(set([c.T for c in config])) == 1
+            N = int(min(hzs) * config[0].T)
+            key, consume = jax.random.split(key)
+            dt = 1 / jax.random.choice(consume, jnp.array(hzs))
+            # makes sys.dt from float to AbstractArray
+            syss = syss.replace(dt=jnp.array(dt))
+        else:
+            N = None
+
         qs = []
-        for _config in config:
-            key, _q = draw_random_q(key, sys, _config)
+        for i, _config in enumerate(config):
+            key, _q = draw_random_q(key, syss[i], _config, N)
             qs.append(_q)
         qs = jnp.stack(qs)
 
-        key, *consume = jax.random.split(key, len(config) + 1)
-        syss = jax.vmap(_setup_fn, (0, None))(jnp.array(consume), sys)
-
         @jax.vmap
         def _vmapped_context(key, q, sys):
             x, _ = jax.vmap(kinematics.forward_kinematics_transforms, (None, 0))(sys, q)
-            Xy, extras = ({}, {}), (key, q, x, sys)
+            X = {"dt": jnp.array(sys.dt)} if randomize_hz else {}
+            Xy, extras = (X, {}), (key, q, x, sys)
             return _finalize_fn(Xy, extras)
 
         keys = jax.random.split(key, len(config))
@@ -340,6 +365,7 @@ def draw_random_q(
     key: types.PRNGKey,
     sys: base.System,
     config: jcalc.MotionConfig,
+    N: int | None,
 ) -> tuple[types.Xy, types.OutputExtras]:
 
     key_start = key
@@ -360,7 +386,7 @@ def draw_random_q(
         draw_fn = jcalc.get_joint_model(link_type).rcmg_draw_fn
         if draw_fn is None:
             raise Exception(f"The joint type {link_type} has no draw fn specified.")
-        q_link = draw_fn(config, key_t, key_value, sys.dt, joint_params)
+        q_link = draw_fn(config, key_t, key_value, sys.dt, N, joint_params)
         # even revolute and prismatic joints must be 2d arrays
         q_link = q_link if q_link.ndim == 2 else q_link[:, None]
         q_list.append(q_link)

ring/algorithms/generator/batch.py

@@ -1,7 +1,3 @@
-from pathlib import Path
-import random
-from typing import Optional
-
 import jax
 import jax.numpy as jnp
 import numpy as np
@@ -88,142 +84,3 @@ def generators_eager_to_list(
             data.extend([jax.tree_map(lambda a: a[i], sample) for i in range(size)])
 
     return data
-
-
-def _is_nan(ele: tree_utils.PyTree, i: int, verbose: bool = False):
-    isnan = np.any([np.any(np.isnan(arr)) for arr in jax.tree_util.tree_leaves(ele)])
-    if isnan:
-        X, y = ele
-        dt = X["dt"].flatten()[0]
-        if verbose:
-            print(f"Sample with idx={i} is nan. It will be replaced. (dt={dt})")
-        return True
-    return False
-
-
-def _replace_elements_w_nans(list_of_data: list, include_samples: list[int]) -> list:
-    list_of_data_nonan = []
-    for i, ele in enumerate(list_of_data):
-        if _is_nan(ele, i, verbose=True):
-            while True:
-                j = random.choice(include_samples)
-                if not _is_nan(list_of_data[j], j):
-                    ele = list_of_data[j]
-                    break
-        list_of_data_nonan.append(ele)
-    return list_of_data_nonan
-
-
-_list_of_data = None
-_paths = None
-
-
-def _data_fn_from_paths(
-    paths: list[str],
-    include_samples: list[int] | None,
-    load_all_into_memory: bool,
-    tree_transform,
-):
-    "`data_fn` returns numpy arrays."
-    global _list_of_data, _paths
-
-    # expanduser
-    paths = [utils.parse_path(p, mkdir=False) for p in paths]
-    extensions = list(set([Path(p).suffix for p in paths]))
-    assert len(extensions) == 1, f"{extensions}"
-    h5 = extensions[0] == ".h5"
-
-    if h5 and not load_all_into_memory:
-
-        def data_fn(indices: list[int]):
-            tree = utils.hdf5_load_from_multiple(paths, indices)
-            return tree if tree_transform is None else tree_transform(tree)
-
-        N = sum([utils.hdf5_load_length(p) for p in paths])
-    else:
-
-        load_from_path = utils.hdf5_load if h5 else utils.pickle_load
-
-        def load_fn(path):
-            tree = load_from_path(path)
-            tree = tree if tree_transform is None else tree_transform(tree)
-            return [
-                jax.tree_map(lambda arr: arr[i], tree)
-                for i in range(tree_utils.tree_shape(tree))
-            ]
-
-        if paths != _paths or len(_list_of_data) == 0:
-            _paths = paths
-
-            _list_of_data = []
-            for p in paths:
-                _list_of_data += load_fn(p)
-
-        N = len(_list_of_data)
-        list_of_data = _replace_elements_w_nans(
-            _list_of_data,
-            include_samples if include_samples is not None else list(range(N)),
-        )
-
-        if include_samples is not None:
-            list_of_data = [
-                ele if i in include_samples else None
-                for i, ele in enumerate(list_of_data)
-            ]
-
-        def data_fn(indices: list[int]):
-            return tree_utils.tree_batch(
-                [list_of_data[i] for i in indices], backend="numpy"
-            )
-
-    if include_samples is None:
-        include_samples = list(range(N))
-
-    return data_fn, include_samples.copy()
-
-
-def generator_from_data_fn(
-    data_fn,
-    include_samples: list[int],
-    shuffle: bool,
-    batchsize: int,
-) -> types.BatchedGenerator:
-    # such that we don't mutate out of scope
-    include_samples = include_samples.copy()
-
-    N = len(include_samples)
-    n_batches, i = N // batchsize, 0
-
-    def generator(key: jax.Array):
-        nonlocal i
-        if shuffle and i == 0:
-            random.shuffle(include_samples)
-
-        start, stop = i * batchsize, (i + 1) * batchsize
-        batch = data_fn(include_samples[start:stop])
-
-        i = (i + 1) % n_batches
-        return utils.pytree_deepcopy(batch)
-
-    return generator
-
-
-def generator_from_paths(
-    paths: list[str],
-    batchsize: int,
-    include_samples: Optional[list[int]] = None,
-    shuffle: bool = True,
-    load_all_into_memory: bool = False,
-    tree_transform=None,
-) -> tuple[types.BatchedGenerator, int]:
-    "Returns: gen, where gen(key) -> Pytree[numpy]"
-    data_fn, include_samples = _data_fn_from_paths(
-        paths, include_samples, load_all_into_memory, tree_transform
-    )
-
-    N = len(include_samples)
-    assert N >= batchsize
-
-    generator = generator_from_data_fn(data_fn, include_samples, shuffle, batchsize)
-
-    return generator, N

ring/algorithms/generator/finalize_fns.py

@@ -251,8 +251,8 @@ def _expand_dt(X: dict, T: int):
     return X
 
 
-def _expand_then_flatten(args):
-    X, y = args
+def _expand_then_flatten(Xy):
+    X, y = Xy
     gyr = X["0"]["gyr"]
 
     batched = True

ring/algorithms/generator/motion_artifacts.py

@@ -1,3 +1,4 @@
+import inspect
 import warnings
 
 import jax
@@ -127,6 +128,7 @@ def setup_fn_randomize_damping_stiffness_factory(
     prob_rigid: float = 0.0,
     all_imus_either_rigid_or_flex: bool = False,
     imus_surely_rigid: list[str] = [],
+    **kwargs,
 ):
     assert 0 <= prob_rigid <= 1
     assert prob_rigid != 1, "Use `imu_motion_artifacts`=False instead."
@@ -198,6 +200,18 @@ def setup_fn_randomize_damping_stiffness_factory(
     return setup_fn_randomize_damping_stiffness
 
 
+# assert that there exists no keyword arg duplicate which would induce ambiguity
+kwargs = lambda f: set(inspect.signature(f).parameters.keys())
+assert (
+    len(
+        kwargs(inject_subsystems).intersection(
+            kwargs(setup_fn_randomize_damping_stiffness_factory)
+        )
+    )
+    == 1
+)
+
+
 def _match_q_x_between_sys(
     sys_small: base.System,
     q_large: jax.Array,

ring/algorithms/jcalc.py

@@ -274,8 +274,15 @@ def join_motionconfigs(
 
 
 DRAW_FN = Callable[
-    # config, key_t, key_value, dt, params
-    [MotionConfig, jax.random.PRNGKey, jax.random.PRNGKey, float, jax.Array],
+    # config, key_t, key_value, dt, N, params
+    [
+        MotionConfig,
+        jax.random.PRNGKey,
+        jax.random.PRNGKey,
+        float | jax.Array,
+        int | None,
+        jax.Array,
+    ],
     jax.Array,
 ]
 P_CONTROL_TERM = Callable[
@@ -410,7 +417,8 @@ def _draw_rxyz(
     config: MotionConfig,
     key_t: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     _: jax.Array,
     # TODO, delete these args and pass a modifified `config` with `replace` instead
     enable_range_of_motion: bool = True,
@@ -435,6 +443,7 @@ def _draw_rxyz(
         config.t_max,
         config.T,
         dt,
+        N,
         max_iter,
         config.randomized_interpolation_angle,
         config.range_of_motion_hinge if enable_range_of_motion else False,
@@ -449,7 +458,8 @@ def _draw_pxyz(
     config: MotionConfig,
     _: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     __: jax.Array,
     cor: bool = False,
 ) -> jax.Array:
@@ -467,6 +477,7 @@ def _draw_pxyz(
         config.cor_t_max if cor else config.t_max,
         config.T,
         dt,
+        N,
         max_iter,
         config.randomized_interpolation_position,
         config.cdf_bins_min,
@@ -479,7 +490,8 @@ def _draw_spherical(
     config: MotionConfig,
     key_t: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     _: jax.Array,
 ) -> jax.Array:
     # NOTE: We draw 3 euler angles and then build a quaternion.
@@ -491,6 +503,7 @@ def _draw_spherical(
             key_t,
             key_value,
             dt,
+            N,
             None,
             enable_range_of_motion=False,
             free_spherical=True,
@@ -506,7 +519,8 @@ def _draw_saddle(
     config: MotionConfig,
     key_t: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     _: jax.Array,
 ) -> jax.Array:
     @jax.vmap
@@ -516,6 +530,7 @@ def _draw_saddle(
             key_t,
             key_value,
             dt,
+            N,
             None,
             enable_range_of_motion=False,
             free_spherical=False,
@@ -530,11 +545,12 @@ def _draw_p3d_and_cor(
     config: MotionConfig,
     _: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     __: jax.Array,
     cor: bool,
 ) -> jax.Array:
-    pos = jax.vmap(lambda key: _draw_pxyz(config, None, key, dt, None, cor))(
+    pos = jax.vmap(lambda key: _draw_pxyz(config, None, key, dt, N, None, cor))(
         jax.random.split(key_value, 3)
     )
     return pos.T
@@ -544,22 +560,24 @@ def _draw_p3d(
     config: MotionConfig,
     _: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     __: jax.Array,
 ) -> jax.Array:
-    return _draw_p3d_and_cor(config, _, key_value, dt, None, cor=False)
+    return _draw_p3d_and_cor(config, _, key_value, dt, N, None, cor=False)
 
 
 def _draw_cor(
     config: MotionConfig,
     _: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     __: jax.Array,
 ) -> jax.Array:
     key_value1, key_value2 = jax.random.split(key_value)
-    q_free = _draw_free(config, _, key_value1, dt, None)
-    q_p3d = _draw_p3d_and_cor(config, _, key_value2, dt, None, cor=True)
+    q_free = _draw_free(config, _, key_value1, dt, N, None)
+    q_p3d = _draw_p3d_and_cor(config, _, key_value2, dt, N, None, cor=True)
     return jnp.concatenate((q_free, q_p3d), axis=1)
 
 
@@ -567,12 +585,13 @@ def _draw_free(
     config: MotionConfig,
     key_t: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     __: jax.Array,
 ) -> jax.Array:
     key_value1, key_value2 = jax.random.split(key_value)
-    q = _draw_spherical(config, key_t, key_value1, dt, None)
-    pos = _draw_p3d(config, None, key_value2, dt, None)
+    q = _draw_spherical(config, key_t, key_value1, dt, N, None)
+    pos = _draw_p3d(config, None, key_value2, dt, N, None)
     return jnp.concatenate((q, pos), axis=1)
 
 
@@ -580,7 +599,8 @@ def _draw_free_2d(
     config: MotionConfig,
     key_t: jax.random.PRNGKey,
     key_value: jax.random.PRNGKey,
-    dt: float,
+    dt: float | jax.Array,
+    N: int | None,
     __: jax.Array,
 ) -> jax.Array:
     key_value1, key_value2 = jax.random.split(key_value)
@@ -589,16 +609,20 @@ def _draw_free_2d(
         key_t,
         key_value1,
         dt,
+        N,
         None,
         enable_range_of_motion=False,
         free_spherical=True,
     )[:, None]
-    pos_yz = _draw_p3d(config, None, key_value2, dt, None)[:, :2]
+    pos_yz = _draw_p3d(config, None, key_value2, dt, N, None)[:, :2]
     return jnp.concatenate((angle_x, pos_yz), axis=1)
 
 
-def _draw_frozen(config: MotionConfig, _, __, dt: float, ___) -> jax.Array:
-    N = int(config.T / dt)
+def _draw_frozen(
+    config: MotionConfig, _, __, dt: float | jax.Array, N: int | None, ___
+) -> jax.Array:
+    if N is None:
+        N = int(config.T / dt)
     return jnp.zeros((N, 0))
 
 

ring/ml/ml_utils.py

@@ -3,17 +3,16 @@ from functools import partial
 import os
 from pathlib import Path
 import pickle
-import random
 import time
 from typing import Optional, Protocol
 import warnings
 
 import jax
 import numpy as np
-import ring
-from ring.utils import import_lib
 from tree_utils import PyTree
 
+import ring
+from ring.utils import import_lib
 import wandb
 
 # An arbitrarily nested dictionary with Array leaves; Or strings
@@ -231,42 +230,5 @@ def save_model_tf(jax_func, path: str, *input, validate: bool = True):
         )
 
 
-def train_val_split(
-    tps: list[str],
-    bs: int,
-    n_batches_for_val: int = 1,
-    transform_gen=None,
-    tree_transform=None,
-):
-    "Uses `random` module for shuffeling."
-    if transform_gen is None:
-        transform_gen = lambda gen: gen
-
-    len_val = n_batches_for_val * bs
-
-    _, N = ring.RCMG.eager_gen_from_paths(tps, 1)
-    include_samples = list(range(N))
-    random.shuffle(include_samples)
-
-    train_data, val_data = include_samples[:-len_val], include_samples[-len_val:]
-    X_val, y_val = transform_gen(
-        ring.RCMG.eager_gen_from_paths(
-            tps, len_val, val_data, tree_transform=tree_transform
-        )[0]
-    )(jax.random.PRNGKey(420))
-
-    generator = transform_gen(
-        ring.RCMG.eager_gen_from_paths(
-            tps,
-            bs,
-            train_data,
-            load_all_into_memory=True,
-            tree_transform=tree_transform,
-        )[0]
-    )
-
-    return generator, (X_val, y_val)
-
-
 def _unknown_link_names(N: int):
     return [f"link{i}" for i in range(N)]

ring/utils/__init__.py

@@ -16,6 +16,7 @@ from .utils import pickle_load
 from .utils import pickle_save
 from .utils import primes
 from .utils import pytree_deepcopy
+from .utils import replace_elements_w_nans
 from .utils import sys_compare
 from .utils import to_list
 from .utils import tree_equal

ring/utils/utils.py

@@ -1,11 +1,13 @@
 from importlib import import_module as _import_module
 import io
 import pickle
+import random
 from typing import Optional
 
 import jax
 import jax.numpy as jnp
 import numpy as np
+import tree_utils
 
 from ring.base import _Base
 from ring.base import Geometry
@@ -181,3 +183,36 @@ def gcd(a: int, b: int) -> int:
     while b:
         a, b = b, a % b
     return a
+
+
+def replace_elements_w_nans(
+    list_of_data: list[tree_utils.PyTree],
+    include_elements: Optional[list[int]] = None,
+    verbose: bool = False,
+) -> list[tree_utils.PyTree]:
+    if include_elements is None:
+        include_elements = list(range(len(list_of_data)))
+
+    assert min(include_elements) >= 0
+    assert max(include_elements) < len(list_of_data)
+
+    def _is_nan(ele: tree_utils.PyTree, i: int):
+        isnan = np.any(
+            [np.any(np.isnan(arr)) for arr in jax.tree_util.tree_leaves(ele)]
+        )
+        if isnan:
+            if verbose:
+                print(f"Sample with idx={i} is nan. It will be replaced.")
+            return True
+        return False
+
+    list_of_data_nonan = []
+    for i, ele in enumerate(list_of_data):
+        if _is_nan(ele, i):
+            while True:
+                j = random.choice(include_elements)
+                if not _is_nan(list_of_data[j], j):
+                    ele = list_of_data[j]
+                    break
+        list_of_data_nonan.append(ele)
+    return list_of_data_nonan