returnn 1.20250421.5132__py3-none-any.whl → 1.20250423.105638__py3-none-any.whl

returnn/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250421.5132
+Version: 1.20250423.105638
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn/_setup_info_generated.py

@@ -1,2 +1,2 @@
-version = '1.20250421.005132'
-long_version = '1.20250421.005132+git.c90b436'
+version = '1.20250423.105638'
+long_version = '1.20250423.105638+git.767de47'

returnn/datasets/basic.py

@@ -1116,7 +1116,9 @@ class Dataset:
 
     def serialize_data(self, key: str, data: numpy.ndarray) -> str:
         """
-        In case you have a :class:`Vocabulary`, just use :func:`Vocabulary.get_seq_labels`.
+        This is deprecated, as this is slow!
+        In case you have a :class:`Vocabulary`, just use :func:`Vocabulary.get_seq_labels`
+        or :func:`Vocabulary.serialize_labels`.
 
         :param key: e.g. "classes". self.labels[key] should be set
         :param numpy.ndarray data: 0D or 1D

returnn/datasets/util/vocabulary.py

@@ -339,6 +339,26 @@ class Vocabulary:
         labels = self.labels
         return " ".join(map(labels.__getitem__, seq))
 
+    def serialize_labels(self, data: numpy.ndarray) -> str:
+        """
+        Like :func:`get_seq_labels` but a bit more generic, to not just work on sequences,
+        but any shape.
+
+        Also like :func:`Dataset.serialize_data` but even slightly more generic.
+        """
+        if data.ndim == 0:
+            return self.id_to_label(data.item())
+        if data.ndim == 1:
+            return self.get_seq_labels(data)
+
+        def _s(d_: numpy.ndarray) -> str:
+            assert d_.ndim >= 1
+            if d_.ndim == 1:
+                return ",".join(self._labels[i] for i in d_)
+            return ",".join(f"[{_s(d_[t])}]" for t in range(d_.shape[0]))
+
+        return _s(data)
+
 
 class BytePairEncoding(Vocabulary):
     """

returnn/torch/engine.py

@@ -82,15 +82,17 @@ class Engine(EngineBase):
         )  # type: Union[int,float,Dict[str,int],NumbersDict]
         self._orig_model = None  # type: Optional[Union[rf.Module, torch.nn.Module]]
         self._pt_model = None  # type: Optional[torch.nn.Module]
-        self._train_step_func = None  # type: Optional[Callable]
-        self._forward_step_func = self.config.typed_value("forward_step")  # type: Optional[Callable]
-        self._forward_step_expected_outputs = None  # type: Optional[TensorDict]
+        self._epoch_start_func: Optional[Callable] = self.config.typed_value("epoch_start")
+        self._epoch_end_func: Optional[Callable] = self.config.typed_value("epoch_end")
+        self._train_step_func: Optional[Callable] = None
+        self._forward_step_func: Optional[Callable] = self.config.typed_value("forward_step")
+        self._forward_step_expected_outputs: Optional[TensorDict] = None
         if self.config.typed_value("model_outputs") is not None:
             self._forward_step_expected_outputs = TensorDict()
             self._forward_step_expected_outputs.update(self.config.typed_value("model_outputs"), auto_convert=True)
         self._save_model_epoch_interval = 1
         self._ignore_param_set: Set[str] = set()  # for the updater and for saving the model checkpoint
-        self._updater = None  # type: Optional[Updater]
+        self._updater: Optional[Updater] = None
 
         self._use_autocast = False
         self._autocast_dtype = None  # type: Optional[str]
@@ -319,6 +321,26 @@ class Engine(EngineBase):
             ]
             print(f"Memory usage ({self._device}):", " ".join(stats), file=log.v1)
 
+    def _on_epoch_start(self, *, dataset_name: str):
+        if self._epoch_start_func:
+            self._epoch_start_func(
+                epoch=self.epoch,
+                step=self.global_train_step,
+                model=self._orig_model,
+                dataset_name=dataset_name,
+                **util.get_fwd_compat_kwargs(),
+            )
+
+    def _on_epoch_end(self, *, dataset_name: str):
+        if self._epoch_end_func:
+            self._epoch_end_func(
+                epoch=self.epoch,
+                step=self.global_train_step,
+                model=self._orig_model,
+                dataset_name=dataset_name,
+                **util.get_fwd_compat_kwargs(),
+            )
+
     def train_epoch(self):
         """
         train one (sub)epoch
@@ -346,6 +368,8 @@ class Engine(EngineBase):
         self._maybe_reset_dev_memory_caches()
         self._reset_dev_memory_stats()
 
+        self._on_epoch_start(dataset_name="train")
+
         if self.config.bool("debug_shell_before_train_loop", False):
             print("debug_shell_before_train_loop", file=log.v1)
             debug_shell(user_ns=locals(), user_global_ns=globals(), exit_afterwards=False)
@@ -564,6 +588,8 @@ class Engine(EngineBase):
 
         self._maybe_report_dev_memory_stats()
 
+        self._on_epoch_end(dataset_name="train")
+
         if self.epoch % self._save_model_epoch_interval == 0 or self.epoch == self._final_epoch:
             if self.model_filename:
                 self._save_model()
@@ -612,6 +638,8 @@ class Engine(EngineBase):
 
             print(f"Evaluating dataset {dataset_name!r}", file=log.v3)
 
+            self._on_epoch_start(dataset_name=dataset_name)
+
             accumulated_losses_dict = NumbersDict()
             accumulated_inv_norm_factors_dict = NumbersDict()
             step_idx = 0
@@ -685,6 +713,8 @@ class Engine(EngineBase):
                 _has_data = torch.tensor([False], device="cpu", dtype=torch.int8)
                 torch.distributed.broadcast(_has_data, src=0)
 
+            self._on_epoch_end(dataset_name=dataset_name)
+
         if not self._torch_distributed_ctx or self._torch_distributed_ctx.rank() == 0:
             print(
                 f"Epoch {self.epoch} evaluation:",

returnn/util/basic.py

@@ -3153,42 +3153,53 @@ class LockFile:
         Acquires the lock.
         """
         import time
-        import errno
 
         wait_count = 0
         while True:
-            # Try to create directory if it does not exist.
-            try:
-                os.makedirs(self.directory)
-            except OSError as exc:
-                # Possible errors:
-                # ENOENT (No such file or directory), e.g. if some parent directory was deleted.
-                # EEXIST (File exists), if the dir already exists.
-                if exc.errno not in [errno.ENOENT, errno.EEXIST]:
-                    # Other error, so reraise.
-                    # Common ones are e.g.:
-                    # ENOSPC (No space left on device)
-                    # EACCES (Permission denied)
-                    raise
-                # Ignore those errors.
+            if self.try_lock():
+                break
+            # We did not get the lock. Wait a bit, and then retry.
+            time.sleep(min(wait_count * 0.1, 1.0))
+            wait_count += 1
+            if wait_count == 10:
+                print("Waiting for lock-file: %s" % self.lockfile)
+
+    def try_lock(self) -> bool:
+        """
+        Tries to acquire the lock.
+
+        :return: whether the lock was acquired
+        """
+
+        import errno
+
+        # Try to create directory if it does not exist.
+        try:
+            os.makedirs(self.directory)
+        except OSError as exc:
+            # Possible errors:
+            # ENOENT (No such file or directory), e.g. if some parent directory was deleted.
+            # EEXIST (File exists), if the dir already exists.
+            if exc.errno not in [errno.ENOENT, errno.EEXIST]:
+                # Other error, so reraise.
+                # Common ones are e.g.:
+                # ENOSPC (No space left on device)
+                # EACCES (Permission denied)
+                raise
+            # Ignore those errors.
+
+        for _ in range(2):
             # Now try to create the lock.
             try:
                 self.fd = os.open(self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
-                return
+                return True
             except OSError as exc:
-                # Possible errors:
-                # ENOENT (No such file or directory), e.g. if the directory was deleted.
-                # EEXIST (File exists), if the lock already exists.
                 if exc.errno not in [errno.ENOENT, errno.EEXIST]:
-                    raise  # Other error, so reraise.
-            # We did not get the lock.
-            # Check if it is a really old one.
-            self.maybe_remove_old_lockfile()
-            # Wait a bit, and then retry.
-            time.sleep(1)
-            wait_count += 1
-            if wait_count == 10:
-                print("Waiting for lock-file: %s" % self.lockfile)
+                    raise  # raise any other error
+                # We did not get the lock.
+                # Remove potential stale lockfile before retrying.
+                self.maybe_remove_old_lockfile()
+        return False
 
     def unlock(self):
         """

returnn/util/file_cache.py

@@ -126,11 +126,8 @@ class FileCache:
                     raise e
         if last_error is not None:
             raise last_error
-        info_file = self._get_info_filename(dst_filename)
-        os.utime(dst_filename, None)
-        os.utime(info_file, None)
         # protect info file from tempreaper, which looks at the mtime
-        self._touch_files_thread.files_extend([dst_filename, info_file])
+        self._touch_files_thread.files_extend([dst_filename, self._get_info_filename(dst_filename)])
         return dst_filename
 
     def release_files(self, filenames: Union[str, Iterable[str]]):
@@ -181,6 +178,9 @@ class FileCache:
                 elif self._is_info_filename(fn):
                     # skip keepalive files, they are processed together with the file they guard
                     continue
+                elif self._is_lock_filename(fn):
+                    # skip lock files, removing them would accidentally release locks
+                    continue
                 try:
                     f_stat = os.stat(fn)
                 except Exception as exc:
@@ -203,45 +203,85 @@ class FileCache:
         for mtime, neg_size, fn in all_files:
             size = -neg_size
             delete_reason = None
-            if cur_time - mtime > self._cleanup_files_always_older_than_days * 60 * 60 * 24:
-                delete_reason = f"File is {(cur_time - mtime) / 60 / 60 / 24:.1f} days old"
-            else:
-                reached_more_recent_files = True
-            if not delete_reason and need_at_least_free_space_size > cur_expected_free:
-                # Still must delete some files.
-                if cur_time - mtime > cur_used_time_threshold:
-                    delete_reason = f"Still need more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
-                else:
-                    raise Exception(
-                        f"We cannot free enough space on {self.cache_directory}.\n"
-                        f"Needed: {human_bytes_size(need_at_least_free_space_size)},\n"
-                        f"currently available: {human_bytes_size(cur_expected_free)},\n"
-                        f"oldest file is still too recent: {fn}.\n"
-                        f"{report_size_str}"
-                    )
-            if not delete_reason and want_free_space_size > cur_expected_free:
-                if cur_time - mtime > self._cleanup_files_wanted_older_than_days * 60 * 60 * 24:
-                    delete_reason = f"Still want more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
-                else:
-                    # All further files are even more recent, so we would neither cleanup them,
-                    # so we can also just stop now.
-                    break
 
-            if delete_reason:
-                cur_expected_free += size
-                print(
-                    f"FileCache: Delete file {fn}, size {human_bytes_size(size)}. {delete_reason}."
-                    f" After deletion, have {human_bytes_size(cur_expected_free)} free space."
-                )
-                try:
-                    os.remove(fn)
-                except Exception as exc:
-                    print(f"FileCache: Error while removing {fn}: {type(exc).__name__}: {exc}")
-                    cur_expected_free -= size
+            lock_dir, lock_file_name = self._get_lock_filename(fn)
+            lock_file = LockFile(directory=lock_dir, name=lock_file_name, lock_timeout=self._lock_timeout)
+            if not lock_file.try_lock():
+                print(f"FileCache: lock for {fn} is currently held, skipping.")
+                continue
+            try:
+                # Re-check mtime with lock, could have been updated by another
+                # process in the meantime.
+                # We do not update the `mtime` variable here, because the code
+                # assumes that the list of files is sorted by mtime to abort
+                # early when enough space has been made.
+                # Instead, we treat the case where the mtime was updated during
+                # cleanup as an outlier and continue as if no other mtimes had
+                # changed.
+                # See for discussion:
+                #   - https://github.com/rwth-i6/returnn/issues/1675
+                #   - https://github.com/rwth-i6/returnn/pull/1709
                 try:
-                    os.remove(self._get_info_filename(fn))
+                    cur_mtime = os.stat(fn).st_mtime
+                except FileNotFoundError:
+                    # File was deleted while waiting for the lock, or because it was
+                    # a temporary copy file and was renamed to its final location.
+                    # Since we don't know whether it was actually deleted or just
+                    # renamed, we leave cur_expected_free unchanged.
+                    continue
                 except Exception as exc:
-                    print(f"FileCache: Ignoring error file removing info file of {fn}: {type(exc).__name__}: {exc}")
+                    print(f"FileCache: Error refreshing mtime of {fn}: {type(exc).__name__}: {exc}")
+                    continue
+                if cur_mtime > mtime and (time.time() - cur_mtime) <= cur_used_time_threshold:
+                    print(f"FileCache: {fn} has been updated during cleanup, skipping.")
+                    continue
+                if cur_time - mtime > self._cleanup_files_always_older_than_days * 60 * 60 * 24:
+                    delete_reason = f"File is {(cur_time - mtime) / 60 / 60 / 24:.1f} days old"
+                else:
+                    reached_more_recent_files = True
+                if not delete_reason and need_at_least_free_space_size > cur_expected_free:
+                    # Still must delete some files.
+                    if cur_time - mtime > cur_used_time_threshold:
+                        delete_reason = f"Still need more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
+                    else:
+                        raise Exception(
+                            f"We cannot free enough space on {self.cache_directory}.\n"
+                            f"Needed: {human_bytes_size(need_at_least_free_space_size)},\n"
+                            f"currently available: {human_bytes_size(cur_expected_free)},\n"
+                            f"oldest file is still too recent: {fn}.\n"
+                            f"{report_size_str}"
+                        )
+                if not delete_reason and want_free_space_size > cur_expected_free:
+                    if cur_time - mtime > self._cleanup_files_wanted_older_than_days * 60 * 60 * 24:
+                        delete_reason = f"Still want more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
+                    else:
+                        # All further files are even more recent, so we would neither cleanup them,
+                        # so we can also just stop now.
+                        break
+
+                if delete_reason:
+                    cur_expected_free += size
+                    print(
+                        f"FileCache: Delete file {fn}, size {human_bytes_size(size)}. {delete_reason}."
+                        f" After deletion, have {human_bytes_size(cur_expected_free)} free space."
+                    )
+                    try:
+                        os.remove(fn)
+                    except Exception as exc:
+                        if not isinstance(exc, FileNotFoundError):
+                            print(f"FileCache: Error while removing {fn}: {type(exc).__name__}: {exc}")
+
+                        # We don't know whether the file was just renamed or actually deleted, so
+                        # we do as if its space has not been freed.
+                        cur_expected_free -= size
+                    try:
+                        os.remove(self._get_info_filename(fn))
+                    except FileNotFoundError:
+                        pass
+                    except Exception as exc:
+                        print(f"FileCache: Ignoring error file removing info file of {fn}: {type(exc).__name__}: {exc}")
+            finally:
+                lock_file.unlock()
 
             if reached_more_recent_files and want_free_space_size <= cur_expected_free:
                 # Have enough free space now.
@@ -311,6 +351,16 @@ class FileCache:
         """:return: the name of the corresponding info file to `filename`."""
         return f"{filename}.returnn-info"
 
+    @staticmethod
+    def _get_lock_filename(filename: str) -> Tuple[str, str]:
+        """:return: lock file target directory and lock file name"""
+        return os.path.dirname(filename), os.path.basename(filename) + ".returnn-lock"
+
+    @staticmethod
+    def _is_lock_filename(filename: str) -> bool:
+        """:return: whether `filename` points to a lock file."""
+        return filename.endswith(".returnn-lock")
+
     @staticmethod
     def _is_info_filename(filename: str) -> bool:
         """:return: whether `filename` points to a info file."""
@@ -324,13 +374,22 @@ class FileCache:
         dst_dir = os.path.dirname(dst_filename)
         os.makedirs(dst_dir, exist_ok=True)
 
+        lock_dir, lock_file = self._get_lock_filename(dst_filename)
+        info_file_name = self._get_info_filename(dst_filename)
+
         # Copy the file, while holding a lock. See comment on lock_timeout above.
         with LockFile(
-            directory=dst_dir, name=os.path.basename(dst_filename) + ".lock", lock_timeout=self._lock_timeout
+            directory=lock_dir, name=lock_file, lock_timeout=self._lock_timeout
         ) as lock, self._touch_files_thread.files_added_context(lock.lockfile):
             # Maybe it was copied in the meantime, while waiting for the lock.
             if self._check_existing_copied_file_maybe_cleanup(src_filename, dst_filename):
                 print(f"FileCache: using existing file {dst_filename}")
+                # Update mtime while holding lock, to synchronize with any concurrent cleanup.
+                # See for discussion:
+                #   - https://github.com/rwth-i6/returnn/issues/1675
+                #   - https://github.com/rwth-i6/returnn/pull/1709
+                os.utime(dst_filename, None)
+                os.utime(info_file_name, None)
                 return
 
             print(f"FileCache: Copy file {src_filename} to cache")
@@ -352,7 +411,7 @@ class FileCache:
             with self._touch_files_thread.files_added_context(dst_dir):
                 # save mtime before the copy process to have it pessimistic
                 orig_mtime_ns = os.stat(src_filename).st_mtime_ns
-                FileInfo(mtime_ns=orig_mtime_ns).save(self._get_info_filename(dst_filename))
+                FileInfo(mtime_ns=orig_mtime_ns).save(info_file_name)
 
                 _copy_with_prealloc(src_filename, dst_tmp_filename)
                 os.rename(dst_tmp_filename, dst_filename)

{returnn-1.20250421.5132.dist-info → returnn-1.20250423.105638.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250421.5132
+Version: 1.20250423.105638
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250421.5132.dist-info → returnn-1.20250423.105638.dist-info}/RECORD

@@ -1,9 +1,9 @@
-returnn/PKG-INFO,sha256=OSP8W77Tq_qY5jAtKqcRmNhqW8OlDWCAWtSb7SlcCzU,5213
+returnn/PKG-INFO,sha256=7FZF3sif_hKT_9vHgl4snASut9Lc4uPAOGJfqaOxr_A,5215
 returnn/__init__.py,sha256=biBtRsM0WZ406vShaeH-9WFoqJ8XwTbn6g0EeFJ7l8E,1012
 returnn/__main__.py,sha256=qBFbuB1yN3adgVM5pXt2-Yq9vorjRNchNPL8kDKx44M,31752
 returnn/__old_mod_loader__.py,sha256=nvsNY-xELdS_IPNkv66Q9Rmvg4dbGW0-EBRDcCmctos,7654
 returnn/__setup__.py,sha256=22kQn2fh11iPM0hLb2Fy5sLmoU1JGvmDxXRYuRgQkwU,4659
-returnn/_setup_info_generated.py,sha256=H-jY5CU-_qEI26JD-eey6Pdpp-4t8u0wTnj2gakdhe8,77
+returnn/_setup_info_generated.py,sha256=_2mdcTTEJ__VlmLagOoes0sVhrZN1SeK64wJQZ4crrY,77
 returnn/config.py,sha256=3tmKhB6FnQZaNdtcYsiB61JnEY--iZ2qmJ4yq0b6tE0,29140
 returnn/forward_iface.py,sha256=A_OJiaXsX4MlXQRzST86ylyxSUZbC402PQL1REcqHjM,911
 returnn/learning_rate_control.py,sha256=ZvWryAn_tv9DhV8sh1LV3eE34Yltl3On3mYZAG4hR9s,34684
@@ -13,7 +13,7 @@ returnn/native_op.py,sha256=yqpE7SqBqXq77FCVnWMloUwadWlslEk-VzdK7FMpt_U,244411
 returnn/pretrain.py,sha256=MHiXJZqkQFmDVyaYsGpd_Acv20wxl7Pr6s6qJzAT2FI,22648
 returnn/datasets/__init__.py,sha256=PvDlfDOaaopIeUIt0OSvHD2eHZkdkyE-sjMXf35EH5U,390
 returnn/datasets/audio.py,sha256=Gmj7a08dnvYh7Z-G1TNapz42L50AIcDE9JeIZaO1s1M,23334
-returnn/datasets/basic.py,sha256=BBDXCqesUBJ-4HsaT4zw3wzkS1kd15FDy0OX8TY10pQ,72275
+returnn/datasets/basic.py,sha256=6rMpSuXFCZlqYtXlMW8xb5hu9ZCMlZDgnpmC66TYpQI,72367
 returnn/datasets/bundle_file.py,sha256=KQNrS1MSf-4_idlK0c0KFwON-f5sEK0sWU15WpoMYpE,2380
 returnn/datasets/cached.py,sha256=DIRdWrxBmsZG8O_9eVxBO5mcdo4f5KU-Xb-4wVz59Io,25418
 returnn/datasets/cached2.py,sha256=_6pza3IG68JexaExhj1ld3fP6pE7T-G804driJ9Z_qo,12141
@@ -34,7 +34,7 @@ returnn/datasets/text_dict.py,sha256=BPE73nh6-vtSLy3SiDf4dpFl9RJorE7oO6l5y2FU3MI
 returnn/datasets/util/__init__.py,sha256=rEKhSD6fyhDiQF-x7dUQMwa29JZu72SDm7mYcCcLghY,52
 returnn/datasets/util/feature_extraction.py,sha256=axtXDb9wcNpOmyhmW3WJUj5xda29TKkKvOcGGvq7ExA,23923
 returnn/datasets/util/strings.py,sha256=Xg-Nt2mI5Gi7Eb1bER1bmkZJdQg6QhnMANZOf1IzzJ4,413
-returnn/datasets/util/vocabulary.py,sha256=25bP_jkAbRoJ4A300O2GEG7aZAro4gTKumJ19bMw5Xs,26250
+returnn/datasets/util/vocabulary.py,sha256=hkeuhCt4r6rD5YdICk3_lP0qL7NowK6Mn-oWRjpo3yo,26943
 returnn/engine/__init__.py,sha256=br7hpn8i_hIBi2uTQfnN3BF9g5DREYa_mQi0_Nvlu6o,228
 returnn/engine/base.py,sha256=0n4FtB_B2H3W_9KdoLr0P7YPER-hVkbk69pwFqsqmqw,18467
 returnn/engine/batch.py,sha256=amXW8mGspuSQjo00JdisE2eOLy5Ij1weWWzkE-lXSJM,9912
@@ -207,7 +207,7 @@ returnn/tf/util/open_fst.py,sha256=sZRDw4TbxvhGqpGdUJWy1ebvlZm4_RPhygpRw9uLAOQ,1
 returnn/torch/README.md,sha256=jzJ2FpOHW02vxN69yKaV97C9LI-hmvjBglKfdZXIDdc,85
 returnn/torch/__init__.py,sha256=MHEUyNHB20Vy89uKAqZoj6FxJKF1Gq3HW-i6ra1pNcI,24
 returnn/torch/distributed.py,sha256=skFyutdVztxgTEk3HHJ8S83qRWbNpkNT8Tj16Ic0_hE,6981
-returnn/torch/engine.py,sha256=jJMVR8ltJMfF70-m0HxR4xNS94wNfO4CSAF0jFlB7_g,76771
+returnn/torch/engine.py,sha256=yfqP9jzOH1OjiETqoBh20YOeEaX_kyr_kwPUkhSFxiI,77833
 returnn/torch/updater.py,sha256=GqtBvZpElPVMm0lq84JPl4NVLFFETZAzAbR0rTomSao,28249
 returnn/torch/data/__init__.py,sha256=6cLNEi8KoGI12PF6akN7mI_mtjlx-0hcQAfMYoExwik,132
 returnn/torch/data/extern_data.py,sha256=OSoy3x1KiyiJCr7DfF5uPFAu09We2N2WbA0yo-pYXxM,7601
@@ -233,12 +233,12 @@ returnn/torch/util/gradient_checkpoint.py,sha256=iLy-FB65DC8O6LxzmMvFjnSdpIVpko8
 returnn/torch/util/module.py,sha256=MXHIrF9Isu575DDJIa81212ULKwdqu1oOLxDVZecVSk,1693
 returnn/torch/util/scaled_gradient.py,sha256=3585VuNypBty-pW6r3BKK047H3MqZQSdMjXeYAb4cmU,3192
 returnn/util/__init__.py,sha256=UIG1qw4idqhW71BV60ha7h9PktxvEVcBIu0lYRossK8,336
-returnn/util/basic.py,sha256=F5UZws8FR7-ger8T-pkXV83C_9IlYsUk7H85TujK0Xk,142477
+returnn/util/basic.py,sha256=jwtaaZyOV7fUjKDRXVHDy-K5kwR1mPrkAZrzc5STOvE,142554
 returnn/util/better_exchook.py,sha256=TAtb_ZyM-357UnOg_HMoBZUSxzt0WPgumlvprmlCprA,63921
 returnn/util/bpe.py,sha256=LWFhICZsEOnMwNws0lybPNzKRX6rSr8yKCvP65vjl9Y,19656
 returnn/util/debug.py,sha256=wuRzdg9zB84WWCGyTjmRR_zYypu8gXxlc0nZ6si9OC8,28224
 returnn/util/debug_helpers.py,sha256=0EINLK4uLtoSt5_kHs1M2NIFpMd0S7i4c4rx90U4fJk,2914
-returnn/util/file_cache.py,sha256=JvJ4C7NFr8WpiIN0hLk3c33oX4-JfWSpchTjY7JGpCc,23127
+returnn/util/file_cache.py,sha256=tXM2Leos8fOc1tzmQbGSLYgzQoKAw1klCw-k9TwoAno,26453
 returnn/util/fsa.py,sha256=k2lJ8tyf_g44Xk1EPVLwDwpP4spoMTqIigDVOWocQHY,59177
 returnn/util/literal_py_to_pickle.py,sha256=3dnjWPeeiDT2xp4bRDgIf9yddx7b1AG7mOKEn_jiSl8,2173
 returnn/util/lru_cache.py,sha256=7Q5H3a8b07E8e1iB7PA9jCpRnxMJZOFS2KO07cy0gqk,11446
@@ -253,8 +253,8 @@ returnn/util/sig_proc.py,sha256=Tjz0VOAVyqu2qDCF5HZ1JjALjcFsHcNkcd96WgZeKfE,7265
 returnn/util/task_system.py,sha256=y4sMVXQ25Qd2z0rx03uOlXlkE-jbCYC1Sjfn-XlraVU,26003
 returnn/util/train_proc_manager.py,sha256=Pjht28k6uz6BNQ47uW6Gf880iyq5q4wx7P_K2tmoAM8,3266
 returnn/util/watch_memory.py,sha256=BR5P2kvBN6UI81cE0_1WAA6Hd1SByLbBaiDxvLhPOew,4213
-returnn-1.20250421.5132.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
-returnn-1.20250421.5132.dist-info/METADATA,sha256=OSP8W77Tq_qY5jAtKqcRmNhqW8OlDWCAWtSb7SlcCzU,5213
-returnn-1.20250421.5132.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-returnn-1.20250421.5132.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
-returnn-1.20250421.5132.dist-info/RECORD,,
+returnn-1.20250423.105638.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
+returnn-1.20250423.105638.dist-info/METADATA,sha256=7FZF3sif_hKT_9vHgl4snASut9Lc4uPAOGJfqaOxr_A,5215
+returnn-1.20250423.105638.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+returnn-1.20250423.105638.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
+returnn-1.20250423.105638.dist-info/RECORD,,