returnn 1.20250421.5132__tar.gz → 1.20250422.113157__tar.gz

{returnn-1.20250421.5132/returnn.egg-info → returnn-1.20250422.113157}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250421.5132
+Version: 1.20250422.113157
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250422.113157/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250422.113157'
+long_version = '1.20250422.113157+git.731b3f5'

{returnn-1.20250421.5132 → returnn-1.20250422.113157}/returnn/datasets/basic.py

@@ -1116,7 +1116,9 @@ class Dataset:
 
     def serialize_data(self, key: str, data: numpy.ndarray) -> str:
         """
-        In case you have a :class:`Vocabulary`, just use :func:`Vocabulary.get_seq_labels`.
+        This is deprecated, as this is slow!
+        In case you have a :class:`Vocabulary`, just use :func:`Vocabulary.get_seq_labels`
+        or :func:`Vocabulary.serialize_labels`.
 
         :param key: e.g. "classes". self.labels[key] should be set
         :param numpy.ndarray data: 0D or 1D

{returnn-1.20250421.5132 → returnn-1.20250422.113157}/returnn/datasets/util/vocabulary.py

@@ -339,6 +339,26 @@ class Vocabulary:
         labels = self.labels
         return " ".join(map(labels.__getitem__, seq))
 
+    def serialize_labels(self, data: numpy.ndarray) -> str:
+        """
+        Like :func:`get_seq_labels` but a bit more generic, to not just work on sequences,
+        but any shape.
+
+        Also like :func:`Dataset.serialize_data` but even slightly more generic.
+        """
+        if data.ndim == 0:
+            return self.id_to_label(data.item())
+        if data.ndim == 1:
+            return self.get_seq_labels(data)
+
+        def _s(d_: numpy.ndarray) -> str:
+            assert d_.ndim >= 1
+            if d_.ndim == 1:
+                return ",".join(self._labels[i] for i in d_)
+            return ",".join(f"[{_s(d_[t])}]" for t in range(d_.shape[0]))
+
+        return _s(data)
+
 
 class BytePairEncoding(Vocabulary):
     """

{returnn-1.20250421.5132 → returnn-1.20250422.113157}/returnn/util/basic.py

@@ -3153,42 +3153,53 @@ class LockFile:
         Acquires the lock.
         """
         import time
-        import errno
 
         wait_count = 0
         while True:
-            # Try to create directory if it does not exist.
-            try:
-                os.makedirs(self.directory)
-            except OSError as exc:
-                # Possible errors:
-                # ENOENT (No such file or directory), e.g. if some parent directory was deleted.
-                # EEXIST (File exists), if the dir already exists.
-                if exc.errno not in [errno.ENOENT, errno.EEXIST]:
-                    # Other error, so reraise.
-                    # Common ones are e.g.:
-                    # ENOSPC (No space left on device)
-                    # EACCES (Permission denied)
-                    raise
-                # Ignore those errors.
+            if self.try_lock():
+                break
+            # We did not get the lock. Wait a bit, and then retry.
+            time.sleep(min(wait_count * 0.1, 1.0))
+            wait_count += 1
+            if wait_count == 10:
+                print("Waiting for lock-file: %s" % self.lockfile)
+
+    def try_lock(self) -> bool:
+        """
+        Tries to acquire the lock.
+
+        :return: whether the lock was acquired
+        """
+
+        import errno
+
+        # Try to create directory if it does not exist.
+        try:
+            os.makedirs(self.directory)
+        except OSError as exc:
+            # Possible errors:
+            # ENOENT (No such file or directory), e.g. if some parent directory was deleted.
+            # EEXIST (File exists), if the dir already exists.
+            if exc.errno not in [errno.ENOENT, errno.EEXIST]:
+                # Other error, so reraise.
+                # Common ones are e.g.:
+                # ENOSPC (No space left on device)
+                # EACCES (Permission denied)
+                raise
+            # Ignore those errors.
+
+        for _ in range(2):
             # Now try to create the lock.
             try:
                 self.fd = os.open(self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
-                return
+                return True
             except OSError as exc:
-                # Possible errors:
-                # ENOENT (No such file or directory), e.g. if the directory was deleted.
-                # EEXIST (File exists), if the lock already exists.
                 if exc.errno not in [errno.ENOENT, errno.EEXIST]:
-                    raise  # Other error, so reraise.
-            # We did not get the lock.
-            # Check if it is a really old one.
-            self.maybe_remove_old_lockfile()
-            # Wait a bit, and then retry.
-            time.sleep(1)
-            wait_count += 1
-            if wait_count == 10:
-                print("Waiting for lock-file: %s" % self.lockfile)
+                    raise  # raise any other error
+                # We did not get the lock.
+                # Remove potential stale lockfile before retrying.
+                self.maybe_remove_old_lockfile()
+        return False
 
     def unlock(self):
         """

{returnn-1.20250421.5132 → returnn-1.20250422.113157}/returnn/util/file_cache.py

@@ -126,11 +126,8 @@ class FileCache:
                     raise e
         if last_error is not None:
             raise last_error
-        info_file = self._get_info_filename(dst_filename)
-        os.utime(dst_filename, None)
-        os.utime(info_file, None)
         # protect info file from tempreaper, which looks at the mtime
-        self._touch_files_thread.files_extend([dst_filename, info_file])
+        self._touch_files_thread.files_extend([dst_filename, self._get_info_filename(dst_filename)])
         return dst_filename
 
     def release_files(self, filenames: Union[str, Iterable[str]]):
@@ -181,6 +178,9 @@ class FileCache:
                 elif self._is_info_filename(fn):
                     # skip keepalive files, they are processed together with the file they guard
                     continue
+                elif self._is_lock_filename(fn):
+                    # skip lock files, removing them would accidentally release locks
+                    continue
                 try:
                     f_stat = os.stat(fn)
                 except Exception as exc:
@@ -203,45 +203,85 @@ class FileCache:
         for mtime, neg_size, fn in all_files:
             size = -neg_size
             delete_reason = None
-            if cur_time - mtime > self._cleanup_files_always_older_than_days * 60 * 60 * 24:
-                delete_reason = f"File is {(cur_time - mtime) / 60 / 60 / 24:.1f} days old"
-            else:
-                reached_more_recent_files = True
-            if not delete_reason and need_at_least_free_space_size > cur_expected_free:
-                # Still must delete some files.
-                if cur_time - mtime > cur_used_time_threshold:
-                    delete_reason = f"Still need more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
-                else:
-                    raise Exception(
-                        f"We cannot free enough space on {self.cache_directory}.\n"
-                        f"Needed: {human_bytes_size(need_at_least_free_space_size)},\n"
-                        f"currently available: {human_bytes_size(cur_expected_free)},\n"
-                        f"oldest file is still too recent: {fn}.\n"
-                        f"{report_size_str}"
-                    )
-            if not delete_reason and want_free_space_size > cur_expected_free:
-                if cur_time - mtime > self._cleanup_files_wanted_older_than_days * 60 * 60 * 24:
-                    delete_reason = f"Still want more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
-                else:
-                    # All further files are even more recent, so we would neither cleanup them,
-                    # so we can also just stop now.
-                    break
 
-            if delete_reason:
-                cur_expected_free += size
-                print(
-                    f"FileCache: Delete file {fn}, size {human_bytes_size(size)}. {delete_reason}."
-                    f" After deletion, have {human_bytes_size(cur_expected_free)} free space."
-                )
-                try:
-                    os.remove(fn)
-                except Exception as exc:
-                    print(f"FileCache: Error while removing {fn}: {type(exc).__name__}: {exc}")
-                    cur_expected_free -= size
+            lock_dir, lock_file_name = self._get_lock_filename(fn)
+            lock_file = LockFile(directory=lock_dir, name=lock_file_name, lock_timeout=self._lock_timeout)
+            if not lock_file.try_lock():
+                print(f"FileCache: lock for {fn} is currently held, skipping.")
+                continue
+            try:
+                # Re-check mtime with lock, could have been updated by another
+                # process in the meantime.
+                # We do not update the `mtime` variable here, because the code
+                # assumes that the list of files is sorted by mtime to abort
+                # early when enough space has been made.
+                # Instead, we treat the case where the mtime was updated during
+                # cleanup as an outlier and continue as if no other mtimes had
+                # changed.
+                # See for discussion:
+                #   - https://github.com/rwth-i6/returnn/issues/1675
+                #   - https://github.com/rwth-i6/returnn/pull/1709
                 try:
-                    os.remove(self._get_info_filename(fn))
+                    cur_mtime = os.stat(fn).st_mtime
+                except FileNotFoundError:
+                    # File was deleted while waiting for the lock, or because it was
+                    # a temporary copy file and was renamed to its final location.
+                    # Since we don't know whether it was actually deleted or just
+                    # renamed, we leave cur_expected_free unchanged.
+                    continue
                 except Exception as exc:
-                    print(f"FileCache: Ignoring error file removing info file of {fn}: {type(exc).__name__}: {exc}")
+                    print(f"FileCache: Error refreshing mtime of {fn}: {type(exc).__name__}: {exc}")
+                    continue
+                if cur_mtime > mtime and (time.time() - cur_mtime) <= cur_used_time_threshold:
+                    print(f"FileCache: {fn} has been updated during cleanup, skipping.")
+                    continue
+                if cur_time - mtime > self._cleanup_files_always_older_than_days * 60 * 60 * 24:
+                    delete_reason = f"File is {(cur_time - mtime) / 60 / 60 / 24:.1f} days old"
+                else:
+                    reached_more_recent_files = True
+                if not delete_reason and need_at_least_free_space_size > cur_expected_free:
+                    # Still must delete some files.
+                    if cur_time - mtime > cur_used_time_threshold:
+                        delete_reason = f"Still need more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
+                    else:
+                        raise Exception(
+                            f"We cannot free enough space on {self.cache_directory}.\n"
+                            f"Needed: {human_bytes_size(need_at_least_free_space_size)},\n"
+                            f"currently available: {human_bytes_size(cur_expected_free)},\n"
+                            f"oldest file is still too recent: {fn}.\n"
+                            f"{report_size_str}"
+                        )
+                if not delete_reason and want_free_space_size > cur_expected_free:
+                    if cur_time - mtime > self._cleanup_files_wanted_older_than_days * 60 * 60 * 24:
+                        delete_reason = f"Still want more space, file is {(cur_time - mtime) / 60 / 60:.1f} hours old"
+                    else:
+                        # All further files are even more recent, so we would neither cleanup them,
+                        # so we can also just stop now.
+                        break
+
+                if delete_reason:
+                    cur_expected_free += size
+                    print(
+                        f"FileCache: Delete file {fn}, size {human_bytes_size(size)}. {delete_reason}."
+                        f" After deletion, have {human_bytes_size(cur_expected_free)} free space."
+                    )
+                    try:
+                        os.remove(fn)
+                    except Exception as exc:
+                        if not isinstance(exc, FileNotFoundError):
+                            print(f"FileCache: Error while removing {fn}: {type(exc).__name__}: {exc}")
+
+                        # We don't know whether the file was just renamed or actually deleted, so
+                        # we do as if its space has not been freed.
+                        cur_expected_free -= size
+                    try:
+                        os.remove(self._get_info_filename(fn))
+                    except FileNotFoundError:
+                        pass
+                    except Exception as exc:
+                        print(f"FileCache: Ignoring error file removing info file of {fn}: {type(exc).__name__}: {exc}")
+            finally:
+                lock_file.unlock()
 
             if reached_more_recent_files and want_free_space_size <= cur_expected_free:
                 # Have enough free space now.
@@ -311,6 +351,16 @@ class FileCache:
         """:return: the name of the corresponding info file to `filename`."""
         return f"{filename}.returnn-info"
 
+    @staticmethod
+    def _get_lock_filename(filename: str) -> Tuple[str, str]:
+        """:return: lock file target directory and lock file name"""
+        return os.path.dirname(filename), os.path.basename(filename) + ".returnn-lock"
+
+    @staticmethod
+    def _is_lock_filename(filename: str) -> bool:
+        """:return: whether `filename` points to a lock file."""
+        return filename.endswith(".returnn-lock")
+
     @staticmethod
     def _is_info_filename(filename: str) -> bool:
         """:return: whether `filename` points to a info file."""
@@ -324,13 +374,22 @@ class FileCache:
         dst_dir = os.path.dirname(dst_filename)
         os.makedirs(dst_dir, exist_ok=True)
 
+        lock_dir, lock_file = self._get_lock_filename(dst_filename)
+        info_file_name = self._get_info_filename(dst_filename)
+
         # Copy the file, while holding a lock. See comment on lock_timeout above.
         with LockFile(
-            directory=dst_dir, name=os.path.basename(dst_filename) + ".lock", lock_timeout=self._lock_timeout
+            directory=lock_dir, name=lock_file, lock_timeout=self._lock_timeout
         ) as lock, self._touch_files_thread.files_added_context(lock.lockfile):
             # Maybe it was copied in the meantime, while waiting for the lock.
             if self._check_existing_copied_file_maybe_cleanup(src_filename, dst_filename):
                 print(f"FileCache: using existing file {dst_filename}")
+                # Update mtime while holding lock, to synchronize with any concurrent cleanup.
+                # See for discussion:
+                #   - https://github.com/rwth-i6/returnn/issues/1675
+                #   - https://github.com/rwth-i6/returnn/pull/1709
+                os.utime(dst_filename, None)
+                os.utime(info_file_name, None)
                 return
 
             print(f"FileCache: Copy file {src_filename} to cache")
@@ -352,7 +411,7 @@ class FileCache:
             with self._touch_files_thread.files_added_context(dst_dir):
                 # save mtime before the copy process to have it pessimistic
                 orig_mtime_ns = os.stat(src_filename).st_mtime_ns
-                FileInfo(mtime_ns=orig_mtime_ns).save(self._get_info_filename(dst_filename))
+                FileInfo(mtime_ns=orig_mtime_ns).save(info_file_name)
 
                 _copy_with_prealloc(src_filename, dst_tmp_filename)
                 os.rename(dst_tmp_filename, dst_filename)

{returnn-1.20250421.5132 → returnn-1.20250422.113157/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250421.5132
+Version: 1.20250422.113157
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250421.5132 → returnn-1.20250422.113157}/tools/dump-dataset.py

@@ -18,6 +18,7 @@ from returnn.log import log
 import argparse
 import numpy
 from returnn.datasets import init_dataset, Dataset
+from returnn.datasets.util.vocabulary import Vocabulary
 from returnn.util.basic import Stats, hms, hms_fraction, pretty_print, NumbersDict
 from returnn.util import basic as util
 
@@ -87,6 +88,13 @@ def dump_dataset(options):
         print("Done.")
         return
 
+    # Inlined and cached can_serialize_data / serialize_data.
+    vocabs = {}
+    for key in dataset.get_data_keys():
+        labels = dataset.labels.get(key)
+        if labels and len(labels) > 1:
+            vocabs[key] = Vocabulary.create_vocab_from_labels(labels)
+
     dump_file = None
     if options.type == "numpy":
         print("Dump files: %r*%r" % (options.dump_prefix, options.dump_postfix), file=log.v3)
@@ -192,9 +200,8 @@ def dump_dataset(options):
             elif options.type == "stdout":
                 print("seq %s tag:" % progress, dataset.get_tag(seq_idx))
                 extra = ""
-                if "data" in dataset.labels and len(dataset.labels["data"]) > 1:
-                    assert dataset.can_serialize_data("data")
-                    extra += " (%r)" % dataset.serialize_data(key="data", data=data)
+                if "data" in vocabs:
+                    extra += " (%r)" % vocabs["data"].serialize_labels(data)
                 print("seq %s data: %s%s" % (progress, pretty_print(data), extra))
             elif options.type == "print_shape":
                 print("seq %s data shape:" % progress, data.shape)
@@ -210,9 +217,8 @@ def dump_dataset(options):
                     )
                 elif options.type == "stdout":
                     extra = ""
-                    if target in dataset.labels and len(dataset.labels[target]) > 1:
-                        assert dataset.can_serialize_data(target)
-                        extra += " (%r)" % dataset.serialize_data(key=target, data=targets)
+                    if target in vocabs:
+                        extra += " (%r)" % vocabs[target].serialize_labels(targets)
                     print("seq %i target %r: %s%s" % (seq_idx, target, pretty_print(targets), extra))
                 elif options.type == "print_shape":
                     print("seq %i target %r shape:" % (seq_idx, target), targets.shape)

returnn-1.20250421.5132/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250421.005132'
-long_version = '1.20250421.005132+git.c90b436'