returnn 1.20240621.130142__tar.gz → 1.20240621.152144__tar.gz

{returnn-1.20240621.130142 → returnn-1.20240621.152144}/CHANGELOG.md

@@ -25,6 +25,11 @@ It also was specifically designed with distributed training in mind.
 The distributed random_seed_offset method can be used,
 but sharding is also supported ([PR #1538](https://github.com/rwth-i6/returnn/pull/1538)).
 
+## 2023-11-28: [`lovely_tensors`](https://xl0.github.io/lovely-tensors/) support
+
+Set `use_lovely_tensors = True` in the config.
+Should always be safe to use, thus can always be enabled.
+
 ## 2023-11-09: `LearningRateControl` saves more meta info in learning-rate-file
 
 Like effective learning rate (after `dynamic_learning_rate`),
@@ -33,11 +38,15 @@ training step, GPU, RETURNN version, etc.
 
 ## 2024-01-09: [Train proc manager](https://github.com/rwth-i6/returnn/blob/master/returnn/util/train_proc_manager.py)
 
+Set `use_train_proc_manager = True` (currently PyTorch only).
+Should always be safe to use, thus can always be enabled.
 Auto restart RETURNN on crashes under certain conditions
 (e.g. it must have trained at least one epoch successfully since the most recent restart).
 
 ## 2023-12-30: PyTorch handle OOM for forwarding, auto-split batch
 
+Set `forward_auto_split_batch_on_oom = True` in the config (currently PyTorch only).
+Should always be safe to use, thus can always be enabled.
 This went through several iterations of approaches,
 stumbling through a number of CPython and PyTorch bugs,
 e.g. [CPython #113939](https://github.com/python/cpython/issues/113939),
@@ -50,6 +59,9 @@ In the `torch_distributed` config dict: Set `"reduce_type": "param"` and `"param
 
 ## 2023-10-24: [`watch_memory`](https://github.com/rwth-i6/returnn/blob/master/returnn/util/watch_memory.py): watches memory of all procs
 
+Set `watch_memory = True` in the config.
+Should not influence anything, thus can always be enabled.
+
 ## 2023-10-03: RETURNN frontend (RF) native helpers ([PR #1403](https://github.com/rwth-i6/returnn/pull/1403))
 
 ## 2023-06-09: PyTorch distributed training ([PR #1335](https://github.com/rwth-i6/returnn/pull/1335), [issue #1332](https://github.com/rwth-i6/returnn/issues/1332))
@@ -200,6 +212,8 @@ and `delay_sample_update=True`.
 
 ## 2020-11-06: [PyTorch-to-RETURNN project](https://github.com/rwth-i6/pytorch-to-returnn)
 
+## 2020-08-14: GitHub actions CI to replace Travis ([PR #340](https://github.com/rwth-i6/returnn/pull/340), [issue #308](https://github.com/rwth-i6/returnn/issues/308))
+
 ## 2020-08-03: New code structure ([discussion](https://github.com/rwth-i6/returnn/issues/162))
 
 `TFEngine` (or `returnn.TFEngine`) becomes `returnn.tf.engine`, etc.

{returnn-1.20240621.130142 → returnn-1.20240621.152144}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240621.130142
+Version: 1.20240621.152144
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240621.152144/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240621.152144'
+long_version = '1.20240621.152144+git.6db5121'

{returnn-1.20240621.130142 → returnn-1.20240621.152144}/returnn/util/file_cache.py

@@ -55,8 +55,9 @@ class FileCache:
         *,
         cache_directory: str = "$TMPDIR/$USER/returnn/file_cache",
         cleanup_files_always_older_than_days: float = 31.0,
-        cleanup_files_wanted_older_than_days: float = 7.0,
+        cleanup_files_wanted_older_than_days: float = 1.0,
         cleanup_disk_usage_wanted_free_ratio: float = 0.2,  # try to free at least 20% disk space
+        cleanup_disk_usage_wanted_multiplier: float = 2.0,  # try to free 2x the required space for a file
         num_tries: int = 3,  # retry twice by default
     ):
         """
@@ -66,12 +67,16 @@ class FileCache:
         :param cleanup_files_wanted_older_than_days: if cleanup_disk_usage_wanted_free_ratio not reached,
             cleanup files older than this.
         :param cleanup_disk_usage_wanted_free_ratio: try to free at least this ratio of disk space.
+        :param cleanup_disk_usage_wanted_multiplier: when making space for a new file, try to free at
+            least this times as much space.
         :param num_tries: how many times to try caching a file before giving up
         """
         self.cache_directory = expand_env_vars(cache_directory)
         self._cleanup_files_always_older_than_days = cleanup_files_always_older_than_days
         self._cleanup_files_wanted_older_than_days = cleanup_files_wanted_older_than_days
         self._cleanup_disk_usage_wanted_free_ratio = cleanup_disk_usage_wanted_free_ratio
+        assert cleanup_disk_usage_wanted_multiplier >= 1.0
+        self._cleanup_disk_usage_wanted_multiplier = cleanup_disk_usage_wanted_multiplier
         self._touch_files_thread = _TouchFilesThread(cache_base_dir=self.cache_directory)
         self._touch_files_thread.start()
         self._recent_full_cleanup_time = float("-inf")
@@ -143,7 +148,8 @@ class FileCache:
             return
         disk_usage = shutil.disk_usage(self.cache_directory)
         want_free_space_size = max(
-            need_at_least_free_space_size, int(self._cleanup_disk_usage_wanted_free_ratio * disk_usage.total)
+            int(self._cleanup_disk_usage_wanted_multiplier * need_at_least_free_space_size),
+            int(self._cleanup_disk_usage_wanted_free_ratio * disk_usage.total),
         )
         # If we have enough free space, and we did a full cleanup recently, we don't need to do anything.
         if want_free_space_size <= disk_usage.free and time.monotonic() - self._recent_full_cleanup_time < 60 * 10:
@@ -282,24 +288,24 @@ class FileCache:
         """
         Copy the file to the cache directory.
         """
-        if self._check_existing_copied_file_maybe_cleanup(src_filename, dst_filename):
-            os.utime(dst_filename, None)  # touch
-            return
-
         # Create dirs.
         dst_dir = os.path.dirname(dst_filename)
         os.makedirs(dst_dir, exist_ok=True)
 
         # Copy the file, while holding a lock. See comment on lock_timeout above.
-        with LockFile(directory=self.cache_directory, name="dir.lock", lock_timeout=self._lock_timeout) as lock:
+        with LockFile(
+            directory=dst_dir, name=os.path.basename(dst_filename) + ".lock", lock_timeout=self._lock_timeout
+        ) as lock:
             # Maybe it was copied in the meantime, while waiting for the lock.
             if self._check_existing_copied_file_maybe_cleanup(src_filename, dst_filename):
+                print(f"FileCache: using existing file {dst_filename}")
+                os.utime(dst_filename, None)  # touch
                 return
 
             print(f"FileCache: Copy file {src_filename} to cache")
 
-            # Make sure we have enough disk space.
-            self.cleanup(need_at_least_free_space_size=os.stat(src_filename).st_size)
+            # Make sure we have enough disk space, st_size +1 due to _copy_with_prealloc
+            self.cleanup(need_at_least_free_space_size=os.stat(src_filename).st_size + 1)
 
             dst_tmp_filename = dst_filename + ".copy"
             if os.path.exists(dst_tmp_filename):

{returnn-1.20240621.130142 → returnn-1.20240621.152144}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240621.130142
+Version: 1.20240621.152144
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240621.130142/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240621.130142'
-long_version = '1.20240621.130142+git.08c1ad5'