returnn 1.20260109.93428__py3-none-any.whl → 1.20260113.134416__py3-none-any.whl

returnn/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20260109.93428
+Version: 1.20260113.134416
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn/__old_mod_loader__.py

@@ -17,7 +17,7 @@ This is supported as well.
 import sys
 import os
 import types
-import typing
+from typing import Any, Dict
 import importlib
 
 old_to_new_mod_mapping = {
@@ -122,7 +122,7 @@ class _LazyLoader(types.ModuleType):
             fn = "%s/%s/__init__.py" % (_base_dir, full_mod_name.replace(".", "/"))
             assert os.path.exists(fn), "_LazyLoader: mod %r not found in %r" % (full_mod_name, _base_dir)
         self.__file__ = fn
-        self._lazy_mod_config = dict(full_mod_name=full_mod_name, **kwargs)  # type: typing.Dict[str]
+        self._lazy_mod_config: Dict[str, Any] = dict(full_mod_name=full_mod_name, **kwargs)
 
     def _load(self):
         full_mod_name = self.__name__
@@ -172,6 +172,30 @@ class _LazyLoader(types.ModuleType):
         return super(_LazyLoader, self).__getattribute__(item)
 
     def __getattr__(self, item):
+        if item == "torch":
+            # torch.compile Dynamo hashing can trigger this, when it uses pickle to serialize some object state,
+            # which iterates through sys.modules and does getattr on each module.
+            # In this case, it searches for torch.
+            #   File ".../torch/_inductor/codecache.py", line 607 in dumps
+            #   File ".../torch/_inductor/codecache.py", line 622 in get_hash
+            #   File ".../torch/_inductor/codecache.py", line 961 in compiled_fx_graph_hash
+            #   ...
+            # Unfortunately, Pickler.dump is native code, so we cannot easily check whether that is the parent frame.
+            # The C stacktrace looks like:
+            # ...
+            # 7   Python                              0x0000000102e7d504 call_attribute + 80
+            # 8   Python                              0x0000000102e7d400 _Py_slot_tp_getattr_hook + 576
+            # 9   Python                              0x0000000102e507a0 PyObject_GetOptionalAttr + 248
+            # 10  _pickle.cpython-313-darwin.so       0x0000000102d24fb4 get_deep_attribute + 104
+            # 11  _pickle.cpython-313-darwin.so       0x0000000102d250b8 _checkmodule + 88
+            # 12  _pickle.cpython-313-darwin.so       0x0000000102d22588 save_global + 3024
+            # 13  _pickle.cpython-313-darwin.so       0x0000000102d1eddc save + 3424
+            # ...
+            # Right now, we just check for `item == "torch"` as a heuristic,
+            # which should never exist for any of the old-style wrapped modules here.
+            # We could maybe also check sys._getframe(1).f_code or so and add some other heuristics...
+            raise AttributeError(f"module {self.__name__} has no attribute {item} (lazy loading skipped)")
+
         module = self._load()
         return getattr(module, item)
 

returnn/_setup_info_generated.py

@@ -1,2 +1,2 @@
-version = '1.20260109.093428'
-long_version = '1.20260109.093428+git.68426d7'
+version = '1.20260113.134416'
+long_version = '1.20260113.134416+git.8c8a566'

returnn/datasets/lm.py

@@ -86,6 +86,7 @@ class LmDataset(CachedDataset2):
         delayed_seq_data_start_symbol="[START]",
         dtype: Optional[str] = None,
         tag_prefix: Optional[str] = None,
+        _debug_limit_line_count: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -138,6 +139,8 @@ class LmDataset(CachedDataset2):
           delayed_seq_data_start_symbol + original_sequence[:-1].
         :param str delayed_seq_data_start_symbol: used for add_delayed_seq_data.
         :param dtype: explicit dtype. if not given, automatically determined based on the number of labels.
+        :param tag_prefix: prefix for sequence tags. by default "line-".
+        :param _debug_limit_line_count:
         """
         super(LmDataset, self).__init__(**kwargs)
 
@@ -316,6 +319,10 @@ class LmDataset(CachedDataset2):
         self.num_skipped = 0
         self.num_unknown = 0
 
+        if _debug_limit_line_count is None:
+            _debug_limit_line_count = _get_debug_limit_line_count()
+        self._debug_limit_line_count = _debug_limit_line_count
+
     def _lazy_init(self):
         if self._orths_offsets_and_lens is not None:
             return
@@ -340,6 +347,9 @@ class LmDataset(CachedDataset2):
         lens_per_corpus_file = []
         start_time = time.time()
         last_print_time = start_time
+        debug_limit_line_count = self._debug_limit_line_count
+        debug_limit_est_total = 0
+        debug_limit_hit = False
 
         def _init_tmp_file():
             nonlocal tmp_file, tmp_file_orth_files_index
@@ -368,13 +378,16 @@ class LmDataset(CachedDataset2):
 
             if time.time() - last_print_time > 10:
                 print(
-                    f"  ... loaded {len(self._orths_offsets_and_lens)} sequences,"
+                    f"  ... loaded {len(orths)} sequences,"
                     f" {human_bytes_size(total_bytes_read)},"
                     f" after {hms(time.time() - start_time)}",
                     file=log.v4,
                 )
                 last_print_time = time.time()
 
+            if debug_limit_line_count is not None and len(orths) - prev_orth_len >= debug_limit_line_count:
+                raise _ReachedDebugLimitLineCount()
+
         # If a list of files is provided, concatenate all.
         if isinstance(corpus_file, str):
             corpus_file = [corpus_file]
@@ -383,37 +396,46 @@ class LmDataset(CachedDataset2):
         for file_name in corpus_file:
             if self._use_cache_manager:
                 file_name = cf(file_name)
-            if _is_bliss(file_name):
-                _init_tmp_file()
-                _iter_bliss(filename=file_name, callback=_tmp_file_add_line, decode=False)
-            elif file_name.endswith(".gz"):
-                _init_tmp_file()
-                _iter_txt(
-                    filename=file_name,
-                    callback=_tmp_file_add_line,
-                    skip_empty_lines=self._skip_empty_lines,
-                    decode=False,
-                )
-            else:  # Raw txt file
-                # Directly mmap the file.
-                # We just need to scan once through it to find line offsets.
-                file = open(file_name, "rb")
-                file_mmap = mmap.mmap(file.fileno(), 0, flags=mmap.MAP_PRIVATE)
-                file_index = len(self._orth_files)
-                self._orth_files.append(file)
-                self._orth_mmaps.append(file_mmap)
-
-                pos = 0
-                while True:
-                    next_new_line = file_mmap.find(b"\n", pos)
-                    if next_new_line == -1:
-                        break
-                    line_len = next_new_line - pos
-                    if line_len or not self._skip_empty_lines:
-                        orths.append((file_index, pos, line_len))
-                    total_bytes_read += line_len + 1
-                    pos = next_new_line + 1
-                    _maybe_report_status()
+
+            try:
+                if _is_bliss(file_name):
+                    _init_tmp_file()
+                    _iter_bliss(filename=file_name, callback=_tmp_file_add_line, decode=False)
+                elif file_name.endswith(".gz"):
+                    _init_tmp_file()
+                    _iter_txt(
+                        filename=file_name,
+                        callback=_tmp_file_add_line,
+                        skip_empty_lines=self._skip_empty_lines,
+                        decode=False,
+                    )
+                else:  # Raw txt file
+                    # Directly mmap the file.
+                    # We just need to scan once through it to find line offsets.
+                    file = open(file_name, "rb")
+                    file_mmap = mmap.mmap(file.fileno(), 0, flags=mmap.MAP_PRIVATE)
+                    file_index = len(self._orth_files)
+                    self._orth_files.append(file)
+                    self._orth_mmaps.append(file_mmap)
+
+                    pos = 0
+                    while True:
+                        next_new_line = file_mmap.find(b"\n", pos)
+                        if next_new_line == -1:
+                            break
+                        line_len = next_new_line - pos
+                        if line_len or not self._skip_empty_lines:
+                            orths.append((file_index, pos, line_len))
+                        total_bytes_read += line_len + 1
+                        pos = next_new_line + 1
+                        _maybe_report_status()
+
+            except _ReachedDebugLimitLineCount as exc:
+                assert exc.estimated_total_num_seqs is not None  # currently only for _iter_txt implemented
+                debug_limit_est_total += exc.estimated_total_num_seqs
+                debug_limit_hit = True
+            else:  # iteration completed without hitting debug limit
+                debug_limit_est_total += len(orths) - prev_orth_len
 
             lens_per_corpus_file.append(len(orths) - prev_orth_len)
             prev_orth_len = len(orths)
@@ -447,6 +469,18 @@ class LmDataset(CachedDataset2):
             file=log.v4,
         )
 
+        if debug_limit_hit:
+            est_frac_loaded = len(self._orths_offsets_and_lens) / debug_limit_est_total
+            new_partition_epoch = max(int(self.partition_epoch * est_frac_loaded), 1)
+            print(
+                f"LmDataset: debug limit of {debug_limit_line_count} lines (per file) hit,"
+                f" estimated total num seqs {debug_limit_est_total},"
+                f" loaded {len(self._orths_offsets_and_lens)}, {est_frac_loaded:.2%},"
+                f" adjusting partition_epoch from {self.partition_epoch} to {new_partition_epoch}",
+                file=log.v4,
+            )
+            self.partition_epoch = new_partition_epoch
+
         # It's only estimated because we might filter some out or so.
         self._estimated_num_seqs = len(self._orths_offsets_and_lens) // self.partition_epoch
 
@@ -784,19 +818,34 @@ def _iter_txt(
     :param decode:
     """
     f = open(filename, "rb")
+    f_ = f
     if filename.endswith(".gz"):
         f = gzip.GzipFile(fileobj=f)
 
-    for line in f:
-        if decode:
-            try:
-                line = line.decode("utf8")
-            except UnicodeDecodeError:
-                line = line.decode("latin_1")  # or iso8859_15?
-        line = line.strip()
-        if skip_empty_lines and not line:
-            continue
-        callback(line)
+    count = 0
+    try:
+        for line in f:
+            if decode:
+                try:
+                    line = line.decode("utf8")
+                except UnicodeDecodeError:
+                    line = line.decode("latin_1")  # or iso8859_15?
+            line = line.strip()
+            if skip_empty_lines and not line:
+                continue
+            count += 1
+            callback(line)
+
+    except _ReachedDebugLimitLineCount as exc:
+        print(f"Reached debug limit line count for {filename}, stopping early", file=log.v4)
+        pos = f_.tell()
+        f_.seek(0, os.SEEK_END)
+        size = f_.tell()
+        print(f"  stopped at byte {human_bytes_size(pos)} / {human_bytes_size(size)}", file=log.v4)
+        estimated_num_seqs = int(count * (size / pos))
+        print(f"  estimated total num seqs: {estimated_num_seqs}", file=log.v4)
+        exc.estimated_total_num_seqs = estimated_num_seqs
+        raise
 
 
 def iter_corpus(
@@ -2517,6 +2566,25 @@ def get_post_processor_function(opts):
     return chained_post_processors
 
 
+def _get_debug_limit_line_count() -> Optional[int]:
+    """
+    :return: if set, limit to this many lines for debugging
+    """
+    from returnn.config import get_global_config
+
+    config = get_global_config(raise_exception=False)
+    if not config:
+        return None
+
+    return config.int("lm_dataset_debug_limit_line_count", None)
+
+
+class _ReachedDebugLimitLineCount(Exception):
+    """internal exception to signal reached debug limit line count"""
+
+    estimated_total_num_seqs: Optional[int] = None
+
+
 def _main():
     from returnn.util import better_exchook
 

returnn/frontend/_native/__init__.py

@@ -67,6 +67,24 @@ def _code_hash_md5(filename: str) -> str:
 
 
 _is_set_up = False
+_enabled = True
+
+
+def set_enabled(enabled: bool):
+    """
+    Enable or disable the native code setup.
+
+    :param enabled:
+    """
+    global _enabled
+    _enabled = enabled
+
+
+def is_set_up() -> bool:
+    """
+    :return: whether the native code is set up
+    """
+    return _is_set_up
 
 
 def setup():
@@ -76,6 +94,8 @@ def setup():
     global _is_set_up
     if _is_set_up:
         return
+    if not _enabled:
+        return
     _is_set_up = True  # only try once
 
     from returnn.tensor import Tensor, Dim
@@ -177,6 +197,8 @@ def setup_torch():
     global _is_set_up_torch
     if _is_set_up_torch:
         return
+    if not _enabled:
+        return
     _is_set_up_torch = True  # only try once
 
     import torch

returnn/frontend/_utils.py

@@ -110,7 +110,7 @@ def bin_op_out_template(
             all_dims.extend([dim_ for dim_ in a.dims if dim_ == dim])
         else:
             all_dims.extend([dim_ for dim_ in b.dims if dim_ == dim])
-    if all(set(x.dims) != set(all_dims) for x in (a, b)):
+    if all([set(x.dims) != set(all_dims) for x in (a, b)]):
         if allow_broadcast_all_sources is False:
             raise ValueError(f"compare: sources {a!r} {b!r} not allowed with allow_broadcast_all_sources=False")
         elif allow_broadcast_all_sources is None:

returnn/frontend/array_.py

@@ -195,7 +195,7 @@ def merge_dims(
     if out_dim is None:
         from returnn.util.basic import prod
 
-        if any(d.need_masking() for d in dims[1:]):
+        if any([d.need_masking() for d in dims[1:]]):
             # The dynamic sizes as calculated via dim math would not correctly describe how the tensor looks like.
             # This would then potentially discard some of the data in the tensor in subsequent operations,
             # when masking is applied.
@@ -910,7 +910,7 @@ def scatter(
         else:
             raise ValueError(f"scatter: invalid mode {mode!r}")
     indices_dim = indices_dim if isinstance(indices_dim, (list, tuple)) else [indices_dim]
-    if any(dim.need_masking() for dim in indices_dim):
+    if any([dim.need_masking() for dim in indices_dim]):
         if use_mask is None:
             use_mask = rf.use_mask_default(
                 default=True, default_false_for_behavior_version_up_to=22, func_name="scatter"

returnn/tensor/_dim_extra.py

@@ -858,7 +858,7 @@ class _DimMixin:
         self._make_extra()
         dim_order_default = self.dyn_size_ext.dims + (self,)
         if dim_order is not None:
-            dim_order = tuple(d for d in dim_order if d in dim_order_default)  # filter
+            dim_order = tuple([d for d in dim_order if d in dim_order_default])  # filter
         else:
             dim_order = dim_order_default
         cache_key = (device, dim_order)
@@ -2484,16 +2484,16 @@ _BinOpStrs = {
 
 def _math_get_dim_via_bin_op(dims: Sequence[Union[Dim, int]], op_kind: str) -> Dim:
     dims = [d if isinstance(d, _d.Dim) else _make_constant_static_dim(d) for d in dims]
-    if all(d.dimension is not None for d in dims):
+    if all([d.dimension is not None for d in dims]):
         op = _BinOps[op_kind]
         dim_value = dims[0].dimension
         for d in dims[1:]:
             dim_value = op(dim_value, d.dimension)
     else:
         dim_value = None
-    if all(d.is_constant_static_dim() for d in dims):
+    if all([d.is_constant_static_dim() for d in dims]):
         return _make_constant_static_dim(dim_value, kind=_get_merged_dim_kind(dims))
-    desc = _BinOpStrs[op_kind].join(_get_description(d) for d in dims)
+    desc = _BinOpStrs[op_kind].join([_get_description(d) for d in dims])
     if op_kind.startswith("ceildiv"):
         desc = f"⌈{desc}⌉"
     return _d.Dim(
@@ -2676,16 +2676,16 @@ def _get_description(dim, brackets=True):
 
 
 def _get_merged_dim_kind(dim_tags: Sequence[Dim]) -> Entity:
-    if any(tag.is_batch_dim() for tag in dim_tags):
+    if any([tag.is_batch_dim() for tag in dim_tags]):
         return DimTypes.Batch
-    elif any(tag.is_feature_dim() for tag in dim_tags):
+    elif any([tag.is_feature_dim() for tag in dim_tags]):
         return DimTypes.Feature
     else:
         return DimTypes.Spatial
 
 
 def _representative_tag(terms: Sequence[Dim]) -> Optional[Dim]:
-    if any(not term_.auto_generated for term_ in terms):
+    if any([not term_.auto_generated for term_ in terms]):
         # Always prefer non-auto-generated.
         terms = [term_ for term_ in terms if not term_.auto_generated]
     # First find any dynamic.

returnn/tensor/_tensor_extra.py

@@ -32,8 +32,8 @@ class _TensorExtra:
         tensor: Tensor,
         time_dim_axis=NotSpecified,
         available_for_inference=True,
-        batch=None,
-        beam=None,
+        batch: Optional[BatchInfo] = None,
+        beam: Optional[SearchBeam] = None,
         control_flow_ctx=None,
     ):
         """
@@ -41,8 +41,8 @@ class _TensorExtra:
         :param int|None|NotSpecified time_dim_axis: where we have the time dim axis, after we added the batch-dim.
             this is often 1. however, can be None if there is no time-dim.
         :param bool available_for_inference: e.g. the extern data "classes" is usually not available for inference
-        :param BatchInfo|None batch:
-        :param SearchBeam|None beam: the batch-dim could be extended by a beam-size,
+        :param batch:
+        :param beam: the batch-dim could be extended by a beam-size,
             such that it represents the merged dims [batch, beam_size].
         :param ControlFlowContext|None control_flow_ctx:
         """
@@ -668,11 +668,11 @@ class _TensorMixin(_TensorMixinBase):
         if not perm:
             return self.copy()
         if allow_int and isinstance(perm[0], int):
-            assert all(isinstance(a, int) for a in perm), f"{self}: invalid perm {perm!r} types"
+            assert all([isinstance(a, int) for a in perm]), f"{self}: invalid perm {perm!r} types"
             assert set(perm) == set(range(len(perm))), f"{self}: invalid perm {perm!r}"
             return self._copy_compatible_to_dims_with_perm([self._dims[i] for i in perm], perm)
         else:
-            assert all(isinstance(a, Dim) for a in perm), f"{self}: invalid perm {perm!r} types"
+            assert all([isinstance(a, Dim) for a in perm]), f"{self}: invalid perm {perm!r} types"
             return self.copy_compatible_to_dims(perm)
 
     def copy_move_axis(self, old_axis, new_axis) -> _t.Tensor:
@@ -1155,7 +1155,7 @@ class _TensorMixin(_TensorMixinBase):
                     )
 
         assert v.batch_ndim == data.batch_ndim
-        assert all(mapped_axes[ax] == ax for ax in range(v.batch_ndim))
+        assert all([mapped_axes[ax] == ax for ax in range(v.batch_ndim)])
 
         if self.version == 1:
             # Ensure time_dim_axis and feature_dim_axis is same as in data
@@ -1702,7 +1702,7 @@ class _TensorMixin(_TensorMixinBase):
         """
         :return: shape with added batch-dim. e.g. (batch,time,feat) = (None,None,128)
         """
-        return tuple(tag.dimension for tag in self.dim_tags)
+        return tuple([tag.dimension for tag in self.dim_tags])
 
     # noinspection PyShadowingNames
     def get_batch_shape(self, batch_dim):
@@ -3214,7 +3214,7 @@ class _TensorMixin(_TensorMixinBase):
         if len(sources) == 1:
             return sources[0].copy_template()
         max_ndim = max([s.batch_ndim for s in sources])
-        if any(src.batch for src in sources):
+        if any([src.batch for src in sources]):
             from returnn.tf.util.data import BatchInfo
 
             common_batch = BatchInfo.get_common_batch_info([src.batch for src in sources if src.batch])
@@ -3254,7 +3254,7 @@ class _TensorMixin(_TensorMixinBase):
             else:
                 axis = common.get_default_new_axis_for_dim_tag(dim_tag)
                 common = common.copy_add_dim_by_tag(dim_tag, unbroadcast=True, axis=axis)
-        if all(s.batch_ndim < common.batch_ndim for s in sources):
+        if all([s.batch_ndim < common.batch_ndim for s in sources]):
             from returnn.util.basic import validate_broadcast_all_sources
 
             validate_broadcast_all_sources(

returnn/torch/engine.py

@@ -3,9 +3,11 @@ Main engine for PyTorch
 """
 
 from __future__ import annotations
+
 from typing import Optional, Any, Union, Callable, Dict, Set
 from contextlib import nullcontext, ExitStack, contextmanager
 
+import sys
 import gc
 import os
 import time
@@ -20,6 +22,7 @@ from torch.nn.parallel import DistributedDataParallel
 from torch.utils.data import DataLoader
 from torch import autocast
 from torch.cuda import amp
+from torch.profiler import record_function
 import numpy as np
 
 import returnn
@@ -404,10 +407,14 @@ class Engine(EngineBase):
         total_data_size_packed = NumbersDict()
         total_data_size_padded = NumbersDict()
 
+        prof = _opt_torch_profiler_from_opts(self.config.opt_typed_value("torch_profile"))
+        if prof:
+            prof.__enter__()
+
         report_prefix = f"ep {self.epoch} train"
         try:
             while True:
-                with torch.no_grad():
+                with torch.no_grad(), record_function("data_loading"):
                     extern_data_raw = next(data_iter, None)
 
                 step_begin_time = time.monotonic()
@@ -485,7 +492,8 @@ class Engine(EngineBase):
                 with (
                     self._ddp_pt_model.no_sync()
                     if (self._ddp_pt_model is not None and not perform_update_step)
-                    else nullcontext()
+                    else nullcontext(),
+                    record_function("backward"),
                 ):
                     if self._grad_scaler is not None:
                         self._grad_scaler.scale(total_loss.raw_tensor).backward()
@@ -500,7 +508,8 @@ class Engine(EngineBase):
 
                 # only update the weights when every gradient accumulation loop ends
                 if perform_update_step:
-                    self._updater.step(grad_scaler=self._grad_scaler)
+                    with record_function("optimizer_step"):
+                        self._updater.step(grad_scaler=self._grad_scaler)
                 zero_grad_next_step = perform_update_step
 
                 if self._torch_distributed_ctx:
@@ -582,10 +591,19 @@ class Engine(EngineBase):
                 self._updater.set_current_train_step(
                     global_train_step=self.global_train_step, epoch=self.epoch, epoch_continuous=epoch_continuous
                 )
+
+                if prof:
+                    prof.step()
+
         except Exception as exc:
+            if prof:
+                prof.__exit__(type(exc), exc, exc.__traceback__)
             help_on_torch_exception(exc, step_idx=step_idx, model=self._orig_model, extern_data=extern_data)
             raise
 
+        if prof:
+            prof.__exit__(None, None, None)
+
         elapsed = time.monotonic() - epoch_start_time
         elapsed_computation_percentage = elapsed_computation_time / elapsed
         total_padding_ratio = NumbersDict.constant_like(1.0, total_data_size_packed) - (
@@ -885,6 +903,7 @@ class Engine(EngineBase):
             if self._default_float_dtype:
                 stack.enter_context(rf.set_default_float_dtype_ctx(str(self._default_float_dtype).split(".")[-1]))
                 stack.enter_context(_set_torch_default_dtype_ctx_mgr(self._default_float_dtype))
+            stack.enter_context(record_function("model_step"))
             yield
 
     def _run_step(
@@ -1734,3 +1753,101 @@ def _torch_load(filename: Union[str, os.PathLike], *, device: str) -> Dict[str,
         return safetensors_load(filename, device=device)
 
     return torch.load(filename, map_location=device)
+
+
+class _TorchProfiler:
+    def __init__(self, profiler: torch.profiler.profile, max_step: Optional[int]):
+        self.profiler = profiler
+        self.max_step = max_step
+        self.entered = False
+
+    def __enter__(self):
+        self.profiler.__enter__()
+        self.entered = True
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not self.entered:
+            return
+        self.entered = False
+        self.profiler.__exit__(exc_type, exc_val, exc_tb)
+
+        if exc_type is None:
+            print(
+                "Torch profiling finished, exporting Chrome trace to torch_profile.json,"
+                " memory timeline to torch_memory_profile.html...",
+                file=log.v2,
+            )
+            self.profiler.export_chrome_trace("torch_profile.json")
+            self.profiler.export_memory_timeline("torch_memory_profile.html")
+
+            print("Exiting program after Torch profiling.", file=log.v2)
+            sys.exit(0)
+
+    def step(self):
+        """step"""
+        self.profiler.step()
+        if self.max_step is not None and self.profiler.step_num > self.max_step:
+            print(f"Reached max profiling step {self.max_step}, stopping Torch profiler.", file=log.v2)
+            self.profiler.stop()
+            self.__exit__(None, None, None)
+
+
+def _opt_torch_profiler_from_opts(
+    opts: Union[None, int, bool, str, Dict[str, Any]],
+) -> Optional[_TorchProfiler]:
+    if isinstance(opts, str):
+        from returnn.util.basic import to_bool
+
+        opts = to_bool(opts)
+
+    if opts is None:
+        return None
+    elif isinstance(opts, (bool, int)):
+        if not opts:
+            return None
+        opts = {}
+    elif isinstance(opts, dict):
+        opts = opts.copy()
+    else:
+        raise TypeError(f"Invalid type for torch_profile {opts!r}: {type(opts)}")
+
+    from torch.profiler import profile, ProfilerActivity, schedule
+
+    print("Using Torch profiler...", file=log.v2)
+
+    prof_max_step = None
+
+    if "activities" not in opts:
+        activities = [ProfilerActivity.CPU]
+        if torch.cuda.is_available():
+            activities += [ProfilerActivity.CUDA]
+        elif torch.xpu.is_available():
+            activities += [ProfilerActivity.XPU]
+        opts["activities"] = activities
+
+    opts.setdefault("profile_memory", True)
+    opts.setdefault("record_shapes", True)
+    opts.setdefault("with_stack", True)
+    opts.setdefault("with_flops", True)
+    # Note: active*repeat are the steps we actually profile.
+    opts.setdefault("schedule", dict(skip_first=10, wait=5, warmup=3, active=3, repeat=1))
+
+    if isinstance(opts["schedule"], dict):
+        schedule_opts: Dict[str, Any] = opts["schedule"]
+        schedule_opts = schedule_opts.copy()
+        schedule_opts.setdefault("repeat", 0)
+        schedule_opts.setdefault("skip_first", 0)
+        schedule_opts.setdefault("skip_first_wait", 0)
+        opts["schedule"] = schedule(**schedule_opts)
+
+        if schedule_opts["repeat"] > 0:
+            prof_max_step = (schedule_opts["wait"] + schedule_opts["warmup"] + schedule_opts["active"]) * schedule_opts[
+                "repeat"
+            ]
+            prof_max_step += schedule_opts["skip_first"]
+            if schedule_opts["skip_first_wait"] != 0:
+                prof_max_step -= schedule_opts["wait"]
+            print(f"Profiling will stop automatically after {prof_max_step} steps.", file=log.v3)
+
+    prof = profile(**opts)
+    return _TorchProfiler(prof, prof_max_step)

returnn/torch/frontend/_backend.py

@@ -275,7 +275,7 @@ class TorchBackend(Backend[torch.Tensor]):
         :return: tensor
         """
         assert len(dims) >= 2
-        first_axis = min(source.dims.index(d) for d in dims)
+        first_axis = min([source.dims.index(d) for d in dims])
         pre_dims = source.dims[:first_axis]
         post_dims = [d for d in source.dims if d not in dims and d not in pre_dims]
         source = source.copy_transpose(tuple(pre_dims) + tuple(dims) + tuple(post_dims), allow_int=False)
@@ -884,7 +884,7 @@ class TorchBackend(Backend[torch.Tensor]):
         :param perm: e.g. [0, 2, 1]
         :return: permuted (transposed) raw tensor; wraps torch.permute
         """
-        if all(p == i for i, p in enumerate(perm)):
+        if all([p == i for i, p in enumerate(perm)]):
             return raw_tensor
         return torch.permute(raw_tensor, tuple(perm))
 
@@ -1788,7 +1788,7 @@ class TorchBackend(Backend[torch.Tensor]):
         remaining_dims = [d for d in tensor.dims if d not in mask.dims]
         tensor_templ_dims = tuple(dims) + tuple(remaining_dims)
         in_raw = tensor.copy_compatible_to_dims_raw(tensor_templ_dims)
-        if any(in_raw.shape[i] == 1 < d.get_dim_value() for i, d in enumerate(dims)):
+        if any([in_raw.shape[i] == 1 < d.get_dim_value() for i, d in enumerate(dims)]):
             # unbroadcast
             in_raw = in_raw.expand([d.get_dim_value() for d in tensor_templ_dims])
         if mask.raw_tensor.device.type == "meta":

returnn/torch/frontend/compile_helper.py

@@ -0,0 +1,106 @@
+"""
+Helpers to improve torch.compile on RF code.
+"""
+
+from __future__ import annotations
+from typing import Any, Iterable, List, Tuple
+
+import os
+from returnn.tensor import Tensor, Dim
+
+# noinspection PyProtectedMember
+from returnn.frontend import _native
+
+_is_set_up = False
+
+
+def setup():
+    """
+    Set up the torch.compile helpers for RF code, also including :class:`Tensor` and :class:`Dim`.
+    """
+
+    global _is_set_up
+    if _is_set_up:
+        return
+    _is_set_up = True  # only try once
+
+    assert not _native.is_set_up(), "Call this setup() as early as possible."
+    _native.set_enabled(False)
+
+    # We have lots of dynamic shapes.
+    os.environ["TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS"] = "1"
+
+    # noinspection PyProtectedMember
+    from torch.utils._pytree import register_pytree_node
+
+    register_pytree_node(Tensor, _tensor_flatten, _tensor_unflatten)
+    register_pytree_node(Dim, _dim_flatten, _dim_unflatten)
+
+    Dim.get_dim_value = _dim_get_dim_value
+
+
+def _tensor_flatten(t: Tensor) -> Tuple[List[Any], Any]:
+    """
+    Flatten the tensor for PyTree.
+    """
+    return [t.raw_tensor, t.dims, t.sparse_dim], [
+        t.name,
+        t.dtype,
+        t.version,
+        t.feature_dim_axis_or_unspecified,
+        t.time_dim_axis_or_unspecified,
+    ]
+
+
+def _tensor_unflatten(values: Iterable[Any], metadata: Any) -> Tensor:
+    """
+    Unflatten the tensor from PyTree.
+    """
+    raw_tensor, dims, sparse_dim = values
+    name, dtype, version, feature_dim_axis, time_dim_axis = metadata
+    return Tensor(
+        name=name,
+        dims=dims,
+        dtype=dtype,
+        sparse_dim=sparse_dim,
+        feature_dim_axis=feature_dim_axis,
+        time_dim_axis=time_dim_axis,
+        raw_tensor=raw_tensor,
+        version=version,
+    )
+
+
+def _dim_flatten(d: Dim) -> Tuple[List[Any], Any]:
+    """
+    Flatten the dim for PyTree.
+    """
+    return [d.dyn_size_ext], [d.name, d.dimension, d.size]
+
+
+def _dim_unflatten(values: Iterable[Any], metadata: Any) -> Dim:
+    """
+    Unflatten the dim from PyTree.
+    """
+    (dyn_size_ext,) = values
+    name, dimension, size = metadata
+    # TODO this creates a new instance... this is maybe wrong?
+    return Dim(name=name, dimension=dimension, size=size, dyn_size_ext=dyn_size_ext)
+
+
+def _dim_get_dim_value(self: Dim) -> int:
+    """
+    Infers the dim this axis should have if unbroadcasted.
+    If `self.src_data` has a placeholder, will use the shape from there.
+    Otherwise, uses `self.dimension` (if static) or `self.dyn_size` (if dynamic).
+
+    :return: max(size or dyn_size)
+    """
+    res = self.get_dim_value_tensor()
+    if isinstance(res, Tensor):
+        assert res.dims == ()
+        assert res.raw_tensor is not None
+        # Specifically PyTorch would then treat it as a SymInt in torch.compile,
+        # which is important to have for some torch functions (e.g. torch.tile and others).
+        return int(res.raw_tensor)
+    assert isinstance(res, int)
+    return res

returnn/util/basic.py

@@ -3816,6 +3816,8 @@ def should_write_to_disk(config):
             return False
     if config.is_true("dry_run"):
         return False
+    if config.is_true("torch_profile"):
+        return False
     return True
 
 

returnn/util/debug.py

@@ -309,6 +309,7 @@ def _get_native_signal_handler_lib_filename() -> str:
               old_signal_handler[SIGILL] = signal(SIGILL, signal_handler);
               old_signal_handler[SIGABRT] = signal(SIGABRT, signal_handler);
               old_signal_handler[SIGFPE] = signal(SIGFPE, signal_handler);
+              old_signal_handler[SIGUSR1] = signal(SIGUSR1, signal_handler);
             }
             """
         ),

{returnn-1.20260109.93428.dist-info → returnn-1.20260113.134416.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20260109.93428
+Version: 1.20260113.134416
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20260109.93428.dist-info → returnn-1.20260113.134416.dist-info}/RECORD

@@ -1,9 +1,9 @@
-returnn/PKG-INFO,sha256=8G2OFR-V5IlE98f0vmLneA27jg9-B7eN973G7vJpj0I,5215
+returnn/PKG-INFO,sha256=jhNOEgbBWBglgqkHqni28aMhOK1nHC1dJlBiKkaWfX0,5216
 returnn/__init__.py,sha256=biBtRsM0WZ406vShaeH-9WFoqJ8XwTbn6g0EeFJ7l8E,1012
 returnn/__main__.py,sha256=lHyZcu_0yc9f7Vf_Kfdy9PmeU0T76XVXnpalHi5WKro,31740
-returnn/__old_mod_loader__.py,sha256=nvsNY-xELdS_IPNkv66Q9Rmvg4dbGW0-EBRDcCmctos,7654
+returnn/__old_mod_loader__.py,sha256=-XAtilhq87CqmWmK2awbfGLoPAwjLGVu8t4QAxCw0fQ,9436
 returnn/__setup__.py,sha256=22kQn2fh11iPM0hLb2Fy5sLmoU1JGvmDxXRYuRgQkwU,4659
-returnn/_setup_info_generated.py,sha256=RHjC4xFQRTza5prANYrhwttWMqAEZoLiKwzMpCmll80,77
+returnn/_setup_info_generated.py,sha256=OVoyfxrF7cQ0OBpiMfzvCidyV9ia6hJFPW-TrKd9BYE,77
 returnn/config.py,sha256=JK8EjDsUdyY2c90s0KY1rLD1kesVfz6vRT0gxy_AQ5I,29142
 returnn/forward_iface.py,sha256=A_OJiaXsX4MlXQRzST86ylyxSUZbC402PQL1REcqHjM,911
 returnn/learning_rate_control.py,sha256=ZvWryAn_tv9DhV8sh1LV3eE34Yltl3On3mYZAG4hR9s,34684
@@ -21,7 +21,7 @@ returnn/datasets/distrib_files.py,sha256=48edqdf7YpnPJ-TOis3Mz5U9A2DSxfiYT1HCMSt
 returnn/datasets/generating.py,sha256=o9-JZ2s5QKssux6GcSaM3oivf_PE6nhSOeytRyGB7pQ,99574
 returnn/datasets/hdf.py,sha256=v5sjBenURR9Z-g7AQ9tsL84yDSye5RtbLpym3M6HSDE,67833
 returnn/datasets/huggingface.py,sha256=ls9WMR6gUcMgGksl80g0An1az5Xjya_V3ojbbbsZqrU,20047
-returnn/datasets/lm.py,sha256=CXl_g-Z28RWlBTzx35uC4r_GCwOP05LIsUp0iSi6JG4,100652
+returnn/datasets/lm.py,sha256=riDa7rkwOuPX53_0y9wgQ_s2A9453BX0gWGV0HX29_M,103614
 returnn/datasets/map.py,sha256=kOBJVZmwDhLsOplzDNByIfa0NRSUaMo2Lsy36lBvxrM,10907
 returnn/datasets/meta.py,sha256=hTtfwINIxP2S4JQ5IQXzvTh2MixwxzeF06pPTW36yl0,101456
 returnn/datasets/multi_proc.py,sha256=BClXq0fActi1XQa4vcMhHmhYF0Q-fnnDzlIlbBM6_DM,22614
@@ -80,8 +80,8 @@ returnn/frontend/_backend.py,sha256=MVZn2HSkF3tsqchYvy2QM9pA4ILdKq07kj-_AAHGUy0,
 returnn/frontend/_cache.py,sha256=Uao2xzfvVaKABk1fkxcpXzxKIGJaI9FwwlTvvoNUstk,8550
 returnn/frontend/_numpy_backend.py,sha256=fZjks7p3dgxVZ6tSDazTTgBxNjJqXjfqgw_7mA7rDEE,9066
 returnn/frontend/_random_journal.py,sha256=_ktP_mjgx8vtQQGX_DofdhewJj0aPiczefTWeemPkmo,5457
-returnn/frontend/_utils.py,sha256=uVQldGHyYKIyhSEmumJ04ix5eP5tjZw4CEC0w6-zhyQ,12074
-returnn/frontend/array_.py,sha256=bZwTgNkMsGiSP6TVgI7bxY6zZMjcs9TVsHlajYrHUoA,56791
+returnn/frontend/_utils.py,sha256=LTwYQJBT9XjRdC2kVvHy29eUN5qARNSLGMJk90a8PjI,12076
+returnn/frontend/array_.py,sha256=2VQYtlB6OiKdpkU9H_w_jIUrb7mlxizz7KKOHjnYaeo,56795
 returnn/frontend/attention.py,sha256=bFD9Ei6GxSi-BC1OfueDyTIE-51a3dKKZOWdSIbz7l8,46633
 returnn/frontend/backend.py,sha256=iQ9w4xl8Ea7bgpb0VUaCKq50rV5Bl2E5J8Rhd-oqD_c,883
 returnn/frontend/build_from_dict.py,sha256=rfWa2rjjhIR_kIQED_nMrygrQBunS6unegzWTLVbC98,3017
@@ -120,7 +120,7 @@ returnn/frontend/state.py,sha256=EePdrx6PtWL4mJ2XZmGlh5dl4nq6G9wZpqP4hdDEzfY,293
 returnn/frontend/stepwise_scheduler.py,sha256=fMOTR7npGCDXrXDmSQ4VwmudoHEbY3Yr-QGyjFdQJSc,927
 returnn/frontend/tensor_array.py,sha256=Ej7CHtvpY0yBROlAk5vFe3CTXh-iAuqu9qcXS3Qxt2I,4328
 returnn/frontend/types.py,sha256=r-QsxPQyFSr9WwCRzqTn_X5jQLbjthrtjHavY8XIDmk,1099
-returnn/frontend/_native/__init__.py,sha256=fVjazAujt0rdICXZL-GgW1sjFeL1HB4NPuy2m5rmMsc,6480
+returnn/frontend/_native/__init__.py,sha256=VVK0x6Z7OZa3Sb4QDSz9sRrBhX8FfYdvrwhAg4W9-cc,6839
 returnn/frontend/_native/backend.cpp,sha256=MeHczHypwj_ncntOxRqanK8SqGyV9Eq1X0cpMWb_WII,4768
 returnn/frontend/_native/backend.hpp,sha256=Wq80dcEzXfRNxGOXFnIgHllkiv1rDi3KpHK-xxJsSDI,791
 returnn/frontend/_native/module.cpp,sha256=9BCUoDTZDJ6hlXp4pUus1BlN7-oxcRy6tK9ctyCkwk0,15709
@@ -155,8 +155,8 @@ returnn/sprint/extern_interface.py,sha256=l-v1X-Yg0UpTFe7Y3c4FwWOqpSNuv9Oy5EzqlK
 returnn/sprint/interface.py,sha256=1j5SB0V8hSW8A5song9ciZtcBnZoKKfNipk9ezOIMuA,36491
 returnn/tensor/README.md,sha256=X6BqcRLrPLPnwF9yR69uqIFrMnNluj9pBkOPHwNgzuo,501
 returnn/tensor/__init__.py,sha256=on6j5PEOQpck50UcsR4nJzJSDmoVy34z1Oq4efv6Ax0,154
-returnn/tensor/_dim_extra.py,sha256=tHE3N6hUKqbzedJ8RNhn9aJHxvhTQuI9JckCLsPbKKI,116776
-returnn/tensor/_tensor_extra.py,sha256=1UPNisRAbljkvfMcrEXaPAF-2Dz7AdgC3jAKVVAnAO8,165084
+returnn/tensor/_dim_extra.py,sha256=8HLTvgEnThCp7GdtB714Tvs4ad939jZmhpS3qab03sU,116790
+returnn/tensor/_tensor_extra.py,sha256=ClwZBfaOavDtapXYpYRhDTGE85bzvRqox5mF_OnEHds,165112
 returnn/tensor/_tensor_mixin_base.py,sha256=H5z86I0NejxrSgMH1c5oXQzBqS6L9HpvP4y7oegBaSc,643
 returnn/tensor/_tensor_op_overloads.py,sha256=HklwuTBjy7mH_665VKaCUdu-oC3aa7Uz1ZQiCz4jeZc,5448
 returnn/tensor/control_flow_ctx.py,sha256=L9e32AfYDUDgsEDHL07thSFyYFqwhyVSqzE_bM03Y4M,5252
@@ -208,7 +208,7 @@ returnn/tf/util/open_fst.py,sha256=sZRDw4TbxvhGqpGdUJWy1ebvlZm4_RPhygpRw9uLAOQ,1
 returnn/torch/README.md,sha256=jzJ2FpOHW02vxN69yKaV97C9LI-hmvjBglKfdZXIDdc,85
 returnn/torch/__init__.py,sha256=MHEUyNHB20Vy89uKAqZoj6FxJKF1Gq3HW-i6ra1pNcI,24
 returnn/torch/distributed.py,sha256=_lyJR71HIoCHpMi5GztGM7YwrX54Am8zSkjnDkE1Lbk,7524
-returnn/torch/engine.py,sha256=XaJhVpF181sf8M1iXAs3u0zr37VVUG3SW81-DIZgg3g,81280
+returnn/torch/engine.py,sha256=JnoGrAakIUIsSXVEIVIXqTOVcDYJASVoRNZQrOPNrdA,85368
 returnn/torch/updater.py,sha256=nNd1mBPQyvIB096BEFi0KKmRI-U3jnRETzb743p2B9c,32064
 returnn/torch/data/__init__.py,sha256=6cLNEi8KoGI12PF6akN7mI_mtjlx-0hcQAfMYoExwik,132
 returnn/torch/data/extern_data.py,sha256=5al706ZaYtHWLp5VH2vS-rW69YXP3NHyOFRKY0WY714,7810
@@ -217,9 +217,10 @@ returnn/torch/data/queued_data_iter.py,sha256=PoOsGHdHVZjTmcyfq_ZOw--P6hyfTdmAWI
 returnn/torch/data/returnn_dataset_wrapper.py,sha256=fMahf05G0SPYm6HxSQpVm8JhsIHons-i1Ce4aQv4IjM,8332
 returnn/torch/data/tensor_utils.py,sha256=-Teqi--LLbt6q_5mDRdoHZHmPgSdC83W706ukif_YiU,1284
 returnn/torch/frontend/__init__.py,sha256=AA48HZnC17ASuKA0EWy8loZ-Bib_yUtqF4T1wYvjst4,62
-returnn/torch/frontend/_backend.py,sha256=8EBRGN0jY5rl9Z5-wd4kvoDesssWcVDVXNl25-bG8cA,108882
+returnn/torch/frontend/_backend.py,sha256=wsmalFnT_p2NDADL8N-6AHHCyv2yBe8nKM-0tKAh1cs,108888
 returnn/torch/frontend/_rand.py,sha256=1JgIkV2XmpgJD86zXZ-NCAe-QuoP2swr6NaS1oz3Qa8,1830
 returnn/torch/frontend/bridge.py,sha256=RBtAIlYWn_AC-GaHWperrOncPjMLWAOrU30pWk2789A,9775
+returnn/torch/frontend/compile_helper.py,sha256=ax8ax5mjC8PDHtwTQzHYWUNRoKjZMuYHF6me9VdxiSY,2969
 returnn/torch/frontend/raw_ops.py,sha256=lF0h-KtYYsdaaqQADylVZp9qzPskOOXA4MfmYDyx5IU,296
 returnn/torch/optim/README.md,sha256=0iH5FiKb7iDrVK5n8V6yCh4ciCFG2YSbyh7lPneT5ik,360
 returnn/torch/optim/__init__.py,sha256=yxdbnOkXAHzZ_t6cHi6zn5x_DQNlLZJ-KxZByHTIg1U,29
@@ -234,11 +235,11 @@ returnn/torch/util/gradient_checkpoint.py,sha256=iLy-FB65DC8O6LxzmMvFjnSdpIVpko8
 returnn/torch/util/module.py,sha256=MXHIrF9Isu575DDJIa81212ULKwdqu1oOLxDVZecVSk,1693
 returnn/torch/util/scaled_gradient.py,sha256=C5e79mpqtxdtw08OTSy413TSBSlOertRisc-ioiFIaU,3191
 returnn/util/__init__.py,sha256=UIG1qw4idqhW71BV60ha7h9PktxvEVcBIu0lYRossK8,336
-returnn/util/basic.py,sha256=rFeg3XwjNcNDbBgjkhisStbjTFA8CEfIrdwHjfdkJKw,143212
+returnn/util/basic.py,sha256=Pa2cAdvOJMKK7gR3heAVTol-zYVbThr9b9slVVAaH3M,143273
 returnn/util/better_exchook.py,sha256=hOKazwv2q2-d0XMfxkJXMbLZyNTtraV3jPHplFcrMsg,71014
 returnn/util/bpe.py,sha256=LWFhICZsEOnMwNws0lybPNzKRX6rSr8yKCvP65vjl9Y,19656
 returnn/util/collect_outputs_dict.py,sha256=CjpsftoMgmvyE4wNKTO6F-QQ_44QHXcOZIXMUMQVZ-8,2637
-returnn/util/debug.py,sha256=0ED4etMKG9lVqU0HPKEiCK-HoS8hBgnQza444QCE6ec,28576
+returnn/util/debug.py,sha256=Ndq5nz-tMEG9ZNwZTbgOkQYB9JSvAwF8r0o53Gf2EbM,28653
 returnn/util/debug_helpers.py,sha256=0EINLK4uLtoSt5_kHs1M2NIFpMd0S7i4c4rx90U4fJk,2914
 returnn/util/file_cache.py,sha256=8xE4zMQi38g7ZIGwNohd13_CgjzpIs18ILxFCKttzxE,29439
 returnn/util/fsa.py,sha256=k2lJ8tyf_g44Xk1EPVLwDwpP4spoMTqIigDVOWocQHY,59177
@@ -255,8 +256,8 @@ returnn/util/sig_proc.py,sha256=Tjz0VOAVyqu2qDCF5HZ1JjALjcFsHcNkcd96WgZeKfE,7265
 returnn/util/task_system.py,sha256=7Dz7Nvi_1-o5pDv9OZYdAnlJw6OSvgbYUmQ72P0Fgkw,26002
 returnn/util/train_proc_manager.py,sha256=Pjht28k6uz6BNQ47uW6Gf880iyq5q4wx7P_K2tmoAM8,3266
 returnn/util/watch_memory.py,sha256=BR5P2kvBN6UI81cE0_1WAA6Hd1SByLbBaiDxvLhPOew,4213
-returnn-1.20260109.93428.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
-returnn-1.20260109.93428.dist-info/METADATA,sha256=8G2OFR-V5IlE98f0vmLneA27jg9-B7eN973G7vJpj0I,5215
-returnn-1.20260109.93428.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-returnn-1.20260109.93428.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
-returnn-1.20260109.93428.dist-info/RECORD,,
+returnn-1.20260113.134416.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
+returnn-1.20260113.134416.dist-info/METADATA,sha256=jhNOEgbBWBglgqkHqni28aMhOK1nHC1dJlBiKkaWfX0,5216
+returnn-1.20260113.134416.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+returnn-1.20260113.134416.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
+returnn-1.20260113.134416.dist-info/RECORD,,