returnn 1.20241026.3853__tar.gz → 1.20241105.131828__tar.gz

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241026.3853
+Version: 1.20241105.131828
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241105.131828/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241105.131828'
+long_version = '1.20241105.131828+git.0494bcf'

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/returnn/datasets/multi_proc.py

@@ -68,6 +68,7 @@ class MultiProcDataset(CachedDataset2):
         self._data_keys = None
         self._num_seqs = None
         self._total_num_seqs = None
+        self._all_tags = None
 
         self._worker_parent_conns = None  # type: Optional[List[mpConnection]]
         self._seq_order_proc_parent_conn = None  # type: Optional[mpConnection]
@@ -96,8 +97,9 @@ class MultiProcDataset(CachedDataset2):
         return {
             "num_inputs": self.num_inputs,
             "num_outputs": self.num_outputs,
-            "total_num_seqs": self._total_num_seqs,
             "labels": self.labels,
+            "total_num_seqs": self._total_num_seqs,
+            "all_tags": self._all_tags,
         }
 
     def _lazy_init(self):
@@ -172,8 +174,6 @@ class MultiProcDataset(CachedDataset2):
         assert msg == "num_inputs"
         msg, self.num_outputs = self._seq_order_proc_parent_conn.recv()
         assert msg == "num_outputs"
-        msg, self._total_num_seqs = self._seq_order_proc_parent_conn.recv()
-        assert msg == "total_num_seqs"
         msg, self.labels = self._seq_order_proc_parent_conn.recv()
         assert msg == "labels"
 
@@ -281,12 +281,23 @@ class MultiProcDataset(CachedDataset2):
                         dataset = init_dataset(dataset_dict)
                     parent_conn.send(("num_inputs", dataset.num_inputs))
                     parent_conn.send(("num_outputs", dataset.num_outputs))
+                    parent_conn.send(("labels", dataset.labels))
+                elif msg == "get_total_num_seqs":
+                    assert dataset
                     try:
                         total_num_seqs = dataset.get_total_num_seqs()
-                    except NotImplementedError:
-                        total_num_seqs = None
+                        assert isinstance(total_num_seqs, int)
+                    except NotImplementedError as exc:
+                        total_num_seqs = NotImplementedError(f"{exc} in {dataset}")
                     parent_conn.send(("total_num_seqs", total_num_seqs))
-                    parent_conn.send(("labels", dataset.labels))
+                elif msg == "get_all_tags":
+                    assert dataset
+                    try:
+                        all_tags = dataset.get_all_tags()
+                        assert isinstance(all_tags, list)
+                    except NotImplementedError as exc:
+                        all_tags = NotImplementedError(f"{exc} in {dataset}")
+                    parent_conn.send(("all_tags", all_tags))
                 elif msg == "init_seq_order":
                     if dataset is None:
                         dataset = init_dataset(dataset_dict)
@@ -400,9 +411,31 @@ class MultiProcDataset(CachedDataset2):
 
     def get_total_num_seqs(self, *, fast: bool = False) -> int:
         """total num seqs"""
-        if self._total_num_seqs is not None:
+        if self._total_num_seqs is None:
+            worker = self._seq_order_proc_parent_conn
+            worker.send(("get_total_num_seqs", {}))
+            msg, self._total_num_seqs = worker.recv()
+            assert msg == "total_num_seqs" and self._total_num_seqs is not None
+        if isinstance(self._total_num_seqs, int):
             return self._total_num_seqs
-        raise NotImplementedError
+        elif isinstance(self._total_num_seqs, Exception):
+            raise self._total_num_seqs
+        else:
+            raise TypeError(f"invalid type {type(self._total_num_seqs)} for total_num_seqs")
+
+    def get_all_tags(self):
+        """all tags"""
+        if self._all_tags is None:
+            worker = self._seq_order_proc_parent_conn
+            worker.send(("get_all_tags", {}))
+            msg, self._all_tags = worker.recv()
+            assert msg == "all_tags" and self._all_tags is not None
+        if isinstance(self._all_tags, list):
+            return self._all_tags
+        elif isinstance(self._all_tags, Exception):
+            raise self._all_tags
+        else:
+            raise TypeError(f"invalid type {type(self._all_tags)} for all_tags")
 
     def finish_epoch(self, *, free_resources: bool = False):
         """finish epoch"""

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/returnn/torch/data/pipeline.py

@@ -28,7 +28,7 @@ import numpy
 import torch
 import torch.utils.data
 
-from returnn.util.basic import NumbersDict
+from returnn.util.basic import NumbersDict, get_fwd_compat_kwargs
 
 
 def create_tensor(array: numpy.ndarray) -> Union[torch.Tensor, numpy.ndarray]:
@@ -59,7 +59,7 @@ def collate_batch(batch: List[Dict[str, numpy.ndarray]]) -> Dict[str, Union[torc
 
     res = {}
     for key in data_keys:
-        if key == "num_seqs":
+        if key in ("num_seqs", "epoch"):
             res[key] = batch[0][key]  # it should always be the same
             continue
         ls = [create_tensor(sample[key]) for sample in batch]
@@ -119,7 +119,7 @@ class ChunkingIterDataPipe(torch.utils.data.IterDataPipe):
 
             if not chunking_data_keys:
                 chunking_data_keys = list(data_dict.keys())  # use all if not configured separately
-                chunking_data_key_black_list = ["seq_tag", "seq_idx", "num_seqs"]
+                chunking_data_key_black_list = ["seq_tag", "seq_idx", "num_seqs", "epoch"]
                 for key in chunking_data_key_black_list:
                     if key in chunking_data_keys:
                         chunking_data_keys.remove(key)
@@ -208,20 +208,66 @@ class BatchingIterDataPipe(torch.utils.data.IterDataPipe):
     def __init__(self, dataset: torch.utils.data.IterableDataset, batch_size=1, max_seqs=None):
         """
         :param dataset: dataset to apply batching to
-        :param int|dict[str,int]|None batch_size: Maximum number of time steps (e.g. audio frames / words) in one
-            batch (padding included).
+        :param int|dict[str,int]|None|function batch_size: Maximum number of time steps (e.g. audio frames / words)
+            in one batch (padding included).
             If given as a dict data_key -> value, sets different individual limits per data key.
             If None, no limit.
-        :param int|None max_seqs: maximum number of sequences in a batch,
-            None means unlimited (also -1 to match TF backend)
+            Can also be a callable with kwargs epoch, seq_idx, epoch_continuous, **_other_kwargs,
+            returning the batch size.
+        :param int|None|function max_seqs: maximum number of sequences in a batch,
+            None means unlimited (also -1 to match TF backend).
+            Can also be a callable with kwargs epoch, seq_idx, epoch_continuous, **_other_kwargs,
+            returning the max seqs.
         """
         super().__init__()
         self._dataset = dataset
-        self._max_batch_size = NumbersDict(sys.maxsize if batch_size is None else batch_size)
-        self._max_seqs = sys.maxsize if (max_seqs is None or max_seqs == -1) else max_seqs
+        self._max_batch_size = self._parse_batch_size(batch_size)
+        self._max_seqs = self._parse_max_seqs(max_seqs)
 
-        assert self._max_batch_size.min_value() > 0
-        assert self._max_seqs > 0
+        if not callable(self._max_batch_size):
+            assert isinstance(self._max_batch_size, NumbersDict) and self._max_batch_size.min_value() > 0
+        if not callable(self._max_seqs):
+            assert isinstance(self._max_seqs, int) and self._max_seqs > 0
+
+    @staticmethod
+    def _parse_batch_size(
+        batch_size: Union[int, Dict[str, int], NumbersDict, None, Callable],
+        *,
+        data_dict: Optional[Dict[str, Any]] = None,
+    ) -> Union[NumbersDict, Callable]:
+        """
+        :param batch_size: see __init__()
+        :return: batch_size
+        """
+        if callable(batch_size):
+            if data_dict:
+                batch_size = batch_size(**BatchingIterDataPipe._get_user_func_kwargs_from_data_dict(data_dict))
+            else:
+                return batch_size
+        return NumbersDict(sys.maxsize if batch_size is None else batch_size)
+
+    @staticmethod
+    def _parse_max_seqs(
+        max_seqs: Union[int, None, Callable], *, data_dict: Optional[Dict[str, Any]] = None
+    ) -> Union[int, Callable]:
+        """
+        :param max_seqs: see __init__()
+        :return: max_seqs
+        """
+        if callable(max_seqs):
+            if data_dict:
+                max_seqs = max_seqs(**BatchingIterDataPipe._get_user_func_kwargs_from_data_dict(data_dict))
+            else:
+                return max_seqs
+        return sys.maxsize if (max_seqs is None or max_seqs == -1) else max_seqs
+
+    @staticmethod
+    def _get_user_func_kwargs_from_data_dict(data_dict: Dict[str, Any]) -> Dict[str, Any]:
+        epoch = int(data_dict["epoch"])
+        seq_idx = int(data_dict["seq_idx"])
+        num_seqs = int(data_dict["num_seqs"])  # >=1 if known, otherwise -1
+        epoch_continuous = (epoch - 1 + (seq_idx + 1) / num_seqs) if num_seqs > 0 else None
+        return {"epoch": epoch, "seq_idx": seq_idx, "epoch_continuous": epoch_continuous, **get_fwd_compat_kwargs()}
 
     def __iter__(self):
         """
@@ -233,7 +279,12 @@ class BatchingIterDataPipe(torch.utils.data.IterDataPipe):
         current_max_sequence_lengths = NumbersDict(0)  # data_key -> length of longest sequence in current batch
 
         for data_dict in self._dataset:
-            if len(current_batch) == self._max_seqs:
+            max_seqs = self._parse_max_seqs(self._max_seqs, data_dict=data_dict)
+            max_batch_size = self._parse_batch_size(self._max_batch_size, data_dict=data_dict)
+            assert isinstance(max_seqs, int) and max_seqs > 0
+            assert isinstance(max_batch_size, NumbersDict) and max_batch_size.min_value() > 0
+
+            if len(current_batch) >= max_seqs:
                 yield current_batch
                 current_batch = []
                 current_max_sequence_lengths = NumbersDict(0)
@@ -246,7 +297,7 @@ class BatchingIterDataPipe(torch.utils.data.IterDataPipe):
             max_sequence_lengths_if_included = NumbersDict.max([current_max_sequence_lengths, sequence_lengths])
             batch_size_if_included = max_sequence_lengths_if_included * (len(current_batch) + 1)  # including padding
 
-            if current_batch and batch_size_if_included.any_compare(self._max_batch_size, (lambda a, b: a > b)):
+            if current_batch and batch_size_if_included.any_compare(max_batch_size, (lambda a, b: a > b)):
                 yield current_batch
                 current_batch = [data_dict]
                 current_max_sequence_lengths = sequence_lengths

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/returnn/torch/data/returnn_dataset_wrapper.py

@@ -81,6 +81,8 @@ class ReturnnDatasetIterDataPipe(torch.utils.data.IterDataPipe):
         except Exception:  # might not work for all datasets
             num_seqs = -1
         num_seqs = numpy.array(num_seqs)
+        assert self._dataset.epoch is not None
+        epoch = numpy.array(self._dataset.epoch)
 
         try:
             data_keys = self._dataset.get_data_keys()
@@ -94,6 +96,8 @@ class ReturnnDatasetIterDataPipe(torch.utils.data.IterDataPipe):
                 # It's slightly redundant to have num_seqs in each entry,
                 # but it's difficult to pass this back to the main proc otherwise.
                 data["num_seqs"] = num_seqs
+                # epoch is also redundant, but that's the cleanest/simplest way to pass it on to BatchingIterDataPipe.
+                data["epoch"] = epoch
                 yield data
                 seq_index += 1
 

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/returnn/torch/engine.py

@@ -34,6 +34,7 @@ from returnn.util import NumbersDict
 from returnn.util.basic import hms, NotSpecified
 from returnn.util.result_with_reason import ResultWithReason
 from returnn.util.debug import debug_shell
+from returnn.util.math import simplify_and_format_number
 from returnn.forward_iface import ForwardCallbackIface
 
 from .updater import Updater
@@ -125,6 +126,7 @@ class Engine(EngineBase):
         self._log_memory_usage = config.bool("torch_log_memory_usage", False)
         self._log_batch_size = config.bool("log_batch_size", False) and log.verbose[5]
         self._calculate_exp_loss = config.bool("calculate_exp_loss", False)
+        self._log_grad_norm = _parse_log_grad_norm(config)
         self._reset_dev_memory_caches = config.bool("reset_dev_memory_caches", False)
         self._forward_auto_split_batch_on_oom = config.bool("forward_auto_split_batch_on_oom", False)
         self._stop_on_nonfinite_train_score = config.bool("stop_on_nonfinite_train_score", True)
@@ -429,6 +431,12 @@ class Engine(EngineBase):
                     else:
                         total_loss.raw_tensor.backward()
 
+                if self._log_grad_norm and perform_update_step:
+                    key = f"grad_norm:p{simplify_and_format_number(self._log_grad_norm)}"
+                    assert key not in losses_dict
+                    inv_norm_factors_dict[key] = 1.0  # once per update step
+                    losses_dict[key] = _get_total_grad_norm(self._pt_model, p=self._log_grad_norm)
+
                 # only update the weights when every gradient accumulation loop ends
                 if perform_update_step:
                     self._updater.step(grad_scaler=self._grad_scaler)
@@ -684,7 +692,7 @@ class Engine(EngineBase):
         batch_size = self.config.typed_value("batch_size", -1)
         batch_size = self.config.typed_value(f"batch_size_{'train' if train else 'dev'}", batch_size)
         assert batch_size != -1, f"batch_size or batch_size_{'train' if train else 'dev'} not defined in config"
-        max_seqs = self.config.int("max_seqs", -1)
+        max_seqs = self.config.typed_value("max_seqs", -1)
         batches_dataset = data_pipeline.BatchingIterDataPipe(wrapped_dataset, batch_size=batch_size, max_seqs=max_seqs)
 
         loader_opts = self.config.typed_value("torch_dataloader_opts") or {}
@@ -1286,9 +1294,9 @@ def _print_process(
     if log.verbose[5]:  # report every minibatch
         info = [report_prefix, "step %i" % step]
         if eval_info:  # Such as score.
-            info += ["%s %s" % (k, _format_value(v)) for k, v in eval_info.items()]
+            info += ["%s %s" % (k, _format_score_value(v)) for k, v in eval_info.items()]
         if batch_size_info:
-            info += ["%s %s" % (k, _format_value(v)) for k, v in batch_size_info.items()]
+            info += ["%s %s" % (k, _format_score_value(v)) for k, v in batch_size_info.items()]
         if log_memory_usage_device:
             dev = torch.device(log_memory_usage_device)
             if dev.type == "cuda":
@@ -1324,11 +1332,11 @@ def _format_score(score: Dict[str, float]) -> str:
     if not score:
         return "None"
     if len(score) == 1:
-        return _format_value(list(score.values())[0])
-    return " ".join(["%s %s" % (key.split(":", 2)[-1], _format_value(score[key])) for key in score.keys()])
+        return _format_score_value(list(score.values())[0])
+    return " ".join(["%s %s" % (key.split(":", 2)[-1], _format_score_value(score[key])) for key in score.keys()])
 
 
-def _format_value(v: Any) -> str:
+def _format_score_value(v: Any) -> str:
     if isinstance(v, float):
         if abs(v) > 1.0e3 or abs(v) < 1.0e-3:
             return f"{v:.3e}"
@@ -1422,3 +1430,39 @@ def _set_torch_default_dtype_ctx_mgr(dtype: torch.dtype):
         yield
     finally:
         torch.set_default_dtype(old_dtype)
+
+
+def _parse_log_grad_norm(config: Config) -> Optional[Union[int, float]]:
+    log_grad_norm = config.opt_typed_value("log_grad_norm", False)
+    if isinstance(log_grad_norm, str):
+        if log_grad_norm.lower() in ["true", "false", "none"]:
+            log_grad_norm = {"true": True, "false": False, "none": None}[log_grad_norm.lower()]
+        else:
+            raise ValueError(f"Invalid value for log_grad_norm: {log_grad_norm!r}")
+    if log_grad_norm is None:
+        pass
+    elif isinstance(log_grad_norm, bool):
+        if log_grad_norm:
+            log_grad_norm = 2
+        else:
+            log_grad_norm = None
+    elif isinstance(log_grad_norm, (int, float)):
+        assert log_grad_norm > 0, f"log_grad_norm {log_grad_norm} > 0 expected"  # otherwise fine...
+    else:
+        raise TypeError(f"Invalid type for log_grad_norm: {log_grad_norm!r} type {type(log_grad_norm)}")
+    return log_grad_norm
+
+
+def _get_total_grad_norm(model: torch.nn.Module, p: float) -> float:
+    return float(
+        torch.norm(
+            torch.stack(
+                [
+                    param.grad.norm(p=p).detach().cpu()
+                    for param in model.parameters()
+                    if param.requires_grad and param.grad is not None
+                ]
+            ),
+            p=p,
+        ).item()
+    )

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/returnn/util/basic.py

@@ -554,12 +554,11 @@ def describe_torch_version() -> str:
     return "%s (%s in %s)" % (version, git_info, tdir)
 
 
-def get_tensorflow_version_tuple():
+def get_tensorflow_version_tuple() -> Tuple[int, ...]:
     """
     :return: tuple of ints, first entry is the major version
-    :rtype: tuple[int]
     """
-    import tensorflow as tf
+    import tensorflow as tf  # noqa
     import re
 
     return tuple([int(re.sub("(-rc[0-9]|-dev[0-9]*)", "", s)) for s in tf.__version__.split(".")])
@@ -2031,6 +2030,9 @@ class NumbersDict:
     def __delitem__(self, key):
         del self.dict[key]
 
+    def __contains__(self, item):
+        return item in self.dict
+
     def get(self, key, default=None):
         """
         :param str key:
@@ -2070,7 +2072,7 @@ class NumbersDict:
     def items(self):
         """
         :return: dict items. this excludes self.value
-        :rtype: str[(str,object)]
+        :rtype: set[(str,object)]
         """
         return self.dict.items()
 

returnn-1.20241105.131828/returnn/util/math.py

@@ -0,0 +1,87 @@
+"""
+Some mathematical functions, in pure NumPy.
+"""
+
+from __future__ import annotations
+from typing import Union, Optional, Dict
+import numpy
+
+
+def ceil_div(a: int, b: int) -> int:
+    """ceil(a / b)"""
+    return -(-a // b)
+
+
+def next_power_of_two(n: int) -> int:
+    """next power of two, >= n"""
+    return 2 ** (int(n - 1).bit_length())
+
+
+class PiecewiseLinear:
+    """
+    Piecewise linear function.
+    (Basically wraps ``numpy.interp``.)
+    """
+
+    def __init__(
+        self,
+        values: Dict[Union[int, float], Union[int, float]],
+        *,
+        kw_name: Optional[str] = None,
+        ignore_other_kwargs: bool = False,
+    ):
+        """
+        :param values: dict x -> y. Everything between the x values is linearly interpolated.
+            Everything outside is repeated from the nearest x value.
+        :param kw_name: keyword argument name to use in the __call__. Other keyword arguments are ignored.
+        :param ignore_other_kwargs: if True, ignore other keyword arguments in the __call__.
+        """
+        self._sorted_items = sorted(values.items())
+        self._sorted_keys = numpy.array([x for x, _ in self._sorted_items])
+        self._sorted_values = numpy.array([y for _, y in self._sorted_items])
+        self._kw_name = kw_name
+        self._ignore_other_kwargs = ignore_other_kwargs
+
+    def __getstate__(self):
+        # Note: I was implementing __getnewargs_ex__, but we cannot use this because of this Sisyphus bug:
+        # https://github.com/rwth-i6/sisyphus/issues/215
+        kwargs = {"values": dict(self._sorted_items)}
+        if self._kw_name is not None:
+            kwargs["kw_name"] = self._kw_name
+        if self._ignore_other_kwargs:
+            kwargs["ignore_other_kwargs"] = True
+        return kwargs
+
+    def __setstate__(self, state):
+        self.__init__(**state)
+
+    def __repr__(self) -> str:
+        kwargs = self.__getstate__()
+        values = kwargs.pop("values")
+        all_args_str = ", ".join([repr(values)] + [f"{k}={v!r}" for k, v in kwargs.items()])
+        return f"{self.__class__.__name__}({all_args_str})"
+
+    def __call__(self, *args: Union[int, float], **kwargs) -> Union[int, float]:
+        if self._kw_name:
+            if args:
+                raise TypeError(f"{self}: Expected zero positional arguments, got {args!r}")
+            x = kwargs.pop(self._kw_name, None)
+        else:
+            if len(args) != 1:
+                raise TypeError(f"{self}: Expected one positional argument, got {args!r}")
+            x = args[0]
+        if not self._ignore_other_kwargs:
+            if kwargs:
+                raise TypeError(f"{self}: Unexpected keyword arguments: {kwargs!r}")
+        assert x is not None
+        steps = self._sorted_keys
+        values = self._sorted_values
+        return numpy.interp(x, steps, values)
+
+
+def simplify_and_format_number(n: Union[int, float]) -> str:
+    """Format a number, removing trailing zeros and the decimal point if it is an integer"""
+    if isinstance(n, (int, float)):
+        return str(n).rstrip("0").rstrip(".")
+    else:
+        raise TypeError(f"Expected int or float, got {n!r} type {type(n)}")

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241026.3853
+Version: 1.20241105.131828
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20241026.3853 → returnn-1.20241105.131828}/tests/test_Util.py

@@ -101,6 +101,7 @@ def test_math_PiecewiseLinear():
 
     eps = 1e-5
     f = PiecewiseLinear({1: 2, 3: 4, 5: 1})
+    assert str(f) == "PiecewiseLinear({1: 2, 3: 4, 5: 1})"
     assert_equal(f(0), 2)
     assert_equal(f(1 - eps), 2)
     assert_equal(f(1), 2)
@@ -116,6 +117,35 @@ def test_math_PiecewiseLinear():
     assert_equal(f(6), 1)
 
 
+def test_math_PiecewiseLinear_kwargs():
+    from returnn.util.math import PiecewiseLinear
+
+    f = PiecewiseLinear({1: 2, 3: 4, 5: 1}, kw_name="epoch_continuous")
+    try:
+        f(0)
+    except TypeError:
+        pass  # this is expected
+    else:
+        assert False, "TypeError expected (wrong args)"
+    assert f(epoch_continuous=0) == 2
+    try:
+        f(epoch_continuous=0, seq_idx=123)
+    except TypeError:
+        pass  # this is expected
+    else:
+        assert False, "TypeError expected (wrong args)"
+
+    f = PiecewiseLinear({1: 2, 3: 4, 5: 1}, kw_name="epoch_continuous", ignore_other_kwargs=True)
+    try:
+        f(0)
+    except TypeError:
+        pass  # this is expected
+    else:
+        assert False, "TypeError expected (wrong args)"
+    assert f(epoch_continuous=0) == 2
+    assert f(epoch_continuous=0, seq_idx=123) == 2
+
+
 def test_parse_orthography_into_symbols():
     assert_equal(list("hi"), parse_orthography_into_symbols("hi"))
     assert_equal(list(" hello "), parse_orthography_into_symbols(" hello "))

returnn-1.20241026.3853/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241026.003853'
-long_version = '1.20241026.003853+git.98e9755'

returnn-1.20241026.3853/returnn/util/math.py

@@ -1,34 +0,0 @@
-"""
-Some mathematical functions, in pure NumPy.
-"""
-
-from __future__ import annotations
-from typing import Union, Dict
-import numpy
-
-
-def ceil_div(a: int, b: int) -> int:
-    """ceil(a / b)"""
-    return -(-a // b)
-
-
-def next_power_of_two(n: int) -> int:
-    """next power of two, >= n"""
-    return 2 ** (int(n - 1).bit_length())
-
-
-class PiecewiseLinear:
-    """
-    Piecewise linear function.
-    """
-
-    def __init__(self, values: Dict[Union[int, float], Union[int, float]]):
-        self._sorted_items = sorted(values.items())
-        self._sorted_keys = numpy.array([x for x, _ in self._sorted_items])
-        self._sorted_values = numpy.array([y for _, y in self._sorted_items])
-
-    def __call__(self, x: Union[int, float]) -> Union[int, float]:
-        assert x is not None
-        steps = self._sorted_keys
-        values = self._sorted_values
-        return numpy.interp(x, steps, values)