onnx-diagnostic 0.8.0__py3-none-any.whl

onnx_diagnostic/reference/torch_ops/controlflow_ops.py

@@ -0,0 +1,121 @@
+from typing import Any, Dict, Optional
+import onnx
+import torch
+from . import OpRunKernel, OpRunTensor
+
+
+class OpRunControlFlow(OpRunKernel):
+    """Common ancestor for control flows."""
+
+    @classmethod
+    def has_subgraphs(cls) -> bool:
+        """Returns True if the kernel has subgraphs."""
+        return True
+
+    def __init__(
+        self,
+        node: onnx.NodeProto,
+        version: Optional[int] = None,
+        parent: Optional["onnx_diagnostic.reference.TorchOnnxEvaluator"] = None,  # noqa: F821
+        verbose: int = 0,
+    ):
+        super().__init__(node, version, verbose=verbose)
+        assert (
+            parent is not None
+        ), f"parent must be specified for operator {self.__class__.__name__!r}"
+        for att in node.attribute:
+            if att.type == onnx.AttributeProto.GRAPH:
+                rt = parent.__class__(
+                    att.g,
+                    providers=parent.providers,
+                    opsets=parent.opsets,
+                    local_functions=parent.functions,
+                    verbose=parent.verbose,
+                    custom_kernels=parent.custom_kernels,
+                )
+                setattr(self, att.name, rt)
+
+
+class If_1(OpRunControlFlow):
+    "If"
+
+    def run(self, cond, context: Optional[Dict[str, Any]] = None):
+        rt = self.then_branch if cond.tensor.item() else self.else_branch  # type: ignore[attr-defined]
+        return rt.run_with_values(context=context)
+
+
+class Loop_16(OpRunControlFlow):
+    "Loop"
+
+    def __init__(
+        self,
+        node: onnx.NodeProto,
+        version: Optional[int] = None,
+        parent: Optional["onnx_diagnostic.reference.TorchOnnxEvaluator"] = None,  # noqa: F821
+        verbose: int = 0,
+    ):
+        super().__init__(node, version, parent, verbose=verbose)
+        self.output_index = {n: i for i, n in enumerate(self.body.output_names)}
+        self.N = len(self.body.input_names) - 2
+        self.K = len(self.body.output_names) - self.N - 1
+
+    def run(self, M, cond, *args, context: Optional[Dict[str, Any]] = None):
+        if args:
+            v_initial = args[0]
+            args = args[1:]
+        else:
+            v_initial = None
+        assert M is None or hasattr(
+            M, "dtype"
+        ), f"M must be empty or an array but its type is {type(M)}."
+        body = self.body
+        loop_inputs = body.input_names
+        inputs = dict.fromkeys(loop_inputs)
+        if v_initial is not None:
+            inputs[loop_inputs[2]] = v_initial
+        cond_name = body.output_names[0]
+        if args:
+            begin = len(loop_inputs) - len(args)
+            all_inputs = loop_inputs[begin:]
+            for name, val in zip(all_inputs, args):
+                inputs[name] = val
+        if context is not None:
+            for a in context:
+                inputs[a] = context[a]
+
+        k_carried_away = [[] for i in range(self.K)]  # type: ignore
+        it = 0
+        while (cond is None or cond.tensor is None or cond.tensor.item()) and (
+            M is None or M.tensor is None or it < M.tensor.item()
+        ):
+            if len(body.input_names) > 0 and body.input_names[0] is not None:
+                inputs[body.input_names[0]] = OpRunTensor(
+                    torch.tensor(it, dtype=None if M is None else M.dtype)
+                )
+            if len(body.input_names) > 1 and body.input_names[1] is not None:
+                inputs[body.input_names[1]] = cond
+            outputs = list(
+                self.body.run_with_values(
+                    *[inputs[k] for k in self.body.input_names], context=context
+                )
+            )
+            if self.K > 0:
+                for k in range(self.K):
+                    k_carried_away[k].append(outputs[-self.K + k])
+            index_cond = self.output_index[cond_name]
+            cond = outputs[index_cond]
+            assert (
+                cond is not None
+            ), f"Condition {cond_name!r} returned by the subgraph cannot be None."
+            for i, o in zip(body.input_names[2:], body.output_names[1:]):
+                inputs[i] = outputs[self.output_index[o]]
+            it += 1
+
+        if it == 0:
+            outputs = [inputs[i] for i in body.input_names[2:]]
+        else:
+            outputs = outputs[1 : 1 + self.N]
+        outputs.extend([OpRunTensor(torch.cat(x, axis=0)) for x in k_carried_away])
+        while len(outputs) < len(self.body.output_names):
+            outputs.append(OpRunTensor(torch.empty(())))
+        return tuple(outputs)

onnx_diagnostic/reference/torch_ops/generator_ops.py

@@ -0,0 +1,36 @@
+from typing import Optional
+import onnx
+import torch
+from . import OpRunKernel, OpRunTensor
+
+
+class Range_11(OpRunKernel):
+    """Range"""
+
+    @classmethod
+    def device_dependent(cls) -> bool:
+        """
+        Returns True if the kernel needs a device to be efficiently initialized.
+        """
+        return True
+
+    def __init__(
+        self,
+        node: onnx.NodeProto,
+        version: Optional[int] = None,
+        device: Optional[torch.device] = None,
+        verbose: int = 0,
+    ):
+        super().__init__(node, version, verbose=verbose)
+        self.device = device
+
+    def run(self, starts: OpRunTensor, limit: OpRunTensor, delta: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(
+            torch.arange(
+                starts.tensor,
+                limit.tensor,
+                delta.tensor,
+                dtype=starts.dtype,
+                device=self.device,
+            )
+        )

onnx_diagnostic/reference/torch_ops/nn_ops.py

@@ -0,0 +1,196 @@
+from typing import Optional, Tuple
+import onnx
+import torch
+from ...helpers.torch_helper import onnx_dtype_to_torch_dtype
+from . import OpRunKernel, OpRunTensor
+
+
+class AveragePool_11(OpRunKernel):
+    "AveragePool"
+
+    def __init__(
+        self,
+        node: onnx.NodeProto,
+        version: Optional[int] = None,
+        verbose: int = 0,
+    ):
+        super().__init__(node, version, verbose=verbose)
+        self.auto_pad = self.get_attribute_string(node, "auto_pad", "NOTSET")
+        self.ceil_mode = bool(self.get_attribute_int(node, "ceil_mode", 0))
+        self.count_include_pad = bool(self.get_attribute_int(node, "count_include_pad", 0))
+        self.dilations = self.get_attribute_ints(node, "dilations", None)
+        self.kernel_shape: Tuple[int, ...] = (
+            self.get_attribute_ints(node, "kernel_shape") or tuple()
+        )
+        self.pads = self.get_attribute_ints(node, "pads", None)
+        self.strides = self.get_attribute_ints(node, "strides", None)
+
+    def run(self, x):
+        kernel_shape = self.kernel_shape
+        dilations = self.dilations or [1 for _ in x.shape[2:]]
+        strides = self.strides or [1 for _ in x.shape[2:]]
+        pads = self.pads or ([0 for _ in x.shape[2:]] * 2)
+        assert (
+            self.auto_pad == "NOTSET"
+        ), f"conv not implemented for auto_pad={self.auto_pad!r}"
+        assert len(set(pads)) == 1, f"conv not implemented for pads={pads}"
+        assert set(dilations) == {1}, f"conv not implemented for dilations={dilations}"
+        avg_pool = getattr(torch.nn.functional, f"avg_pool{len(kernel_shape)}d")
+        return OpRunTensor(
+            avg_pool(
+                x.tensor,
+                kernel_size=tuple(kernel_shape),
+                stride=tuple(strides),
+                padding=pads[0],
+                ceil_mode=self.ceil_mode,
+                count_include_pad=self.count_include_pad,
+                # dilation=tuple(dilations),
+            )
+        )
+
+
+class Conv_11(OpRunKernel):
+    "Conv"
+
+    def __init__(
+        self,
+        node: onnx.NodeProto,
+        version: Optional[int] = None,
+        verbose: int = 0,
+    ):
+        super().__init__(node, version, verbose=verbose)
+        self.auto_pad = self.get_attribute_string(node, "auto_pad", "NOTSET")
+        self.dilations = self.get_attribute_ints(node, "dilations", None)
+        self.group = self.get_attribute_int(node, "group", 1)
+        self.kernel_shape: Tuple[int, ...] = (
+            self.get_attribute_ints(node, "kernel_shape") or tuple()
+        )
+        self.pads = self.get_attribute_ints(node, "pads", None)
+        self.strides = self.get_attribute_ints(node, "strides", None)
+
+    def run(self, x, w, b=None):
+        kernel_shape = self.kernel_shape or w.shape[2:]
+        assert (
+            tuple(kernel_shape) == w.shape[-len(kernel_shape) :]
+        ), f"conv not implemented for kernel_shape={kernel_shape} and w.shape={w.shape}"
+        dilations = self.dilations or [1 for _ in x.shape[2:]]
+        strides = self.strides or [1 for _ in x.shape[2:]]
+
+        if self.auto_pad in {"SAME_LOWER", "SAME_UPPER"}:
+            head = []
+            tail = []
+            for i in range(len(x.shape) - 2):
+                d = x.shape[i + 2]
+                target_size = (d + strides[i] - 1) // strides[i]
+                pad_needed = (target_size - 1) * strides[i] + kernel_shape[i] - d
+                pad_head = (
+                    (pad_needed + 1) // 2 if self.auto_pad == "SAME_LOWER" else pad_needed // 2
+                )
+                pad_tail = pad_needed - pad_head
+                head.append(pad_head)
+                tail.append(pad_tail)
+            pads = head + tail
+        else:
+            pads = self.pads or ([0 for _ in x.shape[2:]] * 2)
+
+        assert len(set(pads)) == 1, (
+            f"conv not implemented for pads={pads}, "
+            f"auto_pad={self.auto_pad!r}, strides={strides}, "
+            f"x.shape={x.shape}, kernel_shape={kernel_shape}"
+        )
+
+        if b is None:
+            bias = None
+        else:
+            bias = b.tensor.squeeze()
+            if not bias.shape:
+                bias = bias.unsqueeze(0)
+        return OpRunTensor(
+            torch.nn.functional.conv2d(
+                x.tensor,
+                w.tensor,
+                bias=bias,
+                stride=tuple(strides),
+                padding=pads[0],
+                dilation=tuple(dilations),
+                groups=self.group,
+            )
+        )
+
+
+class LayerNormalization_17(OpRunKernel):
+    "LayerNormalization"
+
+    def __init__(
+        self,
+        node: onnx.NodeProto,
+        version: Optional[int] = None,
+        verbose: int = 0,
+    ):
+        super().__init__(node, version, verbose=verbose)
+        self.axis = self.get_attribute_int(node, "axis", -1)
+        self.epsilon = self.get_attribute_float(node, "epsilon", 1e-5)
+        self.stash_type = onnx_dtype_to_torch_dtype(
+            self.get_attribute_int(node, "stash_type", onnx.TensorProto.FLOAT)  # type: ignore[arg-type]
+        )
+        self.compute_std = len(node.output) > 1
+
+    def run(self, x, scale, bias=None):
+        original_dtype = x.dtype
+        if self.stash_type == torch.float32 and x.tensor.dtype != torch.float64:
+            xt = x.tensor
+            res = torch.nn.functional.layer_norm(
+                xt,
+                xt.shape[self.axis :],
+                weight=scale.tensor,
+                bias=None if bias is None else bias.tensor,
+                eps=self.epsilon,
+            )
+        else:
+            xt = x.tensor.to(self.stash_type)
+            res = torch.nn.functional.layer_norm(
+                xt,
+                xt.shape[self.axis :],
+                weight=scale.tensor.to(self.stash_type),
+                bias=None if bias is None else bias.tensor.to(self.stash_type),
+                eps=self.epsilon,
+            )
+        if not self.compute_std:
+            return OpRunTensor(res.to(original_dtype))
+        axes = tuple(range(len(xt.shape)))[self.axis :]
+        mean, var = torch.var(xt, dim=axes, keepdim=False)
+        x_inv_std_dev = torch.reciprocal(torch.sqrt(var + self.epsilon))
+        return (
+            OpRunTensor(res.to(original_dtype)),
+            OpRunTensor(mean),
+            OpRunTensor(x_inv_std_dev),
+        )
+
+
+class Softmax_13(OpRunKernel):
+    "Softmax"
+
+    def __init__(
+        self,
+        node: onnx.NodeProto,
+        version: Optional[int] = None,
+        verbose: int = 0,
+    ):
+        super().__init__(node, version, verbose=verbose)
+        self.axis = self.get_attribute_int(node, "axis", -1)
+        assert isinstance(self.axis, int), f"Unexpected value for attribute axis={self.axis!r}"
+        # this is out of spec
+        stash_type = self.get_attribute_int(node, "stash_type", None)
+        self.stash_type = None if stash_type is None else onnx_dtype_to_torch_dtype(stash_type)
+
+    def run(self, data: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(
+            torch.nn.functional.softmax(data.tensor, dim=self.axis, dtype=self.stash_type)
+        )
+
+
+class Tanh_6(OpRunKernel):
+    "Tanh"
+
+    def run(self, data: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(torch.nn.functional.tanh(data.tensor))

onnx_diagnostic/reference/torch_ops/other_ops.py

@@ -0,0 +1,106 @@
+from typing import Optional
+import onnx
+import torch
+from ...helpers.torch_helper import onnx_dtype_to_torch_dtype
+from . import OpRunKernel, OpRunTensor
+
+
+class Cast_6(OpRunKernel):
+    "Cast"
+
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        to = self.get_attribute_int(node, "to", 0)
+        assert isinstance(to, int), f"Unexpected value for attribute to={to!r}"
+        self.to = onnx_dtype_to_torch_dtype(to)
+        self.saturate = self.get_attribute_int(node, "saturate", 1)
+        assert self.saturate == 1, f"saturate={self.saturate} not implemented for Cast"
+
+    def run(self, data: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(data.tensor.to(self.to))
+
+
+class CastLike_15(OpRunKernel):
+    "Cast"
+
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        self.saturate = self.get_attribute_int(node, "saturate", 1)
+        assert self.saturate == 1, f"saturate={self.saturate} not implemented for CastLike"
+
+    def run(self, data: OpRunTensor, like: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(data.tensor.to(like.tensor.dtype))
+
+
+class Concat_1(OpRunKernel):
+    "Concat"
+
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        axis = self.get_attribute_int(node, "axis", 0)
+        assert isinstance(axis, int), f"Unexpected value for attribute axis={axis!r}"
+        self.axis = axis
+
+    def run(self, *data: OpRunTensor) -> OpRunTensor:
+        assert data, f"No tensor to concatenate in node name {self.name!r}"
+        devices = [d.get_device() for d in data]
+        if len(set(devices)) == 1:
+            return OpRunTensor(torch.cat([t.tensor for t in data], axis=self.axis))
+        if (
+            data[0].dtype == torch.int64
+            and self.axis == 0
+            and max(d.tensor.ndim for d in data) == 1
+            and max(d.tensor.numel() for d in data) <= 8
+        ):
+            # This is a shape
+            return OpRunTensor(torch.cat([t.tensor.cpu() for t in data], axis=self.axis))
+        index = devices.index(max(devices))
+        device = data[index].tensor.device
+        return OpRunTensor(torch.cat([t.tensor.to(device) for t in data], axis=self.axis))
+
+
+class NonZero_13(OpRunKernel):
+    "NonZero"
+
+    def run(self, x: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(torch.nonzero(x.tensor).T)
+
+
+class Tile_6(OpRunKernel):
+    "Tile"
+
+    def run(self, x: OpRunTensor, repeat: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(torch.tile(x.tensor, repeat.as_tuple_int))
+
+
+class Transpose_1(OpRunKernel):
+    "Transpose"
+
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        self.perm = self.get_attribute_ints(node, "perm", None)
+
+    def run(self, data: OpRunTensor) -> OpRunTensor:
+        return OpRunTensor(torch.permute(data.tensor, self.perm))
+
+
+class Trilu_14(OpRunKernel):
+    "Trilu"
+
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        self.upper = self.get_attribute_int(node, "upper", 1)
+
+    def run(self, data: OpRunTensor, k: Optional[OpRunTensor] = None) -> OpRunTensor:
+        diagonal = 0 if k is None else k.tensor.item()
+        if self.upper:
+            return OpRunTensor(torch.triu(data.tensor, diagonal=diagonal))
+        return OpRunTensor(torch.tril(data.tensor, diagonal=diagonal))
+
+
+class Where_9(OpRunKernel):
+    "Where"
+
+    def run(self, cond: OpRunTensor, x: OpRunTensor, y: OpRunTensor) -> OpRunTensor:
+        tcond, tx, ty = self.same_device(cond.tensor, x.tensor, y.tensor)
+        return OpRunTensor(torch.where(tcond, tx, ty))

onnx_diagnostic/reference/torch_ops/reduce_ops.py

@@ -0,0 +1,130 @@
+from typing import Optional, Tuple
+import onnx
+import torch
+from ...helpers.torch_helper import onnx_dtype_to_torch_dtype
+from . import OpRunKernel, OpRunTensor
+
+
+class ReduceOp(OpRunKernel):
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        self.keepdims = bool(self.get_attribute_int(node, "keepdims", 1))
+        self.noop_with_empty_axes = bool(
+            self.get_attribute_int(node, "noop_with_empty_axes", 0)
+        )
+        assert isinstance(
+            self.keepdims, bool
+        ), f"Unexpected value for attribute keepdims={self.keepdims!r}"
+        assert isinstance(self.noop_with_empty_axes, bool), (
+            f"Unexpected value for attribute "
+            f"noop_with_empty_axes={self.noop_with_empty_axes!r}"
+        )
+        assert (
+            not self.noop_with_empty_axes
+        ), f"Not implemented with noop_with_empty_axes={self.noop_with_empty_axes}"
+        # this is out of spec
+        stash_type = self.get_attribute_int(node, "stash_type", None)
+        self.stash_type = None if stash_type is None else onnx_dtype_to_torch_dtype(stash_type)
+
+
+class ReduceOpAxes(ReduceOp):
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        self.axes: Tuple[int, ...] = self.get_attribute_ints(node, "axes") or tuple()
+
+
+class ReduceMax_18(ReduceOp):
+    """ReduceMax"""
+
+    def run(self, x: OpRunTensor, axes: Optional[OpRunTensor] = None) -> OpRunTensor:
+        assert self.stash_type is None, f"Not implemented with stash_type={self.stash_type}"
+        if axes is None:
+            assert (
+                not self.keepdims
+            ), f"axes is Empty, keepdims={self.keepdims} for {self.__class__.__name__}"
+            return OpRunTensor(x.tensor.max())
+        taxes = axes.as_tuple_int
+        if len(taxes) == 1:
+            t = x.tensor.max(taxes[0], keepdim=self.keepdims)
+            return OpRunTensor(t.values)
+        t = x.tensor
+        for a in reversed(taxes):
+            t = t.max(a, keepdim=self.keepdims).values
+        return OpRunTensor(t)
+
+
+class ReduceMean_18(ReduceOp):
+    """ReduceMean"""
+
+    def run(self, x: OpRunTensor, axes: Optional[OpRunTensor] = None) -> OpRunTensor:
+        assert self.stash_type is None, f"Not implemented with stash_type={self.stash_type}"
+        if axes is None:
+            assert (
+                not self.keepdims
+            ), f"axes is Empty, keepdims={self.keepdims} for {self.__class__.__name__}"
+            return OpRunTensor(torch.mean(x.tensor))
+        taxes = axes.as_tuple_int
+        if len(taxes) == 1:
+            t = x.tensor.mean(taxes[0], keepdim=self.keepdims)
+            return OpRunTensor(t)
+        t = x.tensor.mean(taxes, keepdim=self.keepdims)
+        return OpRunTensor(t)
+
+
+class ReduceMin_17(ReduceOpAxes):
+    """ReduceMin"""
+
+    def run(self, x: OpRunTensor) -> OpRunTensor:
+        assert self.stash_type is None, f"Not implemented with stash_type={self.stash_type}"
+        axes = self.axes
+        if not axes:
+            assert (
+                not self.keepdims
+            ), f"axes is Empty, keepdims={self.keepdims} for {self.__class__.__name__}"
+            return OpRunTensor(x.tensor.min())
+        taxes = tuple(axes)
+        if len(taxes) == 1:
+            t = x.tensor.min(taxes[0], keepdim=self.keepdims)
+            return OpRunTensor(t.values)
+        t = x.tensor
+        for a in reversed(taxes):
+            t = t.min(a, keepdim=self.keepdims).values
+        return OpRunTensor(t)
+
+
+class ReduceMin_18(ReduceOp):
+    """ReduceMin"""
+
+    def run(self, x: OpRunTensor, axes: Optional[OpRunTensor] = None) -> OpRunTensor:
+        assert self.stash_type is None, f"Not implemented with stash_type={self.stash_type}"
+        if axes is None:
+            assert (
+                not self.keepdims
+            ), f"axes is empty, keepdims={self.keepdims} for {self.__class__.__name__}"
+            return OpRunTensor(torch.min(x.tensor))
+        taxes = axes.as_tuple_int
+        if len(taxes) == 1:
+            t = x.tensor.min(taxes[0], keepdim=self.keepdims)
+            return OpRunTensor(t.values)
+        t = x.tensor
+        for a in reversed(taxes):
+            t = t.min(a, keepdim=self.keepdims).values
+        return OpRunTensor(t)
+
+
+class ReduceSum_13(ReduceOp):
+    """ReduceSum"""
+
+    def run(self, x: OpRunTensor, axes: Optional[OpRunTensor] = None) -> OpRunTensor:
+        assert self.stash_type is None, f"Not implemented with stash_type={self.stash_type}"
+        if axes is None:
+            assert (
+                not self.keepdims
+            ), f"axes is Empty, keepdims={self.keepdims} for {self.__class__.__name__}"
+            return OpRunTensor(torch.sum(x.tensor))
+        taxes = axes.as_tuple_int
+        if len(taxes) == 1:
+            t = x.tensor.sum(taxes[0], keepdim=self.keepdims)
+            return OpRunTensor(t)
+        t = x.tensor.sum(taxes, keepdim=self.keepdims)
+        return OpRunTensor(t)

onnx_diagnostic/reference/torch_ops/sequence_ops.py

@@ -0,0 +1,65 @@
+from typing import Optional
+import onnx
+import torch
+from ...helpers.torch_helper import onnx_dtype_to_torch_dtype
+from . import OpRunKernel, OpRunSequence, OpRunTensor
+
+
+class OpRunOpSequence(OpRunKernel):
+    "Ancestor for kernel using sequences."
+
+
+class ConcatFromSequence_11(OpRunOpSequence):
+    "ConcatFromSequence"
+
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        axis = self.get_attribute_int(node, "axis", None)
+        assert isinstance(axis, int), f"Unexpected value for attribute axis={axis!r}"
+        self.axis = axis
+        self.new_axis = self.get_attribute_int(node, "new_axis", 0)
+
+    def run(self, input_sequence: OpRunSequence) -> OpRunTensor:
+        assert isinstance(
+            input_sequence, OpRunSequence
+        ), f"Unexpected type {type(input_sequence)} for input_sequence"
+        seq = input_sequence.sequence
+        if self.new_axis == 1:
+            if self.axis == -1:
+                seq2 = [s.unsqueeze(len(s.shape)) for s in seq]
+                res = torch.cat(seq2, axis=-1)
+            else:
+                seq2 = [s.expand(self.axis) for s in seq]
+                res = torch.cat(seq2, axis=self.axis)
+        else:
+            res = torch.cat(seq, axis=self.axis)
+        return OpRunTensor(res)
+
+
+class SequenceEmpty_11(OpRunOpSequence):
+    "SqeuenceEmpty"
+
+    def __init__(self, node: onnx.NodeProto, version: Optional[int] = None, verbose: int = 0):
+        super().__init__(node, version, verbose=verbose)
+        self.dtype = onnx_dtype_to_torch_dtype(
+            self.get_attribute_int(node, "dtype", onnx.TensorProto.FLOAT)  # type: ignore[arg-type]
+        )
+
+    def run(self) -> OpRunSequence:
+        return OpRunSequence(dtype=self.dtype)
+
+
+class SequenceInsert_11(OpRunOpSequence):
+    "SqeuenceInsert"
+
+    def run(
+        self,
+        input_sequence: OpRunSequence,
+        tensor: OpRunTensor,
+        position: Optional[OpRunTensor] = None,
+    ) -> OpRunSequence:
+        assert isinstance(input_sequence, OpRunSequence), (
+            f"Unexpected type {type(input_sequence)} for input_sequence: "
+            f"{input_sequence.string_type()}"
+        )
+        return input_sequence.insert_at(tensor, position)