onnxruntime-directml 1.20.0__cp313-cp313-win_amd64.whl

onnxruntime/tools/ort_format_model/ort_flatbuffers_py/fbs/TensorTypeAndShape.py

@@ -0,0 +1,71 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: fbs
+
+import flatbuffers
+from flatbuffers.compat import import_numpy
+np = import_numpy()
+
+class TensorTypeAndShape(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAs(cls, buf, offset=0):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = TensorTypeAndShape()
+        x.Init(buf, n + offset)
+        return x
+
+    @classmethod
+    def GetRootAsTensorTypeAndShape(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
+    @classmethod
+    def TensorTypeAndShapeBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4F\x52\x54\x4D", size_prefixed=size_prefixed)
+
+    # TensorTypeAndShape
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # TensorTypeAndShape
+    def ElemType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Int32Flags, o + self._tab.Pos)
+        return 0
+
+    # TensorTypeAndShape
+    def Shape(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            x = self._tab.Indirect(o + self._tab.Pos)
+            from ort_flatbuffers_py.fbs.Shape import Shape
+            obj = Shape()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+def TensorTypeAndShapeStart(builder):
+    builder.StartObject(2)
+
+def Start(builder):
+    TensorTypeAndShapeStart(builder)
+
+def TensorTypeAndShapeAddElemType(builder, elemType):
+    builder.PrependInt32Slot(0, elemType, 0)
+
+def AddElemType(builder, elemType):
+    TensorTypeAndShapeAddElemType(builder, elemType)
+
+def TensorTypeAndShapeAddShape(builder, shape):
+    builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(shape), 0)
+
+def AddShape(builder, shape):
+    TensorTypeAndShapeAddShape(builder, shape)
+
+def TensorTypeAndShapeEnd(builder):
+    return builder.EndObject()
+
+def End(builder):
+    return TensorTypeAndShapeEnd(builder)

onnxruntime/tools/ort_format_model/ort_flatbuffers_py/fbs/TypeInfo.py

@@ -0,0 +1,83 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: fbs
+
+import flatbuffers
+from flatbuffers.compat import import_numpy
+np = import_numpy()
+
+class TypeInfo(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAs(cls, buf, offset=0):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = TypeInfo()
+        x.Init(buf, n + offset)
+        return x
+
+    @classmethod
+    def GetRootAsTypeInfo(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
+    @classmethod
+    def TypeInfoBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4F\x52\x54\x4D", size_prefixed=size_prefixed)
+
+    # TypeInfo
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # TypeInfo
+    def Denotation(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # TypeInfo
+    def ValueType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # TypeInfo
+    def Value(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+def TypeInfoStart(builder):
+    builder.StartObject(3)
+
+def Start(builder):
+    TypeInfoStart(builder)
+
+def TypeInfoAddDenotation(builder, denotation):
+    builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(denotation), 0)
+
+def AddDenotation(builder, denotation):
+    TypeInfoAddDenotation(builder, denotation)
+
+def TypeInfoAddValueType(builder, valueType):
+    builder.PrependUint8Slot(1, valueType, 0)
+
+def AddValueType(builder, valueType):
+    TypeInfoAddValueType(builder, valueType)
+
+def TypeInfoAddValue(builder, value):
+    builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(value), 0)
+
+def AddValue(builder, value):
+    TypeInfoAddValue(builder, value)
+
+def TypeInfoEnd(builder):
+    return builder.EndObject()
+
+def End(builder):
+    return TypeInfoEnd(builder)

onnxruntime/tools/ort_format_model/ort_flatbuffers_py/fbs/TypeInfoValue.py

@@ -0,0 +1,9 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: fbs
+
+class TypeInfoValue(object):
+    NONE = 0
+    tensor_type = 1
+    sequence_type = 2
+    map_type = 3

onnxruntime/tools/ort_format_model/ort_flatbuffers_py/fbs/ValueInfo.py

@@ -0,0 +1,84 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: fbs
+
+import flatbuffers
+from flatbuffers.compat import import_numpy
+np = import_numpy()
+
+class ValueInfo(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAs(cls, buf, offset=0):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = ValueInfo()
+        x.Init(buf, n + offset)
+        return x
+
+    @classmethod
+    def GetRootAsValueInfo(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
+    @classmethod
+    def ValueInfoBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4F\x52\x54\x4D", size_prefixed=size_prefixed)
+
+    # ValueInfo
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # ValueInfo
+    def Name(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ValueInfo
+    def DocString(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ValueInfo
+    def Type(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            x = self._tab.Indirect(o + self._tab.Pos)
+            from ort_flatbuffers_py.fbs.TypeInfo import TypeInfo
+            obj = TypeInfo()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+def ValueInfoStart(builder):
+    builder.StartObject(3)
+
+def Start(builder):
+    ValueInfoStart(builder)
+
+def ValueInfoAddName(builder, name):
+    builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0)
+
+def AddName(builder, name):
+    ValueInfoAddName(builder, name)
+
+def ValueInfoAddDocString(builder, docString):
+    builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(docString), 0)
+
+def AddDocString(builder, docString):
+    ValueInfoAddDocString(builder, docString)
+
+def ValueInfoAddType(builder, type):
+    builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(type), 0)
+
+def AddType(builder, type):
+    ValueInfoAddType(builder, type)
+
+def ValueInfoEnd(builder):
+    return builder.EndObject()
+
+def End(builder):
+    return ValueInfoEnd(builder)

onnxruntime/tools/ort_format_model/ort_flatbuffers_py/fbs/__init__.py

@@ -0,0 +1,6 @@
+from os.path import dirname, basename, isfile, join, splitext
+import glob
+modules = glob.glob(join(dirname(__file__), "*.py"))
+__all__ = [splitext(basename(f))[0] for f in modules if isfile(f) and not f.endswith('__init__.py')]
+
+from . import *

onnxruntime/tools/ort_format_model/ort_model_processor.py

@@ -0,0 +1,86 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import ort_flatbuffers_py.fbs as fbs
+
+from .operator_type_usage_processors import OperatorTypeUsageManager
+
+
+class OrtFormatModelProcessor:
+    "Class to process an ORT format model and determine required operators and types."
+
+    def __init__(self, model_path: str, required_ops: dict, processors: OperatorTypeUsageManager):
+        """
+        Initialize ORT format model processor
+        :param model_path: Path to model to load
+        :param required_ops: Dictionary required operator information will be added to.
+        :param processors: Operator type usage processors which will be called for each matching Node.
+        """
+        self._required_ops = required_ops  # dictionary of {domain: {opset:[operators]}}
+        self._file = open(model_path, "rb").read()  # noqa: SIM115
+        self._buffer = bytearray(self._file)
+        if not fbs.InferenceSession.InferenceSession.InferenceSessionBufferHasIdentifier(self._buffer, 0):
+            raise RuntimeError(f"File does not appear to be a valid ORT format model: '{model_path}'")
+        self._model = fbs.InferenceSession.InferenceSession.GetRootAsInferenceSession(self._buffer, 0).Model()
+        self._op_type_processors = processors
+
+    @staticmethod
+    def _setup_type_info(graph: fbs.Graph, outer_scope_value_typeinfo={}):  # noqa: B006
+        """
+        Setup the node args for this level of Graph.
+        We copy the current list which represents the outer scope values, and add the local node args to that
+        to create the valid list of values for the current Graph.
+        :param graph: Graph to create NodeArg list for
+        :param outer_scope_value_typeinfo: TypeInfo for outer scope values. Empty for the top-level graph in a model.
+        :return: Dictionary of NodeArg name to TypeInfo
+        """
+        value_name_to_typeinfo = outer_scope_value_typeinfo.copy()
+        for j in range(graph.NodeArgsLength()):
+            n = graph.NodeArgs(j)
+            value_name_to_typeinfo[n.Name()] = n.Type()  # TypeInfo for this NodeArg's name
+
+        return value_name_to_typeinfo
+
+    def _add_required_op(self, domain: str, opset: int, op_type: str):
+        if domain not in self._required_ops:
+            self._required_ops[domain] = {opset: {op_type}}
+        elif opset not in self._required_ops[domain]:
+            self._required_ops[domain][opset] = {op_type}
+        else:
+            self._required_ops[domain][opset].add(op_type)
+
+    def _process_graph(self, graph: fbs.Graph, outer_scope_value_typeinfo: dict):
+        """
+        Process one level of the Graph, descending into any subgraphs when they are found
+        :param outer_scope_value_typeinfo: Outer scope NodeArg dictionary from ancestor graphs
+        """
+        # Merge the TypeInfo for all values in this level of the graph with the outer scope value TypeInfo.
+        value_name_to_typeinfo = OrtFormatModelProcessor._setup_type_info(graph, outer_scope_value_typeinfo)
+
+        for i in range(graph.NodesLength()):
+            node = graph.Nodes(i)
+
+            optype = node.OpType().decode()
+            domain = node.Domain().decode() or "ai.onnx"  # empty domain defaults to ai.onnx
+
+            self._add_required_op(domain, node.SinceVersion(), optype)
+
+            if self._op_type_processors:
+                self._op_type_processors.process_node(node, value_name_to_typeinfo)
+
+            # Read all the attributes
+            for j in range(node.AttributesLength()):
+                attr = node.Attributes(j)
+                attr_type = attr.Type()
+                if attr_type == fbs.AttributeType.AttributeType.GRAPH:
+                    self._process_graph(attr.G(), value_name_to_typeinfo)
+                elif attr_type == fbs.AttributeType.AttributeType.GRAPHS:
+                    # the ONNX spec doesn't currently define any operators that have multiple graphs in an attribute
+                    # so entering this 'elif' isn't currently possible
+                    for k in range(attr.GraphsLength()):
+                        self._process_graph(attr.Graphs(k), value_name_to_typeinfo)
+
+    def process(self):
+        graph = self._model.Graph()
+        outer_scope_value_typeinfo = {}  # no outer scope values for the main graph
+        self._process_graph(graph, outer_scope_value_typeinfo)

onnxruntime/tools/ort_format_model/types.py

@@ -0,0 +1,84 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import ort_flatbuffers_py.fbs as fbs
+
+
+class FbsTypeInfo:
+    "Class to provide conversion between ORT flatbuffers schema values and C++ types"
+    tensordatatype_to_string = {  # noqa: RUF012
+        fbs.TensorDataType.TensorDataType.FLOAT: "float",
+        fbs.TensorDataType.TensorDataType.UINT8: "uint8_t",
+        fbs.TensorDataType.TensorDataType.INT8: "int8_t",
+        fbs.TensorDataType.TensorDataType.UINT16: "uint16_t",
+        fbs.TensorDataType.TensorDataType.INT16: "int16_t",
+        fbs.TensorDataType.TensorDataType.INT32: "int32_t",
+        fbs.TensorDataType.TensorDataType.INT64: "int64_t",
+        fbs.TensorDataType.TensorDataType.STRING: "std::string",
+        fbs.TensorDataType.TensorDataType.BOOL: "bool",
+        fbs.TensorDataType.TensorDataType.FLOAT16: "MLFloat16",
+        fbs.TensorDataType.TensorDataType.DOUBLE: "double",
+        fbs.TensorDataType.TensorDataType.UINT32: "uint32_t",
+        fbs.TensorDataType.TensorDataType.UINT64: "uint64_t",
+        # fbs.TensorDataType.TensorDataType.COMPLEX64: 'complex64 is not supported',
+        # fbs.TensorDataType.TensorDataType.COMPLEX128: 'complex128 is not supported',
+        fbs.TensorDataType.TensorDataType.BFLOAT16: "BFloat16",
+        fbs.TensorDataType.TensorDataType.FLOAT8E4M3FN: "Float8E4M3FN",
+        fbs.TensorDataType.TensorDataType.FLOAT8E4M3FNUZ: "Float8E4M3FNUZ",
+        fbs.TensorDataType.TensorDataType.FLOAT8E5M2: "Float8E5M2",
+        fbs.TensorDataType.TensorDataType.FLOAT8E5M2FNUZ: "Float8E5M2FNUZ",
+    }
+
+    @staticmethod
+    def typeinfo_to_str(type: fbs.TypeInfo):
+        value_type = type.ValueType()
+        value = type.Value()
+        type_str = "unknown"
+
+        if value_type == fbs.TypeInfoValue.TypeInfoValue.tensor_type:
+            tensor_type_and_shape = fbs.TensorTypeAndShape.TensorTypeAndShape()
+            tensor_type_and_shape.Init(value.Bytes, value.Pos)
+            elem_type = tensor_type_and_shape.ElemType()
+            type_str = FbsTypeInfo.tensordatatype_to_string[elem_type]
+
+        elif value_type == fbs.TypeInfoValue.TypeInfoValue.map_type:
+            map_type = fbs.MapType.MapType()
+            map_type.init(value.Bytes, value.Pos)
+            key_type = map_type.KeyType()  # TensorDataType
+            key_type_str = FbsTypeInfo.tensordatatype_to_string[key_type]
+            value_type = map_type.ValueType()  # TypeInfo
+            value_type_str = FbsTypeInfo.typeinfo_to_str(value_type)
+            type_str = f"std::map<{key_type_str},{value_type_str}>"
+
+        elif value_type == fbs.TypeInfoValue.TypeInfoValue.sequence_type:
+            sequence_type = fbs.SequenceType.SequenceType()
+            sequence_type.Init(value.Bytes, value.Pos)
+            elem_type = sequence_type.ElemType()  # TypeInfo
+            elem_type_str = FbsTypeInfo.typeinfo_to_str(elem_type)
+            # TODO: Decide if we need to wrap the type in a std::vector. Issue is that the element type is internal
+            # to the onnxruntime::Tensor class so we're really returning the type inside the Tensor not vector<Tensor>.
+            # For now, return the element type (which will be the Tensor element type, or a map<A,B>) as
+            # an operator input or output will either be a sequence or a not, so we don't need to disambiguate
+            # between the two (i.e. we know if the returned value refers to the contents of a sequence, and can
+            # handle whether it's the element type of a Tensor in the sequence, or the map type in a sequence of maps
+            # due to this).
+            type_str = elem_type_str
+        else:
+            raise ValueError(f"Unknown or missing value type of {value_type}")
+
+        return type_str
+
+
+def get_typeinfo(name: str, value_name_to_typeinfo: dict) -> fbs.TypeInfo:
+    "Lookup a name in a dictionary mapping value name to TypeInfo."
+    if name not in value_name_to_typeinfo:
+        raise RuntimeError("Missing TypeInfo entry for " + name)
+
+    return value_name_to_typeinfo[name]  # TypeInfo object
+
+
+def value_name_to_typestr(name: str, value_name_to_typeinfo: dict):
+    "Lookup TypeInfo for value name and convert to a string representing the C++ type."
+    type = get_typeinfo(name, value_name_to_typeinfo)
+    type_str = FbsTypeInfo.typeinfo_to_str(type)
+    return type_str

onnxruntime/tools/ort_format_model/utils.py

@@ -0,0 +1,62 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import pathlib
+import typing
+
+from ..logger import get_logger
+from .operator_type_usage_processors import OperatorTypeUsageManager
+from .ort_model_processor import OrtFormatModelProcessor
+
+log = get_logger("ort_format_model.utils")
+
+
+def _extract_ops_and_types_from_ort_models(model_files: typing.Iterable[pathlib.Path], enable_type_reduction: bool):
+    required_ops = {}
+    op_type_usage_manager = OperatorTypeUsageManager() if enable_type_reduction else None
+
+    for model_file in model_files:
+        if not model_file.is_file():
+            raise ValueError(f"Path is not a file: '{model_file}'")
+        model_processor = OrtFormatModelProcessor(str(model_file), required_ops, op_type_usage_manager)
+        model_processor.process()  # this updates required_ops and op_type_processors
+
+    return required_ops, op_type_usage_manager
+
+
+def create_config_from_models(
+    model_files: typing.Iterable[pathlib.Path], output_file: pathlib.Path, enable_type_reduction: bool
+):
+    """
+    Create a configuration file with required operators and optionally required types.
+    :param model_files: Model files to use to generate the configuration file.
+    :param output_file: File to write configuration to.
+    :param enable_type_reduction: Include required type information for individual operators in the configuration.
+    """
+
+    required_ops, op_type_processors = _extract_ops_and_types_from_ort_models(model_files, enable_type_reduction)
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(output_file, "w") as out:
+        out.write("# Generated from model/s:\n")
+        for model_file in sorted(model_files):
+            out.write(f"# - {model_file}\n")
+
+        for domain in sorted(required_ops.keys()):
+            for opset in sorted(required_ops[domain].keys()):
+                ops = required_ops[domain][opset]
+                if ops:
+                    out.write(f"{domain};{opset};")
+                    if enable_type_reduction:
+                        # type string is empty if op hasn't been seen
+                        entries = [
+                            "{}{}".format(op, op_type_processors.get_config_entry(domain, op) or "")
+                            for op in sorted(ops)
+                        ]
+                    else:
+                        entries = sorted(ops)
+
+                    out.write("{}\n".format(",".join(entries)))
+
+    log.info("Created config in %s", output_file)

onnxruntime/tools/pytorch_export_contrib_ops.py

@@ -0,0 +1,108 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+Support for registering ONNX Runtime's built-in contrib ops with
+PyTorch-ONNX exporter (torch.onnx.export).
+"""
+import typing
+
+try:
+    # TODO(justinchuby): Create a function to alert users when torch is not installed
+    import torch
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(  # noqa: B904
+        "This module is only useful in combination with PyTorch. To install PyTorch see https://pytorch.org/."
+    )
+
+from torch.onnx import symbolic_helper
+
+_OPSET_VERSION = 1
+_registered_ops: typing.AbstractSet[str] = set()
+
+
+def _reg(symbolic_fn: typing.Callable):
+    name = f"::{symbolic_fn.__name__}"
+    torch.onnx.register_custom_op_symbolic(name, symbolic_fn, _OPSET_VERSION)
+    _registered_ops.add(name)
+
+
+def register():
+    """Register ONNX Runtime's built-in contrib ops.
+
+    Should be run before torch.onnx.export().
+    """
+
+    def grid_sampler(g, input, grid, mode, padding_mode, align_corners):
+        # mode
+        #   'bilinear'      : onnx::Constant[value={0}]
+        #   'nearest'       : onnx::Constant[value={1}]
+        #   'bicubic'       : onnx::Constant[value={2}]
+        # padding_mode
+        #   'zeros'         : onnx::Constant[value={0}]
+        #   'border'        : onnx::Constant[value={1}]
+        #   'reflection'    : onnx::Constant[value={2}]
+        mode = symbolic_helper._maybe_get_const(mode, "i")
+        padding_mode = symbolic_helper._maybe_get_const(padding_mode, "i")
+        mode_str = ["bilinear", "nearest", "bicubic"][mode]
+        padding_mode_str = ["zeros", "border", "reflection"][padding_mode]
+        align_corners = int(symbolic_helper._maybe_get_const(align_corners, "b"))
+
+        # From opset v13 onward, the output shape can be specified with
+        # (N, C, H, W) (N, H_out, W_out, 2) => (N, C, H_out, W_out)
+        # input_shape = input.type().sizes()
+        # gird_shape = grid.type().sizes()
+        # output_shape = input_shape[:2] + gird_shape[1:3]
+        # g.op(...).setType(input.type().with_sizes(output_shape))
+
+        return g.op(
+            "com.microsoft::GridSample",
+            input,
+            grid,
+            mode_s=mode_str,
+            padding_mode_s=padding_mode_str,
+            align_corners_i=align_corners,
+        )
+
+    _reg(grid_sampler)
+
+    def inverse(g, self):
+        return g.op("com.microsoft::Inverse", self).setType(self.type())
+
+    _reg(inverse)
+
+    @torch.onnx.symbolic_helper.parse_args("v", "s")
+    def gelu(g, self: torch._C.Value, approximate: str = "none"):
+        # Use microsoft::Gelu for performance if possible. It only supports approximate == "none"
+        if approximate == "none":
+            return g.op("com.microsoft::Gelu", self).setType(self.type())
+        return torch.onnx.symbolic_opset9.gelu(g, self, approximate)
+
+    _reg(gelu)
+
+    def triu(g, self, diagonal):
+        return g.op("com.microsoft::Trilu", self, diagonal, upper_i=1).setType(self.type())
+
+    _reg(triu)
+
+    def tril(g, self, diagonal):
+        return g.op("com.microsoft::Trilu", self, diagonal, upper_i=0).setType(self.type())
+
+    _reg(tril)
+
+
+def unregister():
+    """Unregister ONNX Runtime's built-in contrib ops."""
+    for name in _registered_ops:
+        try:
+            torch.onnx.unregister_custom_op_symbolic(name, _OPSET_VERSION)
+        except AttributeError:
+            # The symbolic_registry module was removed in PyTorch 1.13.
+            # We are importing it here for backwards compatibility
+            # because unregister_custom_op_symbolic is not available before PyTorch 1.12
+            from torch.onnx import symbolic_registry
+
+            namespace, kind = name.split("::")
+            for version in symbolic_helper._onnx_stable_opsets:
+                if version >= _OPSET_VERSION and symbolic_registry.is_registered_op(kind, namespace, version):
+                    del symbolic_registry._registry[(namespace, version)][kind]