onnx 1.15.0__cp311-cp311-win_amd64.whl → 1.16.1__cp311-cp311-win_amd64.whl

onnx/test/test_backend_onnxruntime.py

@@ -4,11 +4,9 @@
 
 from __future__ import annotations
 
-import os
 import platform
-import sys
 import unittest
-from typing import Any, ClassVar
+from typing import Any
 
 import numpy
 from packaging.version import Version
@@ -17,21 +15,16 @@ import onnx.backend.base
 import onnx.backend.test
 import onnx.shape_inference
 import onnx.version_converter
-from onnx import ModelProto
 from onnx.backend.base import Device, DeviceType
 
 try:
-    from onnxruntime import InferenceSession
-    from onnxruntime import __version__ as ort_version
-    from onnxruntime import get_available_providers
-    from onnxruntime.capi.onnxruntime_pybind11_state import InvalidArgument
+    import onnxruntime as ort
+
+    ort_version = Version(ort.__version__)
 except ImportError:
     # onnxruntime is not installed, all tests are skipped.
-    InferenceSession = None
-    ort_version = None
-
-    def get_available_providers():
-        return []
+    ort: Any = None  # type: ignore[no-redef]
+    ort_version: Any = None  # type: ignore[no-redef]
 
 
 # The following just executes a backend based on InferenceSession through the backend test
@@ -42,6 +35,7 @@ class InferenceSessionBackendRep(onnx.backend.base.BackendRep):
         self._session = session
 
     def run(self, inputs, **kwargs):
+        del kwargs  # Unused
         if isinstance(inputs, numpy.ndarray):
             inputs = [inputs]
         if isinstance(inputs, list):
@@ -66,249 +60,269 @@ class InferenceSessionBackendRep(onnx.backend.base.BackendRep):
         return outs
 
 
-class InferenceSessionBackend(onnx.backend.base.Backend):
-    providers: ClassVar[set[str]] = set(get_available_providers())
+def _create_inference_session(model: onnx.ModelProto, device: str):
+    if device == "CPU":
+        providers = ("CPUExecutionProvider",)
+    elif device == "CUDA":
+        providers = ("CUDAExecutionProvider",)
+    else:
+        raise ValueError(f"Unexpected device {device!r}.")
+    try:
+        session = ort.InferenceSession(model.SerializeToString(), providers=providers)
+    except Exception as e:
+        raise RuntimeError(
+            f"Unable to create inference session. Model is:\n\n{onnx.printer.to_text(model)}"
+        ) from e
+    return session
 
-    @classmethod
-    def is_opset_supported(cls, model):
-        return True, ""
 
+class InferenceSessionBackend(onnx.backend.base.Backend):
     @classmethod
     def supports_device(cls, device: str) -> bool:
+        providers = set(ort.get_available_providers())
         d = Device(device)
-        if d.type == DeviceType.CPU and "CPUExecutionProvider" in cls.providers:
+        if d.type == DeviceType.CPU and "CPUExecutionProvider" in providers:
             return True
-        if d.type == DeviceType.CUDA and "CUDAExecutionProvider" in cls.providers:
+        if d.type == DeviceType.CUDA and "CUDAExecutionProvider" in providers:
             return True
         return False
 
-    @classmethod
-    def convert_version_opset_before(cls, model):
-        opsets = {d.domain: d.version for d in model.opset_import}
-        if "" not in opsets:
-            return None
-        try:
-            return onnx.version_converter.convert_version(model, opsets[""] - 1)
-        except RuntimeError:
-            # Let's try without any change.
-            del model.opset_import[:]
-            for k, v in opsets.items():
-                d = model.opset_import.add()
-                d.domain = k
-                d.version = v if k != "" else v - 1
-            return model
-
-    @classmethod
-    def create_inference_session(cls, model, device):
-        if device == "CPU":
-            providers = ["CPUExecutionProvider"]
-        elif device == "CUDA":
-            providers = ["CUDAExecutionProvider"]
-        else:
-            raise ValueError(f"Unexepcted device {device!r}.")
-        try:
-            return InferenceSession(model.SerializeToString(), providers=providers)
-        except InvalidArgument as e:
-            if "Unsupported model IR version" in str(e):
-                model.ir_version -= 1
-                return cls.create_inference_session(model, device)
-            if "Current official support for domain ai.onnx is till opset" in str(e):
-                new_model = cls.convert_version_opset_before(model)
-                if new_model is not None:
-                    return cls.create_inference_session(new_model, device)
-            raise e
-
     @classmethod
     def prepare(
-        cls, model: Any, device: str = "CPU", **kwargs: Any
+        cls, model: onnx.ModelProto, device: str = "CPU", **kwargs: Any
     ) -> InferenceSessionBackendRep:
-        # if isinstance(model, InferenceSessionBackendRep):
-        #    return model
-        if isinstance(model, InferenceSession):
-            return InferenceSessionBackendRep(model)
-        if isinstance(model, (str, bytes, ModelProto)):
-            inf = cls.create_inference_session(model, device)
-            return cls.prepare(inf, device, **kwargs)
-        raise TypeError(f"Unexpected type {type(model)} for model.")
+        del kwargs  # Unused
+        if not isinstance(model, (str, bytes, onnx.ModelProto)):
+            raise TypeError(f"Unexpected type {type(model)} for model.")
+
+        session = _create_inference_session(model, device)
+        return InferenceSessionBackendRep(session)
 
     @classmethod
-    def run_model(cls, model, inputs, device=None, **kwargs):
-        rep = cls.prepare(model, device, **kwargs)
-        return rep.run(inputs, **kwargs)
+    def run_model(cls, model: onnx.ModelProto, inputs, device=None, **kwargs):
+        return super().run_model(model, inputs, device=device, **kwargs)
 
     @classmethod
     def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs):
         raise NotImplementedError("Unable to run the model node by node.")
 
 
-backend_test = onnx.backend.test.BackendTest(InferenceSessionBackend, __name__)
-
-if os.getenv("APPVEYOR"):
-    backend_test.exclude("(test_vgg19|test_zfnet)")
-if platform.architecture()[0] == "32bit":
-    backend_test.exclude("(test_vgg19|test_zfnet|test_bvlc_alexnet)")
-if platform.system() == "Windows":
-    backend_test.exclude("test_sequence_model")
-
-# The following tests cannot pass because they consists in generating random number.
-backend_test.exclude("(test_bernoulli)")
-
-# The following tests are not supported by onnxruntime.
-backend_test.exclude(
-    "("
-    "test_adagrad"
-    "|test_adam"
-    "|test_add_uint8"
-    "|bitshift_left_uint16"
-    "|bitshift_right_uint16"
-    "|cast_BFLOAT16_to_FLOAT"
-    "|cast_FLOAT_to_BFLOAT16"
-    "|castlike_BFLOAT16_to_FLOAT"
-    "|castlike_FLOAT_to_BFLOAT16"
-    "|clip_default_int8_min_expanded"
-    "|clip_default_int8_max_expanded"
-    "|div_uint8"
-    "|gru_batchwise"  # Batchwise recurrent operations (layout == 1) are not supported.
-    "|loop16_seq_none"  # The graph is missing type information needed to construct the ORT tensor.
-    "|lstm_batchwise"  # Batchwise recurrent operations (layout == 1) are not supported.
-    "|m(in|ax)_u?int(16|8)"
-    "|momentum"
-    "|mul_uint8"
-    "|pow_types_float32_uint32"
-    "|pow_types_float32_uint64"
-    "|simple_rnn_batchwise"  # Batchwise recurrent operations (layout == 1) are not supported.
-    "|sub_uint8"
-    "|gradient_of_add"
-    "|test_batchnorm_epsilon_training_mode"  # Training mode does not support BN opset 14 (or higher) yet.
-    "|test_batchnorm_example_training_mode"  # Training mode does not support BN opset 14 (or higher) yet.
-    "|_to_FLOAT8E4M3FN"  # No corresponding Numpy type for Tensor Type.
-    "|_to_FLOAT8E5M2"  # No corresponding Numpy type for Tensor Type.
-    "|cast_FLOAT8E"  # No corresponding Numpy type for Tensor Type.
-    "|castlike_FLOAT8E"  # No corresponding Numpy type for Tensor Type.
-    "|test_dequantizelinear_axis"  # y_scale must be a scalar or 1D tensor of size 1.
-    "|test_dequantizelinear"  # No corresponding Numpy type for Tensor Type.
-    "|test_quantizelinear_axis"  # y_scale must be a scalar or 1D tensor of size 1.
-    "|test_quantizelinear"  # No corresponding Numpy type for Tensor Type.
-    "|test_affine_grid_"  # new IR version 9 and opset version 20 not supported yet.
-    ")"
-)
-
-# The following tests fail due to small discrepancies.
-backend_test.exclude("(cast_FLOAT_to_STRING|castlike_FLOAT_to_STRING|dft|stft)")
-
-# The following tests fail due to huge discrepancies.
-backend_test.exclude(
-    "("
-    "resize_downsample_scales_cubic_align_corners"
-    "|resize_downsample_scales_linear_align_corners"
-    "|training_dropout"
-    ")"
-)
-
-# The followiing tests fail due to a bug in onnxruntime in handling reduction
-# ops that perform reduction over an empty set of values.
-backend_test.exclude(
-    "("
-    "test_reduce_sum_empty_set"
-    "|test_reduce_prod_empty_set"
-    "|test_reduce_min_empty_set"
-    "|test_reduce_max_empty_set"
-    "|test_reduce_sum_square_empty_set"
-    "|test_reduce_log_sum_empty_set"
-    "|test_reduce_log_sum_exp_empty_set"
-    "|test_reduce_l1_empty_set"
-    "|test_reduce_l2_empty_set"
-    ")"
-)
-
-# The following tests fail for no obvious reason.
-backend_test.exclude(
-    "("
-    "maxunpool_export_with_output_shape"  # not the same expected output
-    "|softplus_example_expanded"  # Could not find an implementation for Exp(1) node with name ''
-    "|softplus_expanded"  # Could not find an implementation for Exp(1) node with name ''
-    "|AvgPool[1-3]d"  # Could not find an implementation for AveragePool(1) node with name ''
-    "|BatchNorm1d_3d_input_eval"  # Could not find an implementation for BatchNormalization(6) node with name ''
-    "|BatchNorm[2-3]d_eval"  # Could not find an implementation for BatchNormalization(6) node with name ''
-    "|GLU"  # Could not find an implementation for Mul(6) node with name ''
-    "|Linear"  # Could not find an implementation for Gemm(6) node with name ''
-    "|PReLU"  # Could not find an implementation for PRelu(6) node with name ''
-    "|PoissonNLL"  # Could not find an implementation for Mul(6) node with name ''
-    "|Softsign"  # Could not find an implementation for Gemm(6) node with name ''
-    "|operator_add_broadcast"  # Could not find an implementation for Gemm(6) node with name ''
-    "|operator_add_size1"  # Could not find an implementation for Gemm(6) node with name ''
-    "|operator_addconstant"  # Could not find an implementation for Gemm(6) node with name ''
-    "|operator_addmm"  # Could not find an implementation for Gemm(6) node with name ''
-    "|operator_basic"  # Could not find an implementation for Add(6) node with name ''
-    "|operator_mm"  # Could not find an implementation for Gemm(6) node with name ''
-    "|operator_non_float_params"  # Could not find an implementation for Add(6) node with name ''
-    "|operator_params"  # Could not find an implementation for Add(6) node with name ''
-    "|operator_pow"  # Could not find an implementation for Pow(1) node with name ''
-    ")"
-)
-
-# The following tests are new with opset 19 and 20, or ai.onnx.ml 4
-if ort_version is not None and Version(ort_version) < Version("1.16"):
-    # version should be 1.15 but there is no development version number.
+if ort is not None:
+    backend_test = onnx.backend.test.BackendTest(InferenceSessionBackend, __name__)
+
+    if platform.architecture()[0] == "32bit":
+        backend_test.exclude("(test_vgg19|test_zfnet|test_bvlc_alexnet)")
+    if platform.system() == "Windows":
+        backend_test.exclude("test_sequence_model")
+
+    # The following tests cannot pass because they consists in generating random number.
+    backend_test.exclude("(test_bernoulli)")
+
+    # The following tests are not supported by onnxruntime.
+    backend_test.exclude(
+        "("
+        "test_adagrad"
+        "|test_adam"
+        "|test_add_uint8"
+        "|bitshift_left_uint16"
+        "|bitshift_right_uint16"
+        "|cast_BFLOAT16_to_FLOAT"
+        "|cast_FLOAT_to_BFLOAT16"
+        "|castlike_BFLOAT16_to_FLOAT"
+        "|castlike_FLOAT_to_BFLOAT16"
+        "|clip_default_int8_min_expanded"
+        "|clip_default_int8_max_expanded"
+        "|div_uint8"
+        "|gru_batchwise"  # Batchwise recurrent operations (layout == 1) are not supported.
+        "|loop16_seq_none"  # The graph is missing type information needed to construct the ORT tensor.
+        "|lstm_batchwise"  # Batchwise recurrent operations (layout == 1) are not supported.
+        "|m(in|ax)_u?int(16|8)"
+        "|momentum"
+        "|mul_uint8"
+        "|pow_types_float32_uint32"
+        "|pow_types_float32_uint64"
+        "|simple_rnn_batchwise"  # Batchwise recurrent operations (layout == 1) are not supported.
+        "|sub_uint8"
+        "|gradient_of_add"
+        "|test_batchnorm_epsilon_training_mode"  # Training mode does not support BN opset 14 (or higher) yet.
+        "|test_batchnorm_example_training_mode"  # Training mode does not support BN opset 14 (or higher) yet.
+        "|_to_FLOAT8E4M3FN"  # No corresponding Numpy type for Tensor Type.
+        "|_to_FLOAT8E5M2"  # No corresponding Numpy type for Tensor Type.
+        "|cast_FLOAT8E"  # No corresponding Numpy type for Tensor Type.
+        "|castlike_FLOAT8E"  # No corresponding Numpy type for Tensor Type.
+        "|test_dequantizelinear_axis"  # y_scale must be a scalar or 1D tensor of size 1.
+        "|test_dequantizelinear"  # No corresponding Numpy type for Tensor Type.
+        "|test_quantizelinear_axis"  # y_scale must be a scalar or 1D tensor of size 1.
+        "|test_quantizelinear"  # No corresponding Numpy type for Tensor Type.
+        "|test_affine_grid_"  # new IR version 9 and opset version 20 not supported yet.
+        "|test_quantizelinear_uint4"  # No corresponding Numpy type for Tensor Type.
+        "|test_quantizelinear_int4"  # No corresponding Numpy type for Tensor Type.
+        "|test_dequantizelinear_uint4"  # No corresponding Numpy type for Tensor Type.
+        "|test_dequantizelinear_int4"  # No corresponding Numpy type for Tensor Type.
+        "|test_cast_UINT4_to_FLOAT"  # No corresponding Numpy type for Tensor Type.
+        "|test_cast_INT4_to_FLOAT"  # No corresponding Numpy type for Tensor Type.
+        "|test_cast_UINT4_to_FLOAT16"  # No corresponding Numpy type for Tensor Type.
+        "|test_cast_INT4_to_FLOAT16"  # No corresponding Numpy type for Tensor Type.
+        "|test_maxpool_2d_ceil_output_size_reduce_by_one"  # TODO: remove after https://github.com/microsoft/onnxruntime/pull/18377 in Ort release.
+        ")"
+    )
+
+    # Exclude all tests that require IR10 until onnxruntime aligns
+    # TODO: Unwaive tests once onnxruntime supports Opset21/IR10 https://github.com/onnx/onnx/issues/5840
     backend_test.exclude(
         "("
-        "averagepool"
-        "|_pad_"
-        "|_resize_"
-        "|_size_"
-        "|cast"
-        "|castlike"
-        "|equal_string_broadcast"
-        "|equal_string"
-        "|equal"
-        "|half_pixel_symmetric"
-        "|identity"
-        "|reshape"
+        "test_cast_"
+        "|test_castlike_"
+        "|test_constant"
+        "|test_edge_pad_cpu"
+        "|test_flatten_"
+        "|test_identity"
+        "|test_reflect_pad"
+        "|test_reshape_"
+        "|test_shape_"
+        "|test_size_"
+        "|test_squeeze_"
+        "|test_transpose_"
+        "|test_unsqueeze_"
+        "|test_wrap_pad_"
         ")"
     )
 
-if ort_version is not None and Version(ort_version) < Version("1.17"):
-    # version should be 1.15 but there is no development version number.
+    # The following tests fail due to small discrepancies.
+    backend_test.exclude("(cast_FLOAT_to_STRING|castlike_FLOAT_to_STRING|stft)")
+
+    # The following tests fail due to huge discrepancies.
     backend_test.exclude(
         "("
-        "deform_conv"
-        "|dft"
-        "|gelu"
-        "|gridsample"
-        "|identity_opt"
-        "|image_decoder"
-        "|isinf_float16"
-        "|label_encoder"
-        "|optional_get_element_optional_sequence"
-        "|reduce_max_bool_inputs"
-        "|reduce_min_bool_inputs"
-        "|regex_full_match"
-        "|string_concat"
-        "|string_split"
+        "resize_downsample_scales_cubic_align_corners"
+        "|resize_downsample_scales_linear_align_corners"
+        "|training_dropout"
         ")"
     )
 
-if sys.version_info[:2] < (3, 8) or Version(numpy.__version__) < Version("1.23.5"):
-    # Version 1.21.5 causes segmentation faults.
-    # onnxruntime should be tested with the same numpy API
-    # onnxruntime was compiled with.
-    backend_test.exclude("")
+    # The followiing tests fail due to a bug in onnxruntime in handling reduction
+    # ops that perform reduction over an empty set of values.
+    backend_test.exclude(
+        "("
+        "test_reduce_sum_empty_set"
+        "|test_reduce_prod_empty_set"
+        "|test_reduce_min_empty_set"
+        "|test_reduce_max_empty_set"
+        "|test_reduce_sum_square_empty_set"
+        "|test_reduce_log_sum_empty_set"
+        "|test_reduce_log_sum_exp_empty_set"
+        "|test_reduce_l1_empty_set"
+        "|test_reduce_l2_empty_set"
+        ")"
+    )
 
-# import all test cases at global scope to make them visible to python.unittest
-if InferenceSession is not None:
+    # The following tests fail for no obvious reason.
+    backend_test.exclude(
+        "("
+        "maxunpool_export_with_output_shape"  # not the same expected output
+        "|softplus_example_expanded"  # Could not find an implementation for Exp(1) node with name ''
+        "|softplus_expanded"  # Could not find an implementation for Exp(1) node with name ''
+        "|AvgPool[1-3]d"  # Could not find an implementation for AveragePool(1) node with name ''
+        "|BatchNorm1d_3d_input_eval"  # Could not find an implementation for BatchNormalization(6) node with name ''
+        "|BatchNorm[2-3]d_eval"  # Could not find an implementation for BatchNormalization(6) node with name ''
+        "|GLU"  # Could not find an implementation for Mul(6) node with name ''
+        "|Linear"  # Could not find an implementation for Gemm(6) node with name ''
+        "|PReLU"  # Could not find an implementation for PRelu(6) node with name ''
+        "|PoissonNLL"  # Could not find an implementation for Mul(6) node with name ''
+        "|Softsign"  # Could not find an implementation for Gemm(6) node with name ''
+        "|operator_add_broadcast"  # Could not find an implementation for Gemm(6) node with name ''
+        "|operator_add_size1"  # Could not find an implementation for Gemm(6) node with name ''
+        "|operator_addconstant"  # Could not find an implementation for Gemm(6) node with name ''
+        "|operator_addmm"  # Could not find an implementation for Gemm(6) node with name ''
+        "|operator_basic"  # Could not find an implementation for Add(6) node with name ''
+        "|operator_mm"  # Could not find an implementation for Gemm(6) node with name ''
+        "|operator_non_float_params"  # Could not find an implementation for Add(6) node with name ''
+        "|operator_params"  # Could not find an implementation for Add(6) node with name ''
+        "|operator_pow"  # Could not find an implementation for Pow(1) node with name ''
+        ")"
+    )
+
+    # The following tests are new with opset 19 and 20, or ai.onnx.ml 4
+    if ort_version is not None and ort_version < Version("1.16"):
+        backend_test.exclude(
+            "("
+            "averagepool"
+            "|_pad_"
+            "|_resize_"
+            "|_size_"
+            "|cast"
+            "|castlike"
+            "|equal_string_broadcast"
+            "|equal_string"
+            "|equal"
+            "|half_pixel_symmetric"
+            "|identity"
+            "|reshape"
+            ")"
+        )
+    if ort_version is not None and ort_version < Version("1.17"):
+        backend_test.exclude(
+            "("
+            "deform_conv"
+            "|dequantizelinear_uint16"
+            "|dequantizelinear_int16"
+            "|quantizelinear_uint16"
+            "|quantizelinear_int16"
+            "|dft"
+            "|gelu"
+            "|gridsample"
+            "|group_normalization"
+            "|identity_opt"
+            "|image_decoder"
+            "|isinf_float16"
+            "|label_encoder"
+            "|optional_get_element_optional_sequence"
+            "|qlinearmatmul_2D_int8"
+            "|qlinearmatmul_2D_uint8_float16"
+            "|qlinearmatmul_3D_int8"
+            "|qlinearmatmul_3D_uint8_float16"
+            "|reduce_max_bool_inputs"
+            "|reduce_min_bool_inputs"
+            "|regex_full_match"
+            "|string_concat"
+            "|string_split"
+            "|constantofshape_float_ones"
+            "|constantofshape_int_shape_zero"
+            "|constantofshape_int_zeros"
+            "|isinf"
+            "|isinf_negative"
+            "|isinf_positive"
+            "|isnan"
+            "|isnan_float16"
+            "|qlinearmatmul_2D_uint8_float32"
+            "|qlinearmatmul_3D_uint8_float32"
+            ")"
+        )
+    if ort_version is not None and ort_version < Version("1.18"):
+        # when adding new tests to the list, please add a comment with the reason for exclusion
+        # for tests that "not supported by onnxruntime 1.17", it will be solved in the next
+        # onnxruntime release with ONNX 1.16.0 integrated. The work is covered in ONNX integration procedure.
+        backend_test.exclude(
+            "("
+            "deform_conv"  # deform_conv is not supported in onnxruntime
+            "|dft"  # Max absolute difference > atol=1e-07. shall be able to set atol (https://github.com/onnx/onnx/issues/5897)
+            "|group_normalization"  # new/updated test cases with opset and/or IR version not supported by onnxruntime 1.17
+            "|identity_opt"  # fixed in ort 1.18 (https://github.com/microsoft/onnxruntime/pull/19273)
+            "|image_decoder"  # image_decoder is not supported in onnxruntime
+            "|optional_get_element_optional_sequence"  # fixed in ort 1.18 (https://github.com/microsoft/onnxruntime/pull/19273)
+            "|qlinearmatmul_2D_int8"  # new/updated test cases with opset and/or IR version not supported by onnxruntime 1.17
+            "|qlinearmatmul_2D_uint8_float16"  # new/updated test cases with opset and/or IR version not supported by onnxruntime 1.17
+            "|qlinearmatmul_3D_int8"  # new/updated test cases with opset and/or IR version not supported by onnxruntime 1.17
+            "|qlinearmatmul_3D_uint8_float16"  # new/updated test cases with opset and/or IR version not supported by onnxruntime 1.17
+            "|qlinearmatmul_2D_uint8_float32"  # new/updated test cases with opset and/or IR version not supported by onnxruntime 1.17
+            "|qlinearmatmul_3D_uint8_float32"  # new/updated test cases with opset and/or IR version not supported by onnxruntime 1.17
+            "|tree_ensemble"  # tree_ensemble not yet implemented in ort
+            ")"
+        )
+
+    # Import all test cases at global scope to make them visible to python.unittest
     globals().update(backend_test.test_cases)
 
 
 if __name__ == "__main__":
-    res = unittest.main(verbosity=2, exit=False)
-    tests_run = res.result.testsRun
-    errors = len(res.result.errors)
-    skipped = len(res.result.skipped)
-    unexpected_successes = len(res.result.unexpectedSuccesses)
-    expected_failures = len(res.result.expectedFailures)
-    print("---------------------------------")
-    print(
-        f"tests_run={tests_run} errors={errors} skipped={skipped} "
-        f"unexpected_successes={unexpected_successes} "
-        f"expected_failures={expected_failures}"
-    )
+    unittest.main()

onnx/test/test_backend_reference.py

@@ -120,12 +120,18 @@ backend_test.exclude(
     "|test_cast_FLOAT16_to_FLOAT8"
     "|test_castlike_FLOAT_to_FLOAT8"
     "|test_castlike_FLOAT16_to_FLOAT8"
+    "|test_cast_FLOAT_to_UINT4"
+    "|test_cast_FLOAT16_to_UINT4"
+    "|test_cast_FLOAT_to_INT4"
+    "|test_cast_FLOAT16_to_INT4"
     "|test_cast_no_saturate_FLOAT_to_FLOAT8"
     "|test_cast_no_saturate_FLOAT16_to_FLOAT8"
     "|test_cast_BFLOAT16_to_FLOAT"
     "|test_castlike_BFLOAT16_to_FLOAT"
     "|test_quantizelinear_e4m3"
     "|test_quantizelinear_e5m2"
+    "|test_quantizelinear_uint4"
+    "|test_quantizelinear_int4"
     ")"
 )
 
@@ -181,6 +187,11 @@ if sys.platform == "win32":
     backend_test.exclude("test_regex_full_match_empty_cpu")
     backend_test.exclude("test_image_decoder_decode_")
 
+if sys.platform == "darwin":
+    # FIXME: https://github.com/onnx/onnx/issues/5792
+    backend_test.exclude("test_qlinearmatmul_3D_int8_float16_cpu")
+    backend_test.exclude("test_qlinearmatmul_3D_int8_float32_cpu")
+
 # op_dft and op_stft requires numpy >= 1.21.5
 if version_utils.numpy_older_than("1.21.5"):
     backend_test.exclude("test_stft")

onnx/test/test_external_data.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
+import itertools
 import os
 import pathlib
 import tempfile
@@ -204,6 +205,52 @@ class TestLoadExternalDataSingleFile(TestLoadExternalDataBase):
         attribute_tensor = new_model.graph.node[0].attribute[0].t
         np.testing.assert_allclose(to_array(attribute_tensor), self.attribute_value)
 
+    @parameterized.parameterized.expand(itertools.product((True, False), (True, False)))
+    def test_save_external_invalid_single_file_data_and_check(
+        self, use_absolute_path: bool, use_model_path: bool
+    ) -> None:
+        model = onnx.load_model(self.model_filename, self.serialization_format)
+
+        model_dir = os.path.join(self.temp_dir, "save_copy")
+        os.mkdir(model_dir)
+
+        traversal_external_data_dir = os.path.join(
+            self.temp_dir, "invlid_external_data"
+        )
+        os.mkdir(traversal_external_data_dir)
+
+        if use_absolute_path:
+            traversal_external_data_location = os.path.join(
+                traversal_external_data_dir, "tensors.bin"
+            )
+        else:
+            traversal_external_data_location = "../invlid_external_data/tensors.bin"
+
+        external_data_dir = os.path.join(self.temp_dir, "external_data")
+        os.mkdir(external_data_dir)
+        new_model_filepath = os.path.join(model_dir, "model.onnx")
+
+        def convert_model_to_external_data_no_check(model: ModelProto, location: str):
+            for tensor in model.graph.initializer:
+                if tensor.HasField("raw_data"):
+                    set_external_data(tensor, location)
+
+        convert_model_to_external_data_no_check(
+            model,
+            location=traversal_external_data_location,
+        )
+
+        onnx.save_model(model, new_model_filepath, self.serialization_format)
+        if use_model_path:
+            with self.assertRaises(onnx.checker.ValidationError):
+                _ = onnx.load_model(new_model_filepath, self.serialization_format)
+        else:
+            onnx_model = onnx.load_model(
+                new_model_filepath, self.serialization_format, load_external_data=False
+            )
+            with self.assertRaises(onnx.checker.ValidationError):
+                load_external_data_for_model(onnx_model, external_data_dir)
+
 
 @parameterized.parameterized_class(
     [
@@ -674,7 +721,8 @@ class TestExternalDataToArray(unittest.TestCase):
 
 class TestNotAllowToLoadExternalDataOutsideModelDirectory(TestLoadExternalDataBase):
     """Essential test to check that onnx (validate) C++ code will not allow to load external_data outside the model
-    directory."""
+    directory.
+    """
 
     def create_external_data_tensor(
         self, value: list[Any], tensor_name: str, location: str = ""
@@ -713,7 +761,8 @@ class TestNotAllowToLoadExternalDataOutsideModelDirectoryOnWindows(
     TestNotAllowToLoadExternalDataOutsideModelDirectory
 ):
     """Essential test to check that onnx (validate) C++ code will not allow to load external_data outside the model
-    directory."""
+    directory.
+    """
 
     def test_check_model(self) -> None:
         """We only test the model validation as onnxruntime uses this to load the model."""

onnx/test/version_converter/automatic_conversion_test_base.py

@@ -53,7 +53,8 @@ class TestAutomaticConversion(unittest.TestCase):
         optional_outputs: Sequence[int] = (),
         is_upgrade: bool = True,
     ) -> None:
-        """
+        """Test conversion.
+
         Args:
             op: A string representing the name of the operator to test.
             from_opset: An integer representing the lowest opset version to convert.