onnx-diagnostic 0.6.3__py3-none-any.whl → 0.7.1__py3-none-any.whl

onnx_diagnostic/torch_models/{test_helper.py → validate.py}

@@ -147,7 +147,7 @@ def version_summary() -> Dict[str, Union[int, float, str]]:
         :showcode:
 
         import pprint
-        from onnx_diagnostic.torch_models.test_helper import version_summary
+        from onnx_diagnostic.torch_models.validate import version_summary
 
         pprint.pprint(version_summary())
     """
@@ -259,7 +259,8 @@ def validate_model(
     verbose: int = 0,
     dtype: Optional[Union[str, torch.dtype]] = None,
     device: Optional[Union[str, torch.device]] = None,
-    trained: bool = False,
+    same_as_pretrained: bool = False,
+    use_pretrained: bool = False,
     optimization: Optional[str] = None,
     quiet: bool = False,
     patch: bool = False,
@@ -275,6 +276,7 @@ def validate_model(
     runtime: str = "onnxruntime",
     repeat: int = 1,
     warmup: int = 0,
+    inputs2: bool = True,
 ) -> Tuple[Dict[str, Union[int, float, str]], Dict[str, Any]]:
     """
     Validates a model.
@@ -293,7 +295,9 @@ def validate_model(
     :param verbose: verbosity level
     :param dtype: uses this dtype to check the model
     :param device: do the verification on this device
-    :param trained: use the trained model, not the untrained one
+    :param same_as_pretrained: use a model equivalent to the trained,
+        this is not always possible
+    :param use_pretrained: use the trained model, not the untrained one
     :param optimization: optimization to apply to the exported model,
         depend on the the exporter
     :param quiet: if quiet, catches exception if any issue
@@ -307,7 +311,7 @@ def validate_model(
     :param drop_inputs: drops this list of inputs (given their names)
     :param ortfusiontype: runs ort fusion, the parameters defines the fusion type,
         it accepts multiple values separated by ``|``,
-        see :func:`onnx_diagnostic.torch_models.test_helper.run_ort_fusion`
+        see :func:`onnx_diagnostic.torch_models.validate.run_ort_fusion`
     :param input_options: additional options to define the dummy inputs
         used to export
     :param model_options: additional options when creating the model such as
@@ -318,6 +322,8 @@ def validate_model(
         only if `do_run` is true
     :param repeat: number of time to measure the model
     :param warmup: warmup the model first
+    :param inputs2: checks that the second set of inputs is reunning as well,
+        this ensures that the model does support dynamism
     :return: two dictionaries, one with some metrics,
         another one with whatever the function produces
 
@@ -350,7 +356,8 @@ def validate_model(
             version_do_run=str(do_run),
             version_dtype=str(dtype or ""),
             version_device=str(device or ""),
-            version_trained=str(trained),
+            version_same_as_pretrained=str(same_as_pretrained),
+            version_use_pretrained=str(use_pretrained),
             version_optimization=optimization or "",
             version_quiet=str(quiet),
             version_patch=str(patch),
@@ -361,6 +368,7 @@ def validate_model(
             version_stop_if_static=str(stop_if_static),
             version_exporter=exporter or "",
             version_runtime=runtime,
+            version_inputs2=inputs2,
         )
     )
     if opset:
@@ -404,19 +412,25 @@ def validate_model(
         summary,
         None,
         (
-            lambda mid=model_id, v=verbose, task=task, tr=trained, iop=iop, sub=subfolder: (
+            lambda mid=model_id, v=verbose, task=task, uptr=use_pretrained, tr=same_as_pretrained, iop=iop, sub=subfolder, i2=inputs2: (  # noqa: E501
                 get_untrained_model_with_inputs(
                     mid,
                     verbose=v,
                     task=task,
+                    use_pretrained=uptr,
                     same_as_pretrained=tr,
                     inputs_kwargs=iop,
                     model_kwargs=mop,
                     subfolder=sub,
+                    add_second_input=i2,
                 )
             )
         ),
     )
+    assert not inputs2 or "inputs2" in data, (
+        f"inputs2 is True but second set is missing in data for "
+        f"model id {model_id!r}: {sorted(data)}"
+    )
 
     if exporter == "modelbuilder":
         # Models used with ModelBuilder do not like batch size > 1.
@@ -483,6 +497,16 @@ def validate_model(
         if verbose:
             print(f"[validate_model] new inputs: {string_type(data['inputs'])}")
             print(f"[validate_model] new dynamic_hapes: {string_type(data['dynamic_shapes'])}")
+        if inputs2:
+            assert (
+                "inputs2" in data
+            ), "Cannot test a second set of inputs as it was not defined."
+            data["inputs2"], _ = filter_inputs(
+                data["inputs2"],
+                drop_names=drop_inputs,
+                model=data["model"],
+                dynamic_shapes=data["dynamic_shapes"],
+            )
 
     if not empty(dtype):
         if isinstance(dtype, str):
@@ -492,6 +516,8 @@ def validate_model(
         data["model"] = to_any(data["model"], dtype)  # type: ignore
         data["inputs"] = to_any(data["inputs"], dtype)  # type: ignore
         summary["model_dtype"] = str(dtype)
+        if "inputs2" in data:
+            data["inputs2"] = to_any(data["inputs2"], dtype)  # type: ignore
 
     if not empty(device):
         if verbose:
@@ -499,6 +525,8 @@ def validate_model(
         data["model"] = to_any(data["model"], device)  # type: ignore
         data["inputs"] = to_any(data["inputs"], device)  # type: ignore
         summary["model_device"] = str(device)
+        if "inputs2" in data:
+            data["inputs2"] = to_any(data["inputs2"], device)  # type: ignore
 
     for k in ["task", "size", "n_weights"]:
         summary[f"model_{k.replace('_','')}"] = data[k]
@@ -527,35 +555,13 @@ def validate_model(
         print("[validate_model] --")
 
     if do_run:
-        if verbose:
-            print("[validate_model] -- run the model...")
-            print(f"[validate_model] inputs={string_type(data['inputs'], with_shape=True)}")
-        # We make a copy of the input just in case the model modifies them inplace
-        hash_inputs = string_type(data["inputs"], with_shape=True)
-        inputs = torch_deepcopy(data["inputs"])
-        model = data["model"]
-
-        expected = _quiet_or_not_quiet(
-            quiet,
-            "run",
-            summary,
-            data,
-            (lambda m=model, inp=inputs: m(**torch_deepcopy(inp))),
-            repeat=repeat,
-            warmup=warmup,
-        )
-        if "ERR_run" in summary:
-            return summary, data
-
-        summary["model_expected"] = string_type(expected, with_shape=True)
-        if verbose:
-            print("[validate_model] done (run)")
-        data["expected"] = expected
-        assert hash_inputs == string_type(data["inputs"], with_shape=True), (
-            f"The model execution did modified the inputs:\n"
-            f"before: {hash_inputs}\n"
-            f" after: {string_type(data['inputs'], with_shape=True)}"
+        _validate_do_run_model(
+            data, summary, "inputs", "run", "run_expected", verbose, repeat, warmup, quiet
         )
+        if inputs2:
+            _validate_do_run_model(
+                data, summary, "inputs2", "run2", "run_expected2", verbose, 1, 0, quiet
+            )
 
     if exporter:
         print(
@@ -578,43 +584,7 @@ def validate_model(
                 data["inputs_export"] = modificator(data["inputs"])  # type: ignore
 
                 if do_run:
-                    # We run a second time the model to check the patch did not
-                    # introduce any discrepancies
-                    if verbose:
-                        print("[validate_model] run patched model...")
-                        print(
-                            f"[validate_model] patched inputs="
-                            f"{string_type(data['inputs_export'], with_shape=True)}"
-                        )
-                    hash_inputs = string_type(data["inputs_export"], with_shape=True)
-
-                    # We make a copy of the input just in case the model modifies them inplace
-                    inputs = torch_deepcopy(data["inputs_export"])
-                    model = data["model"]
-
-                    expected = _quiet_or_not_quiet(
-                        quiet,
-                        "run_patched",
-                        summary,
-                        data,
-                        (lambda m=model, inp=inputs: m(**inp)),
-                    )
-                    if "ERR_run_patched" in summary:
-                        return summary, data
-
-                    disc = max_diff(data["expected"], expected)
-                    for k, v in disc.items():
-                        summary[f"disc_patched_{k}"] = str(v)
-                    if verbose:
-                        print("[validate_model] done (patched run)")
-                        print(f"[validate_model] patched discrepancies={string_diff(disc)}")
-                    assert hash_inputs == string_type(
-                        data["inputs_export"], with_shape=True
-                    ), (
-                        f"The model execution did modified the inputs:\n"
-                        f"before: {hash_inputs}\n"
-                        f" after: {string_type(data['inputs_export'], with_shape=True)}"
-                    )
+                    _validate_do_run_exported_program(data, summary, verbose, quiet)
 
                 # data is modified inplace
                 summary_export, data = call_exporter(
@@ -707,6 +677,7 @@ def validate_model(
             runtime=runtime,
             repeat=repeat,
             warmup=warmup,
+            inputs2=inputs2,
         )
         summary.update(summary_valid)
 
@@ -767,6 +738,7 @@ def validate_model(
                     runtime=runtime,
                     repeat=repeat,
                     warmup=warmup,
+                    inputs2=inputs2,
                 )
                 summary.update(summary_valid)
 
@@ -779,6 +751,79 @@ def validate_model(
     return summary, data
 
 
+def _validate_do_run_model(
+    data, summary, key, tag, expected_tag, verbose, repeat, warmup, quiet
+):
+    if verbose:
+        print(f"[validate_model] -- run the model inputs={key!r}...")
+        print(f"[validate_model] {key}={string_type(data[key], with_shape=True)}")
+    # We make a copy of the input just in case the model modifies them inplace
+    hash_inputs = string_type(data[key], with_shape=True)
+    inputs = torch_deepcopy(data[key])
+    model = data["model"]
+
+    expected = _quiet_or_not_quiet(
+        quiet,
+        tag,
+        summary,
+        data,
+        (lambda m=model, inp=inputs: m(**torch_deepcopy(inp))),
+        repeat=repeat,
+        warmup=warmup,
+    )
+    if f"ERR_{tag}" in summary:
+        return summary, data
+
+    summary[expected_tag] = string_type(expected, with_shape=True)
+    if verbose:
+        print(f"[validate_model] done ([{tag}])")
+    data[expected_tag] = expected
+    assert hash_inputs == string_type(data[key], with_shape=True), (
+        f"The model execution did modified the inputs:\n"
+        f"before: {hash_inputs}\n"
+        f" after: {string_type(data[key], with_shape=True)}"
+    )
+
+
+def _validate_do_run_exported_program(data, summary, verbose, quiet):
+
+    # We run a second time the model to check the patch did not
+    # introduce any discrepancies
+    if verbose:
+        print("[validate_model] run patched model...")
+        print(
+            f"[validate_model] patched inputs="
+            f"{string_type(data['inputs_export'], with_shape=True)}"
+        )
+    hash_inputs = string_type(data["inputs_export"], with_shape=True)
+
+    # We make a copy of the input just in case the model modifies them inplace
+    inputs = torch_deepcopy(data["inputs_export"])
+    model = data["model"]
+
+    expected = _quiet_or_not_quiet(
+        quiet,
+        "run_patched",
+        summary,
+        data,
+        (lambda m=model, inp=inputs: m(**inp)),
+    )
+    if "ERR_run_patched" in summary:
+        return summary, data
+
+    disc = max_diff(data["run_expected"], expected)
+    for k, v in disc.items():
+        summary[f"disc_patched_{k}"] = str(v)
+    if verbose:
+        print("[validate_model] done (patched run)")
+        print(f"[validate_model] patched discrepancies={string_diff(disc)}")
+    assert hash_inputs == string_type(data["inputs_export"], with_shape=True), (
+        f"The model execution did modified the inputs:\n"
+        f"before: {hash_inputs}\n"
+        f" after: {string_type(data['inputs_export'], with_shape=True)}"
+    )
+
+
 def call_exporter(
     data: Dict[str, Any],
     exporter: str,
@@ -845,7 +890,11 @@ def call_exporter(
         )
         return summary, data
     raise NotImplementedError(
-        f"export with {exporter!r} and optimization={optimization!r} not implemented yet"
+        f"export with {exporter!r} and optimization={optimization!r} not implemented yet, "
+        f"exporter must startswith 'onnx-', 'custom', 'export', 'modelbuilder' "
+        f"(onnx-dynamo, custom, export), optimization can 'ir', "
+        f"'default', 'default+onnxruntime', "
+        f"'default+onnxruntime+os_ort', 'ir', 'os_ort'"
     )
 
 
@@ -959,7 +1008,7 @@ def call_torch_export_export(
         if "ERR_export_export" in summary:
             return summary, data
 
-        disc = max_diff(data["expected"], expected)
+        disc = max_diff(data["run_expected"], expected)
         for k, v in disc.items():
             summary[f"disc_exported_{k}"] = str(v)
         if verbose:
@@ -981,6 +1030,7 @@ def validate_onnx_model(
     runtime: str = "onnxruntime",
     repeat: int = 1,
     warmup: int = 0,
+    inputs2: bool = True,
 ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """
     Verifies that an onnx model produces the same
@@ -996,6 +1046,8 @@ def validate_onnx_model(
     :param runtime: onnx runtime to use, onnxruntime or torch
     :param repeat: run that number of times the model
     :param warmup: warmup the model
+    :param inputs: to validate the model on the second input set
+        to make sure the exported model supports dynamism
     :return: two dictionaries, one with some metrics,
         another one with whatever the function produces
     """
@@ -1065,43 +1117,51 @@ def validate_onnx_model(
     if verbose:
         print(f"[validate_onnx_model] done (ort_session) flavour={flavour!r}")
 
-    # make_feeds
-    if verbose:
-        print("[validate_onnx_model] -- make_feeds...")
-        print(f"[validate_onnx_model] inputs={string_type(data['inputs'], with_shape=True)}")
-    feeds = make_feeds(sess, data["inputs"], use_numpy=True, check_flatten=False)
-    if verbose:
-        print(f"[validate_onnx_model] ort inputs={string_type(feeds, with_shape=True)}")
-    summary[_mk("onnx_ort_inputs")] = string_type(feeds, with_shape=True)
-    if verbose:
-        print("[validate_onnx_model] done (make_feeds)")
+    keys = [("inputs", "run_expected", "")]
+    if inputs2:
+        keys.append(("inputs2", "run_expected2", "2"))
+    for k_input, k_expected, suffix in keys:
+        # make_feeds
+        if verbose:
+            print(f"[validate_onnx_model] -- make_feeds for {k_input!r}...")
+            print(
+                f"[validate_onnx_model] inputs={string_type(data[k_input], with_shape=True)}"
+            )
+        feeds = make_feeds(sess, data[k_input], use_numpy=True, check_flatten=False)
+        if verbose:
+            print(f"[validate_onnx_model] ort inputs={string_type(feeds, with_shape=True)}")
+        summary[_mk(f"onnx_ort_inputs{suffix}")] = string_type(feeds, with_shape=True)
+        if verbose:
+            print("[validate_onnx_model] done (make_feeds)")
 
-    # run ort
-    if verbose:
-        print("[validate_onnx_model] run session...")
+        # run ort
+        if verbose:
+            print("[validate_onnx_model] run session...")
 
-    got = _quiet_or_not_quiet(
-        quiet,
-        _mk("time_onnx_ort_run"),
-        summary,
-        data,
-        (lambda sess=sess, feeds=feeds: sess.run(None, feeds)),
-        repeat=repeat,
-        warmup=warmup,
-    )
-    if f"ERR_{_mk('time_onnx_ort_run')}" in summary:
-        return summary, data
+        got = _quiet_or_not_quiet(
+            quiet,
+            _mk(f"time_onnx_ort_run{suffix}"),
+            summary,
+            data,
+            (lambda sess=sess, feeds=feeds: sess.run(None, feeds)),
+            repeat=repeat,
+            warmup=warmup,
+        )
+        if f"ERR_{_mk(f'time_onnx_ort_run{suffix}')}" in summary:
+            return summary, data
 
-    if verbose:
-        print("[validate_onnx_model] done (run)")
-        print(f"[validate_onnx_model] got={string_type(got, with_shape=True)}")
+        summary[f"run_feeds_{k_input}"] = string_type(feeds, with_shape=True, with_device=True)
+        summary[f"run_output_{k_input}"] = string_type(got, with_shape=True, with_device=True)
+        if verbose:
+            print("[validate_onnx_model] done (run)")
+            print(f"[validate_onnx_model] got={string_type(got, with_shape=True)}")
 
-    # compute discrepancies
-    disc = max_diff(data["expected"], got, flatten=True)
-    if verbose:
-        print(f"[validate_onnx_model] discrepancies={string_diff(disc)}")
-    for k, v in disc.items():
-        summary[_mk(f"disc_onnx_ort_run_{k}")] = v
+        # compute discrepancies
+        disc = max_diff(data[k_expected], got, flatten=True)
+        if verbose:
+            print(f"[validate_onnx_model] discrepancies={string_diff(disc)}")
+        for k, v in disc.items():
+            summary[_mk(f"disc_onnx_ort_run{suffix}_{k}")] = v
     return summary, data
 
 

{onnx_diagnostic-0.6.3.dist-info → onnx_diagnostic-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: onnx-diagnostic
-Version: 0.6.3
+Version: 0.7.1
 Summary: Investigate ONNX models
 Home-page: https://github.com/sdpython/onnx-diagnostic
 Author: Xavier Dupré
@@ -59,7 +59,7 @@ onnx-diagnostic: investigate onnx models
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/psf/black
 
-.. image:: https://codecov.io/gh/sdpython/onnx-diagnostic/branch/main/graph/badge.svg?token=Wb9ZGDta8J 
+.. image:: https://codecov.io/gh/sdpython/onnx-diagnostic/graph/badge.svg?token=91T5ZVIP96 
     :target: https://codecov.io/gh/sdpython/onnx-diagnostic
 
 The main feature is about `patches <https://github.com/sdpython/onnx-diagnostic/tree/main/onnx_diagnostic/torch_export_patches>`_:

{onnx_diagnostic-0.6.3.dist-info → onnx_diagnostic-0.7.1.dist-info}/RECORD

@@ -1,32 +1,35 @@
-onnx_diagnostic/__init__.py,sha256=mRkq5dlSo05GQMct7d6mMZLb6s5T24eG_3mD5O3wBo0,173
+onnx_diagnostic/__init__.py,sha256=YwjIZRhfTzRgTOBvmUSNNYX0SBBdmLsWfkMVwHkJloQ,173
 onnx_diagnostic/__main__.py,sha256=YmyV_Aq_ianDlHyKLHMa6h8YK3ZmFPpLVHLKjM91aCk,79
-onnx_diagnostic/_command_lines_parser.py,sha256=pxG3dYpTTpRCjBRzFGEZm4ewb7xprZihP7fG08kWL04,19989
+onnx_diagnostic/_command_lines_parser.py,sha256=WpCri2dqc1a1KYthQcb4-eN0htfiWeLrAkncNK3cZaY,27466
 onnx_diagnostic/api.py,sha256=BhCl_yCd78N7TlVtPOHjeYv1QBEy39TjZ647rcHqLh0,345
-onnx_diagnostic/doc.py,sha256=O_ncetL0G4-oHIxLv8ofTIIxCT_5ESSkKxfYvgccJEc,2038
-onnx_diagnostic/ext_test_case.py,sha256=nhWz75caudvKn-svH1ppUY8uw8MoTD4cEdqMdj6PiPc,42399
+onnx_diagnostic/doc.py,sha256=t3RELgfooYnVMAi0JSpggWkQEgUsREz8NmRvn0TnLI8,2829
+onnx_diagnostic/ext_test_case.py,sha256=IX-DNabvsPw8UkUeXC1amw3nnzdmJ3DeERn4E1Y_omo,42416
 onnx_diagnostic/export/__init__.py,sha256=yEIoWiOeTwBsDhyYt2fTKuhtA0Ya1J9u9ZzMTOTWaWs,101
-onnx_diagnostic/export/dynamic_shapes.py,sha256=EHB7VoWNx8sVetvOgE1vgC7wHtIjWDLjanhbEJNpK88,39892
+onnx_diagnostic/export/dynamic_shapes.py,sha256=HYf2OEi7PmRSj8uxMD-wbdVxxejkWXTPBAkxoFeM27A,40811
+onnx_diagnostic/export/shape_helper.py,sha256=C9cEq_x8I40RKuD89qWIholN1XZoWhaKPfbZQhiPD3g,4725
 onnx_diagnostic/export/validate.py,sha256=_PGUql2DJhIgGKo0WjTGUc5AgsZUx8fEs00MePy-w98,6043
 onnx_diagnostic/helpers/__init__.py,sha256=GJ2GT7cgnlIveVUwMZhuvUwidbTJaKv8CsSIOpZDsJg,83
 onnx_diagnostic/helpers/args_helper.py,sha256=SRWnqC7EENg09RZlA50B_PcdiIhdbgA4C3ACfzl5nMs,4419
 onnx_diagnostic/helpers/bench_run.py,sha256=CGA6VMJZMH2gDhVueT9ypNm4PMcjGrrGFYp08nhWj9k,16539
-onnx_diagnostic/helpers/cache_helper.py,sha256=soKjyIXa7EQgALd9PAUGIKYzXlJGoLevYiQDsxoqkQ4,8349
-onnx_diagnostic/helpers/config_helper.py,sha256=aZATKVbZuw8L56KQpwMNcqJ3Qi5OplzS_N3ETR3hmj0,3351
+onnx_diagnostic/helpers/cache_helper.py,sha256=E_D0z5ks4zUJb9H6K19MKhUZ-nZTC_dgeDO5zXNFE9g,10824
+onnx_diagnostic/helpers/config_helper.py,sha256=CdMeUhmDe0LfKcdPv9-Lzt73RRs29NmUHg9uVrdFwTQ,3479
 onnx_diagnostic/helpers/doc_helper.py,sha256=pl5MZd3_FaE8BqQnqoBuSBxoNCFcd2OJd3eITUSku5c,5897
 onnx_diagnostic/helpers/graph_helper.py,sha256=hevQT5a7_QuriVPQcbT5qe18n99Doyl5h3-qshx1-uk,14093
-onnx_diagnostic/helpers/helper.py,sha256=oPybQruFcVLqvqLDhjphOZ8zZU1HHJWAlABMuTkAO8A,57090
+onnx_diagnostic/helpers/helper.py,sha256=_6K0IvfK7ymBE8uWFAOA1ksU_fMvl2BRtlxj5SA9R2I,58203
+onnx_diagnostic/helpers/log_helper.py,sha256=qZdvHHQqkYdZOf8UsIrByswMYSF_axca27JXRyQk52Y,69163
 onnx_diagnostic/helpers/memory_peak.py,sha256=OT6mz0muBbBZY0pjgW2_eCk_lOtFRo-5w4jFo2Z6Kok,6380
-onnx_diagnostic/helpers/mini_onnx_builder.py,sha256=R1Vu4zHzN7GIUnbMVQzpkaXj8cCyyOweWOI9-TSgAHM,20966
-onnx_diagnostic/helpers/model_builder_helper.py,sha256=xIZmsVMFHfdtYeZHVEffBtxYObAaRPiaSmwwSKkmLwY,13502
+onnx_diagnostic/helpers/mini_onnx_builder.py,sha256=p0Xh2Br38xAqUjB2214GiNOIbCgiVZKeyVEnjdyqyFI,21091
+onnx_diagnostic/helpers/model_builder_helper.py,sha256=RvDyPFqRboEU3HsQV_xi9oy-o3_4KuGFVzs5MhksduY,12552
 onnx_diagnostic/helpers/onnx_helper.py,sha256=pXXQjfyNTSUF-Kt72U4fnBDkYAnWYMxdSw8m0qk3xmE,39670
 onnx_diagnostic/helpers/ort_session.py,sha256=UgUUeUslDxEFBc6w6f3HMq_a7bn4TBlItmojqWquSj4,29281
 onnx_diagnostic/helpers/rt_helper.py,sha256=BXU_u1syk2RyM0HTFHKEiO6rHHhZW2UFPyUTVdeq8BU,4251
-onnx_diagnostic/helpers/torch_helper.py,sha256=mrmn4mBeRvMRJ9cEu7BbNG-AHq2OJfSm8dxgtzh-yQQ,31631
-onnx_diagnostic/reference/__init__.py,sha256=nrd09rRuwMDBCPTSZ6kSKZXp1W9W_ExO1t9duDlBnh8,146
+onnx_diagnostic/helpers/torch_helper.py,sha256=L7qv14q4r1LcDKpEVobhySK6VE_X3h88Acvr6Kt4qEk,32244
+onnx_diagnostic/reference/__init__.py,sha256=rLZsxOlnb7-81F2CzepGnZLejaROg4JvgFaGR9FwVQA,208
 onnx_diagnostic/reference/evaluator.py,sha256=RzNzjFDeMe-4X51Tb22N6aagazY5ktNq-mRmPcfY5EU,8848
-onnx_diagnostic/reference/ort_evaluator.py,sha256=OaWMREF8fuJwimmONpIjQ6WxQT1X2roDsdJsgR8H_Cg,24853
+onnx_diagnostic/reference/ort_evaluator.py,sha256=1O7dHj8Aspolidg6rB2Nm7hT3HaGb4TxAgjCCD0XVcQ,26159
 onnx_diagnostic/reference/quantized_tensor.py,sha256=5u67uS2uGacdMD5VYCbpojNjiesDlV_kO0fAJ0vUWGE,1098
-onnx_diagnostic/reference/torch_evaluator.py,sha256=qAeYvSFwOCMDctc39evBEle_2bX8kuJW2QSLksofzn8,26600
+onnx_diagnostic/reference/report_results_comparison.py,sha256=OsyQN8EHZZoj97u74RQP-7WFpebPOso5GEDpdkLWu6M,3645
+onnx_diagnostic/reference/torch_evaluator.py,sha256=gf8EPoX4C4yGgQ-DqxXxaGU26WdEhn8Gd6iesDLqAV0,27692
 onnx_diagnostic/reference/ops/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 onnx_diagnostic/reference/ops/op_add_add_mul_mul.py,sha256=CXQVtgVrT066gDJFwxL4nDSY4G8r08XNu3EwhWqMapU,1521
 onnx_diagnostic/reference/ops/op_attention.py,sha256=ThALMDF53v3QeG1bohi0bvX2o90HZhGJbbAFOtwEHPE,2027
@@ -69,22 +72,22 @@ onnx_diagnostic/reference/torch_ops/sequence_ops.py,sha256=3EiVKpGfN4d1Iry4hgnr3
 onnx_diagnostic/reference/torch_ops/shape_ops.py,sha256=pJrNR2UB4PlWl6cv4EDl1uGl8YTBUUMQkhJcsh5K4sA,4291
 onnx_diagnostic/reference/torch_ops/unary_ops.py,sha256=E8Ys1eZsOTsucBKoXb1_Kl5LbBDygniDvW2BvN4IPMo,1708
 onnx_diagnostic/tasks/__init__.py,sha256=5XXM-rv-Hk2gSHvqsww9DzVd9mcRifacgcPgvPCjnDM,2412
-onnx_diagnostic/tasks/automatic_speech_recognition.py,sha256=oRoYy56M0Yv_WOcn1hJXv-R9wgHkJ8rbym7j7y8oslw,6851
-onnx_diagnostic/tasks/feature_extraction.py,sha256=V-T5NpZ6EimOz00weWWxGfksZ9jQ5ZQyaP-mxuCEuJo,2223
-onnx_diagnostic/tasks/fill_mask.py,sha256=POUtgvOWv8wTOVLqxPNsj_C2WBiBWkmM72Z9mNlNqxI,2341
-onnx_diagnostic/tasks/image_classification.py,sha256=wtSkZB6Wqh2Y1O_zjfnYRZxUQPVVUlLQ_4D0H-SG-xU,4140
-onnx_diagnostic/tasks/image_text_to_text.py,sha256=6rKbts_p05VZL8wufJa6NP-MhxUOU-fuTAks5QfUVVQ,6037
-onnx_diagnostic/tasks/mixture_of_expert.py,sha256=orMx8Ly4DO0Po0tEmme4gi2flPIGip4TaAyxVik4Zgg,2685
-onnx_diagnostic/tasks/object_detection.py,sha256=o1T8NMztjdFAFA-Z5efx-8nd9W7YZZcbE8Ag5wKVxZA,3930
-onnx_diagnostic/tasks/sentence_similarity.py,sha256=okQ-TQR8j1a92_N-eT6xN56rjtu26CdlU_pk88gdbGs,2356
-onnx_diagnostic/tasks/summarization.py,sha256=qK9E7TdaB9g_Mu-f4Vdr_X8mMAUSlIapLlc8FlWSGFU,7989
-onnx_diagnostic/tasks/text2text_generation.py,sha256=x5p_5n72scF9hFNOVP8BjOr4Lga1nnqtjfStZ_n9EwQ,8406
-onnx_diagnostic/tasks/text_classification.py,sha256=OgC_G9iumzTjTNUEvMoFFNTHCD8_BkdvdYC4jUsfpHM,2412
-onnx_diagnostic/tasks/text_generation.py,sha256=jWKBFt7M-Neyjw6YSOrYY28xOkRkIerDF5YlugKK3qk,10386
-onnx_diagnostic/tasks/zero_shot_image_classification.py,sha256=N3cEG1Lq95wS1N_CWUUUCU5j-4Tp5eR8Ce68U8THYAk,4380
+onnx_diagnostic/tasks/automatic_speech_recognition.py,sha256=7OspFypNHLSL6huvP9ms_euhHqYXyTZjAJ8Z7EXGimk,6967
+onnx_diagnostic/tasks/feature_extraction.py,sha256=CbxbGsv3JvEQ2J9tO2DOpMHcJj5ZlCwY81ZB3hPB4D4,2339
+onnx_diagnostic/tasks/fill_mask.py,sha256=ZWz8swzEeRbkmbY9oZ4CM1LYCWWUxnS5CqrKmUVw-u0,2457
+onnx_diagnostic/tasks/image_classification.py,sha256=UjUAFYnwXIdPMXJdHR5MDzpsfMeIvyuKR4RqJVpGV_Q,4449
+onnx_diagnostic/tasks/image_text_to_text.py,sha256=LmpMdH6oF_EN3WIACzSip4fPZOjZWFOoXg4k8qAio6Q,7639
+onnx_diagnostic/tasks/mixture_of_expert.py,sha256=C0ugEc8OWmVyEZpsh8MJq_te1zgOHhpITtnSmGC16Ls,2801
+onnx_diagnostic/tasks/object_detection.py,sha256=1lF5e2f2Coz1urSptEKgvUGCOSFBf0Anuq_QYOC00dA,4046
+onnx_diagnostic/tasks/sentence_similarity.py,sha256=3MvNxjC1iEMtQL_jH1c8bmrVc5IG1lfUygrCZ0SORJk,2472
+onnx_diagnostic/tasks/summarization.py,sha256=NLwqhpiQrU8UWd3u30VsNA3FsL315S3nlQ7ycUzJueo,8105
+onnx_diagnostic/tasks/text2text_generation.py,sha256=mYvsq-O69fr5pitX0mWugT76QuK4xUs40Vsz9ru_XK8,8522
+onnx_diagnostic/tasks/text_classification.py,sha256=NCCKobBQyCc7dSVj7_5N6S_RuvBlRMAdWkS2rVvrzck,2528
+onnx_diagnostic/tasks/text_generation.py,sha256=PDh870BB-llzlu8h_aZX4Z-9QLzcGmDwX5aKJPy_K90,12504
+onnx_diagnostic/tasks/zero_shot_image_classification.py,sha256=GKaXm8g7cK23h3wJEUc6Q-6mpmLAzQ4YkJbd-eGP7Y4,4496
 onnx_diagnostic/torch_export_patches/__init__.py,sha256=0SaZedwznm1hQUCvXZsGZORV5vby954wEExr5faepGg,720
-onnx_diagnostic/torch_export_patches/onnx_export_errors.py,sha256=MgOQwgLf6-uCGQaiUrhVNfZQ43dCp1iWGbzLbKEVyc8,18810
-onnx_diagnostic/torch_export_patches/onnx_export_serialization.py,sha256=l5HvE_FfdCtgLJBmJczH6nA8jZY1725AggiHwoAa-o0,15763
+onnx_diagnostic/torch_export_patches/onnx_export_errors.py,sha256=op8jgnTa_1T_bGN172A6YFTtkQv_ALMNu1oukrsFt9U,20634
+onnx_diagnostic/torch_export_patches/onnx_export_serialization.py,sha256=Nsf7HUJqu3ZRd0o9vUVCF6ifmS5UaQM6hB_Gmn19dNI,17095
 onnx_diagnostic/torch_export_patches/patch_expressions.py,sha256=vr4tt61cbDnaaaduzMj4UBZ8OUtr6GfDpIWwOYqjWzs,3213
 onnx_diagnostic/torch_export_patches/patch_inputs.py,sha256=9b4pmyT00BwLqi7WG-gliep1RUy3gXEgW6BDnlSSA-M,7689
 onnx_diagnostic/torch_export_patches/patch_module.py,sha256=R2d9IHM-RwsBKDsxuBIJnEqMoxbS9gd4YWFGG2wwV5A,39881
@@ -93,23 +96,23 @@ onnx_diagnostic/torch_export_patches/eval/__init__.py,sha256=VtkQB1o3Q2Fh99OOF6v
 onnx_diagnostic/torch_export_patches/eval/model_cases.py,sha256=DTvdHPtNQh25Akv5o3D4Jxf1L1-SJ7w14tgvj8AAns8,26577
 onnx_diagnostic/torch_export_patches/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_export_patches/patches/patch_torch.py,sha256=KaZ8TjDa9ATgT4HllYzzoNf_51q_yOj_GuF5NYjPCrU,18913
-onnx_diagnostic/torch_export_patches/patches/patch_transformers.py,sha256=Hf-U50vzgzJ4iUjS2LAYkbfmzCEwX80Dzvdrr-Rhlp0,26456
+onnx_diagnostic/torch_export_patches/patches/patch_transformers.py,sha256=GwcPUaSm-Zys2pWHac8Wcvpmy2h4oiFQDmx_D3GZNBA,41007
 onnx_diagnostic/torch_models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_models/llms.py,sha256=soyg4yC87ptGoeulJhKqw5opGmuLvH1pn_ZDXZ4Jr8E,90
-onnx_diagnostic/torch_models/test_helper.py,sha256=tt6bgLjGRayzvkXrTelKHTjr7XU9BvhX7uE4XJq6H6o,59927
+onnx_diagnostic/torch_models/validate.py,sha256=NXFmJKmoO4reaeiu2ibuVgMRLS-l0WSdLhjn40_YsbU,62177
 onnx_diagnostic/torch_models/hghub/__init__.py,sha256=vi1Q7YHdddj1soiBN42MSvJdFqe2_KUoWafHISjwOu8,58
-onnx_diagnostic/torch_models/hghub/hub_api.py,sha256=BgM_p57Q0gT9GOhdrmOYcnbuTTzCWp80jS4OQqWwFhs,9990
-onnx_diagnostic/torch_models/hghub/hub_data.py,sha256=885wKyZkdM-Qp5Sg6C9Ol1dxigmA8FYAko-Ys08sppo,8096
-onnx_diagnostic/torch_models/hghub/hub_data_cached_configs.py,sha256=V4Wv_73QsiWMFTw-xqj-5yZOri1NKuAub6VQa2UVIpw,259582
-onnx_diagnostic/torch_models/hghub/model_inputs.py,sha256=1h1jHJknqDHAa308_bjOzXVPgy3GIF-ikpce8CHNTmE,7804
+onnx_diagnostic/torch_models/hghub/hub_api.py,sha256=q4jUgSJ8AD28mpX7yDAUp0z7EQgb8euuD-P9Hayehds,12672
+onnx_diagnostic/torch_models/hghub/hub_data.py,sha256=lM9IDb5-3X8NSHcSPJFLS3tAvu_FqvcetyoHn-P2FIM,8272
+onnx_diagnostic/torch_models/hghub/hub_data_cached_configs.py,sha256=dE8tHksGOsTk77jpa7mldLYzaQ5joKxDxDB0ZnwQBV4,267246
+onnx_diagnostic/torch_models/hghub/model_inputs.py,sha256=XCllU8_fJBjkCGSV8cdqlpF1QH6AN_OAErK0aAXNQts,10261
 onnx_diagnostic/torch_models/untrained/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_models/untrained/llm_phi2.py,sha256=ynBTDHJHCk44NjLT_t6OiFDBdPP0rFGPteiONDxvztw,3708
-onnx_diagnostic/torch_models/untrained/llm_tiny_llm.py,sha256=7N3fGvT_4Mn4NbIo0Qk57c6DMc3OXGWyvj_P41rjwSY,3513
+onnx_diagnostic/torch_models/untrained/llm_tiny_llm.py,sha256=QXw_Bs2SzfeiQMf-tmtVl83SmVOL4-Um7Qy-f0E48QI,2507
 onnx_diagnostic/torch_onnx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_onnx/runtime_info.py,sha256=1g9F_Jf9AAgYQU4stbsrFXwQl-30mWlQrFbQ7val8Ps,9268
 onnx_diagnostic/torch_onnx/sbs.py,sha256=1EL25DeYFzlBSiFG_XjePBLvsiItRXbdDrr5-QZW2mA,16878
-onnx_diagnostic-0.6.3.dist-info/licenses/LICENSE.txt,sha256=Vv6TXglX6Rc0d-f8aREhayhT-6PMQXEyOmI2NKlUCMc,1045
-onnx_diagnostic-0.6.3.dist-info/METADATA,sha256=eJxj0KTPv1rXf-3T9KImWIF-u8g7wHXBGZm5zvXM7V8,6643
-onnx_diagnostic-0.6.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-onnx_diagnostic-0.6.3.dist-info/top_level.txt,sha256=KwNkXewmcobM3ZT1DJLVWH6ebJzA5qKg7cWqKfpGNT4,16
-onnx_diagnostic-0.6.3.dist-info/RECORD,,
+onnx_diagnostic-0.7.1.dist-info/licenses/LICENSE.txt,sha256=Vv6TXglX6Rc0d-f8aREhayhT-6PMQXEyOmI2NKlUCMc,1045
+onnx_diagnostic-0.7.1.dist-info/METADATA,sha256=7YUf_2f3gFaGkWUbOrZNqREtnQSpRXx5q65HJEDcDI8,6631
+onnx_diagnostic-0.7.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+onnx_diagnostic-0.7.1.dist-info/top_level.txt,sha256=KwNkXewmcobM3ZT1DJLVWH6ebJzA5qKg7cWqKfpGNT4,16
+onnx_diagnostic-0.7.1.dist-info/RECORD,,