onnx-diagnostic 0.8.0__py3-none-any.whl

onnx_diagnostic/torch_onnx/sbs.py

@@ -0,0 +1,440 @@
+from typing import Any, Dict, Iterator, Optional, Tuple, Union
+import onnx
+import torch
+from ..helpers import string_type, string_diff, max_diff
+from ..helpers.onnx_helper import to_array_extended
+from ..helpers.torch_helper import to_numpy
+
+
+def validate_fx_tensor(
+    node: torch.fx.Node, tensor: torch.Tensor, expected_shape: Tuple[Any, ...]
+) -> None:
+    """
+    Validates the shape of tensor is expected.
+
+    :param node: node
+    :param tensor: tensor
+    :param expected_shape: expected shape
+    """
+    assert len(tensor.shape) == len(expected_shape), (
+        f"Shape mismatch, got {tensor.shape} expected {expected_shape}, "
+        f"node.name={node.name!r}, node.target={getattr(node, 'target', None)}, "
+        f"node.args={node.args}, node.kwargs={node.kwargs}, "
+        f"node.meta={node.meta}"
+    )
+    for a, b in zip(tensor.shape, expected_shape):
+        assert not isinstance(b, int) or a == b or {a, b} == {0, 1}, (
+            f"Dimension mismatch, got {tensor.shape} expected {expected_shape}, "
+            f"node.name={node.name!r}, node.target={getattr(node, 'target', None)}, "
+            f"node.args={node.args}, node.kwargs={node.kwargs}, "
+            f"node.meta={node.meta}"
+        )
+
+
+def validate_fx_outputs(node: torch.fx.Node, outputs: Tuple[Any, ...]) -> None:
+    """
+    Validates the outputs of a node using metadata stored in the node.
+
+    :param node: node
+    :param outputs: outputs
+    """
+    if "val" not in node.meta:
+        return
+    if isinstance(outputs, torch.Tensor):
+        validate_fx_tensor(node, outputs, node.meta["val"].shape)
+        return
+    if isinstance(outputs, (tuple, list)):
+        assert isinstance(node.meta["val"], (list, tuple)), (
+            f"Unexpected type {string_type(node.meta['val'])} for node.meta['val'], "
+            f"node.name={node.name!r}, node.target={getattr(node, 'target', None)}, "
+            f"node.args={node.args}, node.kwargs={node.kwargs}, "
+            f"node.meta={node.meta}"
+        )
+        assert len(outputs) == len(node.meta["val"]), (
+            f"Length mismatch, got {len(outputs)} expected {len(node.meta['val'])}, "
+            f"node.name={node.name!r}, node.target={getattr(node, 'target', None)}, "
+            f"node.args={node.args}, node.kwargs={node.kwargs}, "
+            f"node.meta={node.meta}"
+        )
+        for a, b in zip(outputs, node.meta["val"]):
+            validate_fx_tensor(node, a, b.shape)
+        return
+    if isinstance(outputs, int):
+        assert (
+            isinstance(node.meta["val"], (torch.SymInt, torch.SymBool, torch.SymFloat))
+            or outputs == node.meta["val"]
+        ), (
+            f"Int mismatch, got {outputs} expected {node.meta['val']}, "
+            f"node.name={node.name!r}, node.target={getattr(node, 'target', None)}, "
+            f"node.args={node.args}, node.kwargs={node.kwargs}, "
+            f"node.meta={node.meta}"
+        )
+        return
+    if outputs is None:
+        assert node.meta["val"] is None, (
+            f"None mismatch, got {outputs} expected {node.meta['val']}, "
+            f"node.name={node.name!r}, node.target={getattr(node, 'target', None)}, "
+            f"node.args={node.args}, node.kwargs={node.kwargs}, "
+            f"node.meta={node.meta}"
+        )
+        return
+    raise NotImplementedError(
+        f"Validation for output type {type(outputs)} is not implemented, "
+        f"node.name={node.name!r}, node.target={getattr(node, 'target', None)}, "
+        f"node.args={node.args}, node.kwargs={node.kwargs}, "
+        f"node.meta={node.meta}"
+    )
+
+
+def run_fx_node(
+    node: torch.fx.Node, args: Tuple[Any, ...], kwargs: Optional[Dict[str, Any]] = None
+) -> Tuple[Any, ...]:
+    """
+    Executes a node
+
+    :param node: runs a node
+    :param args: unnamed inputs to the node
+    :param kwargs: named inputs to the node
+    :return: results
+    """
+    if node.op == "output":
+        assert len(args) == 1 and not kwargs, (
+            f"Unexpected inputs: args={string_type(args, limit=20)} "
+            f"kwargs={string_type(kwargs, limit=20)}"
+        )
+        return args
+    if node.op == "call_function":
+        assert callable(node.target), f"{node.target!r} not callable in node {node!r}"
+        outputs = node.target(*args, **(kwargs or {}))
+        validate_fx_outputs(node, outputs)
+        return outputs
+    raise NotImplementedError(
+        f"node.op={node.op!r} is not implemented, node.name={node.name!r}"
+    )
+
+
+def _pick_result(torch_results: Dict[str, Any], ref: Any) -> Any:
+    "See :func:`prepare_args_kwargs`."
+    if isinstance(ref, torch.fx.Node):
+        return torch_results[ref.name]
+    if isinstance(ref, list):
+        return [_pick_result(torch_results, n) for n in ref]
+    if isinstance(ref, tuple):
+        return tuple(_pick_result(torch_results, n) for n in ref)
+    if isinstance(ref, dict):
+        return {k: _pick_result(torch_results, v) for k, v in ref.items()}
+    if isinstance(ref, (bool, int, float, str, torch.device, torch.dtype)):
+        return ref
+    if ref is None:
+        return None
+    raise NotImplementedError(f"Unable to process args type {type(ref)}")
+
+
+def prepare_args_kwargs(
+    torch_results: Dict[str, Any], node: torch.fx.Node
+) -> Tuple[Tuple[Any, ...], Dict[str, Any]]:
+    """
+    Prepares args and kwargs before executing a fx node.
+
+    :param torch_results: existing results
+    :param node: node to execute
+    :return: new args and kwargs
+    """
+    new_args = _pick_result(torch_results, node.args)
+    new_kwargs = _pick_result(torch_results, node.kwargs)
+    return new_args, new_kwargs
+
+
+def run_aligned(
+    ep: torch.export.ExportedProgram,
+    onx: Union[onnx.ModelProto, onnx.FunctionProto],
+    args: Tuple[torch.Tensor, ...],
+    check_conversion_cls: Union[Dict[str, Any], type],
+    kwargs: Optional[Dict[str, Any]] = None,
+    verbose: int = 0,
+) -> Iterator[Tuple[Any, ...]]:
+    """
+    Runs in parallel both the exported program
+    and the onnx proto and looks for discrepancies.
+    The function does match on result names so it assumes
+    the exported program and the onnx model have the same names
+    for equivalent results.
+
+    :param ep: exported program
+    :param onx: model or function proto
+    :param args: input args
+    :param check_conversion_cls: defines the runtime to use for this task
+    :param kwargs: input kwargs
+    :param verbose: verbosity level
+    :return: a list of tuples containing the results, they come in tuple,
+
+    Example:
+
+    .. runpython::
+        :showcode:
+        :warningout: UserWarning
+
+        import pprint
+        import pandas
+        import torch
+        from onnx_diagnostic.reference import (
+            # This can be replace by any runtime taking NodeProto as an input.
+            ExtendedReferenceEvaluator as ReferenceEvaluator,
+        )
+        from onnx_diagnostic.torch_onnx.sbs import run_aligned
+
+
+        class Model(torch.nn.Module):
+            def forward(self, x):
+                ry = x.abs()
+                rz = ry.exp()
+                rw = rz + 1
+                ru = rw.log() + rw
+                return ru
+
+
+        def post_process(obs):
+            dobs = dict(zip(["ep_id_node", "onnx_id_node", "ep_name", "onnx_name"], obs))
+            dobs["err_abs"] = obs[-1]["abs"]
+            dobs["err_rel"] = obs[-1]["rel"]
+            return dobs
+
+
+        x = torch.randn((5, 4))
+        Model()(x)  # to make sure the model is running
+        ep = torch.export.export(
+            Model(), (x,), dynamic_shapes=({0: torch.export.Dim("batch")},)
+        )
+        onx = torch.onnx.export(
+            Model(), (x,), dynamic_shapes=({0: torch.export.Dim("batch")},)
+        ).model_proto
+        results = list(
+            map(
+                post_process,
+                run_aligned(
+                    ep,
+                    onx,
+                    (x,),
+                    check_conversion_cls=dict(cls=ReferenceEvaluator, atol=1e-5, rtol=1e-5),
+                    verbose=1,
+                ),
+            ),
+        )
+        print("------------")
+        print("final results")
+        df = pandas.DataFrame(results)
+        print(df)
+    """
+    assert not kwargs, f"Not implemented when kwargs={string_type(kwargs,with_shape=True)}"
+    cls, atol, rtol = (
+        (
+            check_conversion_cls["cls"],
+            check_conversion_cls["atol"],
+            check_conversion_cls["rtol"],
+        )
+        if isinstance(check_conversion_cls, dict)
+        else (check_conversion_cls, None, None)
+    )
+
+    # retrieve the positions
+    positions: Dict[str, Any] = {}
+    for i, node in enumerate(ep.graph.nodes):
+        if isinstance(node.name, str):
+            positions[node.name] = dict(fx=i)
+        else:
+            for n in node.name:
+                positions[n] = dict(fx=i)
+
+    for i, node in enumerate(onx.graph.node):
+        for n in node.output:
+            if n in positions:
+                positions[n]["onnx"] = i
+            else:
+                positions[n] = dict(onnx=i)
+
+    onnx_results: Dict[str, Any] = {}
+    for init in onx.graph.initializer:  # type: ignore
+        positions[init.name] = -1
+        onnx_results[init.name] = to_array_extended(init)
+        param_name = f"p_{init.name.replace('.', '_')}"
+        if param_name == init.name:
+            continue
+        assert param_name not in onnx_results, (
+            f"Some confusion may happen because {init.name!r} -> {param_name!r} "
+            f"and onnx_results has {sorted(onnx_results)}"
+        )
+        onnx_results[param_name] = onnx_results[init.name]
+
+    torch_results: Dict[str, Any] = {
+        k: torch.from_numpy(v.copy())
+        for k, v in onnx_results.items()
+        if not k.startswith("init")
+    }
+    last_position = 0
+    torch_output_names = None
+    for node in ep.graph.nodes:
+        if node.op == "output":
+            torch_output_names = [n.name for n in node.args[0]]
+    onnx_outputs_names = [o.name for o in onx.graph.output]
+    assert torch_output_names is not None and len(torch_output_names) == len(
+        onnx_outputs_names
+    ), (
+        f"Unexpected number of outputs, torch_output_names={torch_output_names}, "
+        f"onnx_outputs_names={onnx_outputs_names}"
+    )
+    mapping_onnx_to_torch = dict(zip(onnx_outputs_names, torch_output_names))
+
+    if verbose:
+        for k, v in torch_results.items():
+            print(
+                f"[run_aligned] +torch-cst: {k}: "
+                f"{string_type(v, with_shape=True, with_min_max=True)}"
+            )
+        for k, v in onnx_results.items():
+            print(
+                f"[run_aligned] +onnx-init: {k}: "
+                f"{string_type(v, with_shape=True, with_min_max=True)}"
+            )
+
+    for inp, v in zip(onx.graph.input, args):
+        onnx_results[inp.name] = to_numpy(v)
+        if verbose:
+            print(
+                f"[run_aligned] +onnx-input: {inp.name}: "
+                f"{string_type(v, with_shape=True, with_min_max=True)}"
+            )
+
+    for i, node in enumerate(ep.graph.nodes):
+        if verbose:
+            if node.op == "call_function":
+                print(
+                    f"[run_aligned] run ep.graph.nodes[{i}]: "
+                    f"{node.op}[{node.target}] -> {node.name!r}"
+                )
+            else:
+                print(f"[run_aligned] run ep.graph.nodes[{i}]: {node.op} -> {node.name!r}")
+
+        if node.op == "placeholder":
+            if node.name in onnx_results:
+                torch_results[node.name] = torch.from_numpy(onnx_results[node.name].copy())
+                if verbose:
+                    t = torch_results[node.name]
+                    print(
+                        f"[run_aligned] +torch {node.name}="
+                        f"{string_type(t, with_shape=True, with_min_max=True)}"
+                    )
+                continue
+            raise AssertionError(
+                f"unable to process node {node.op} -> {node.name!r} "
+                f"not in {sorted(onnx_results)}, len(args)={len(args)}, "
+                f"onx.graph.input={[i.name for i in onx.graph.input]}"
+            )
+
+        outputs = [node.name] if isinstance(node.name, str) else list(node.name)
+        args, kwargs = prepare_args_kwargs(torch_results, node)
+        new_outputs = run_fx_node(node, args, kwargs)
+        if isinstance(new_outputs, (torch.Tensor, int, float, list)):
+            new_outputs = (new_outputs,)
+
+        if new_outputs is None:
+            # Probably an assert.
+            continue
+
+        for k, v in zip(outputs, new_outputs):
+            torch_results[k] = v
+        if verbose:
+            for k, v in zip(outputs, new_outputs):
+                print(
+                    f"[run_aligned] +torch {k}="
+                    f"{string_type(v, with_shape=True, with_min_max=True)}"
+                )
+
+        max_pos = -2
+        for n in outputs:
+            if n in positions and "onnx" in positions[n]:
+                max_pos = max(max_pos, positions[n]["onnx"])
+        if max_pos == -2:
+            # we skip.
+            continue
+
+        for i_onnx in range(last_position, max_pos + 1):
+            node = onx.graph.node[i_onnx]
+            if verbose:
+                print(
+                    f"[run_aligned] run onx.graph.node[{i_onnx}]: "
+                    f"{node.op_type}({', '.join(node.input)}) -> {', '.join(node.output)}"
+                )
+            ref = cls(node)
+            feeds = {k: onnx_results[k] for k in node.input}
+            res = ref.run(None, feeds)
+            for o, r in zip(node.output, res):
+                onnx_results[o] = r
+                if verbose:
+                    print(
+                        f"[run_aligned] +onnx {o}="
+                        f"{string_type(r, with_shape=True, with_min_max=True)}"
+                    )
+
+                to = mapping_onnx_to_torch.get(o, o)
+                if to in torch_results:
+                    d = max_diff(torch_results[to], r)
+                    if verbose:
+                        if o == to:
+                            print(f"[run_aligned] =common results {to}: {string_diff(d)}")
+                        else:
+                            print(f"[run_aligned] =common results {to}/{o}: {string_diff(d)}")
+                        if not (
+                            atol is None
+                            or rtol is None
+                            or (d["abs"] <= atol and d["rel"] <= rtol)
+                        ):
+                            skw = dict(with_shape=True, with_min_max=True)
+                            raise ValueError(
+                                f"discrepancies detected for results [{to}/{o}]: "
+                                f"{string_diff(d)}"
+                                f"\n-- torch_results: {string_type(torch_results[to], **skw)}"
+                                f"\n-- onnx_results: {string_type(r, **skw)}"
+                                f"\n-- torch\n{torch_results[to]}\n-- onnx\n{r}"
+                            )
+                    yield (i, i_onnx, o, to, d)
+
+        last_position = max_pos + 1
+
+    # complete the execution of the onnx graph
+    for i_onnx in range(last_position, len(onx.graph.node)):
+        node = onx.graph.node[i_onnx]
+        if verbose:
+            print(
+                f"[run_aligned] run onx.graph.node[{i_onnx}]: "
+                f"{node.op_type}({', '.join(node.input)}) -> {', '.join(node.output)}"
+            )
+        ref = cls(node)
+        feeds = {k: onnx_results[k] for k in node.input}
+        res = ref.run(None, feeds)
+        for o, r in zip(node.output, res):
+            onnx_results[o] = r
+            if verbose:
+                print(
+                    f"[run_aligned] +onnx {o}="
+                    f"{string_type(r, with_shape=True, with_min_max=True)}"
+                )
+
+            to = mapping_onnx_to_torch.get(o, o)
+            if to in torch_results:
+                d = max_diff(torch_results[to], r)
+                if verbose:
+                    if o == to:
+                        print(f"[run_aligned] =common results* {to}: {string_diff(d)}")
+                    else:
+                        print(f"[run_aligned] =common results* {to}/{o}: {string_diff(d)}")
+                    if not (
+                        atol is None or rtol is None or (d["abs"] <= atol and d["rel"] <= rtol)
+                    ):
+                        skw = dict(with_shape=True, with_min_max=True)
+                        raise ValueError(
+                            f"discrepancies detected for results* [{to}/{o}]: {string_diff(d)}"
+                            f"\n-- torch_results: {string_type(torch_results[to], **skw)}"
+                            f"\n-- onnx_results: {string_type(r, **skw)}"
+                            f"\n-- torch\n{torch_results[to]}\n-- onnx\n{r}"
+                        )
+                yield (i, i_onnx, o, to, d)

onnx_diagnostic-0.8.0.dist-info/METADATA

@@ -0,0 +1,213 @@
+Metadata-Version: 2.4
+Name: onnx-diagnostic
+Version: 0.8.0
+Summary: Tools to help converting pytorch models into ONNX.
+Home-page: https://github.com/sdpython/onnx-diagnostic
+Author: Xavier Dupré
+Author-email: Xavier Dupré <xavier.dupre@gmail.com>
+License: MIT
+Project-URL: Homepage, https://sdpython.github.io/doc/onnx-diagnostic/dev/
+Project-URL: Repository, https://github.com/sdpython/onnx-diagnostic/
+Requires-Python: >=3.9
+Description-Content-Type: text/x-rst
+License-File: LICENSE.txt
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+
+
+.. image:: https://github.com/sdpython/onnx-diagnostic/raw/main/_doc/_static/logo.png
+    :width: 120
+
+onnx-diagnostic: investigate onnx models
+========================================
+
+.. image:: https://github.com/sdpython/onnx-diagnostic/actions/workflows/documentation.yml/badge.svg
+    :target: https://github.com/sdpython/onnx-diagnostic/actions/workflows/documentation.yml
+
+.. image:: https://badge.fury.io/py/onnx-diagnostic.svg
+    :target: http://badge.fury.io/py/onnx-diagnostic
+
+.. image:: https://img.shields.io/badge/license-MIT-blue.svg
+    :alt: MIT License
+    :target: https://opensource.org/license/MIT/
+
+.. image:: https://img.shields.io/github/repo-size/sdpython/onnx-diagnostic
+    :target: https://github.com/sdpython/onnx-diagnostic/
+    :alt: size
+
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/psf/black
+
+.. image:: https://codecov.io/gh/sdpython/onnx-diagnostic/graph/badge.svg?token=91T5ZVIP96 
+    :target: https://codecov.io/gh/sdpython/onnx-diagnostic
+
+The main feature is about `patches <https://github.com/sdpython/onnx-diagnostic/tree/main/onnx_diagnostic/torch_export_patches>`_:
+it helps exporting **pytorch models into ONNX**, mostly designed for LLMs using dynamic caches.
+Patches can be enabled as follows:
+
+.. code-block:: python
+
+  from onnx_diagnostic.torch_export_patches import torch_export_patches
+
+  with torch_export_patches(patch_transformers=True) as f:
+      ep = torch.export.export(model, args, kwargs=kwargs, dynamic_shapes=dynamic_shapes)
+      # ...
+
+Dynamic shapes are difficult to guess for caches, one function
+returns a structure defining all dimensions as dynamic.
+You need then to remove those which are not dynamic in your model.
+
+.. code-block:: python
+
+  from onnx_diagnostic.export.shape_helper import all_dynamic_shapes_from_inputs
+
+  dynamic_shapes = all_dynamic_shapes_from_inputs(cache)
+
+It also implements tools to investigate, validate exported models (ExportedProgramm, ONNXProgram, ...).
+See `documentation of onnx-diagnostic <https://sdpython.github.io/doc/onnx-diagnostic/dev/>`_ and
+`torch_export_patches <https://sdpython.github.io/doc/onnx-diagnostic/dev/api/torch_export_patches/index.html#onnx_diagnostic.torch_export_patches.torch_export_patches>`_.
+
+Getting started
++++++++++++++++
+
+::
+
+    git clone https://github.com/sdpython/onnx-diagnostic.git
+    cd onnx-diagnostic
+    pip install -e . -v
+
+or
+
+::
+
+    pip install onnx-diagnostic
+
+Enlightening Examples
++++++++++++++++++++++
+
+**Where to start to export a model**
+
+* `Export microsoft/phi-2
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_export_tiny_phi2.html>`_
+
+**Torch Export**
+
+* `Use DYNAMIC or AUTO when exporting if dynamic shapes has constraints
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_export_with_dynamic_shapes_auto.html>`_
+* `Find and fix an export issue due to dynamic shapes
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_export_locate_issue.html>`_
+* `Export with DynamicCache and guessed dynamic shapes
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_export_with_dynamic_cache.html>`_
+* `Steel method forward to guess the dynamic shapes (with Tiny-LLM)
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_export_tiny_llm.html>`_
+* `Export Tiny-LLM with patches
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_export_tiny_llm_patched.html>`_
+
+**Investigate ONNX models**
+
+* `Find where a model is failing by running submodels
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_failing_model_extract.html>`_
+* `Intermediate results with (ONNX) ReferenceEvaluator
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_failing_reference_evaluator.html>`_
+* `Intermediate results with onnxruntime
+  <https://sdpython.github.io/doc/onnx-diagnostic/dev/auto_examples/plot_failing_onnxruntime_evaluator.html>`_
+
+Snapshot of usefuls tools
++++++++++++++++++++++++++
+
+**torch_export_patches**
+
+.. code-block:: python
+
+  from onnx_diagnostic.torch_export_patches import torch_export_patches
+
+  with torch_export_patches(patch_transformers=True) as f:
+      ep = torch.export.export(model, args, kwargs=kwargs, dynamic_shapes=dynamic_shapes)
+      # ...
+
+**all_dynamic_shapes_from_inputs**
+
+.. code-block:: python
+
+  from onnx_diagnostic.export.shape_helper import all_dynamic_shapes_from_inputs
+
+  dynamic_shapes = all_dynamic_shapes_from_inputs(cache)
+
+**torch_export_rewrite**
+
+.. code-block:: python
+
+  from onnx_diagnostic.torch_export_patches import torch_export_rewrite
+
+  with torch_export_rewrite(rewrite=[Model.forward]) as f:
+      ep = torch.export.export(model, args, kwargs=kwargs, dynamic_shapes=dynamic_shapes)
+      # ...
+
+**string_type**
+
+.. code-block:: python
+
+    import torch
+    from onnx_diagnostic.helpers import string_type
+
+    inputs = (
+        torch.rand((3, 4), dtype=torch.float16),
+        [torch.rand((5, 6), dtype=torch.float16), torch.rand((5, 6, 7), dtype=torch.float16)],
+    )
+
+    # with shapes
+    print(string_type(inputs, with_shape=True))
+
+::
+
+    >>> (T10s3x4,#2[T10s5x6,T10s5x6x7])
+
+**onnx_dtype_name**
+
+.. code-block:: python
+
+        import onnx
+        from onnx_diagnostic.helpers.onnx_helper import onnx_dtype_name
+
+        itype = onnx.TensorProto.BFLOAT16
+        print(onnx_dtype_name(itype))
+        print(onnx_dtype_name(7))
+
+::
+
+    >>> BFLOAT16
+    >>> INT64
+
+**max_diff**
+
+.. code-block:: python
+
+    import torch
+    from onnx_diagnostic.helpers import max_diff
+
+    print(
+        max_diff(
+            (torch.Tensor([1, 2]), (torch.Tensor([1, 2]),)),
+            (torch.Tensor([1, 2]), (torch.Tensor([1, 2]),)),
+        )
+    )
+
+::
+
+    >>> {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 4.0, "dnan": 0.0}s
+
+**guess_dynamic_shapes**
+
+.. code-block:: python
+
+    inputs = [
+        (torch.randn((5, 6)), torch.randn((1, 6))),
+        (torch.randn((7, 8)), torch.randn((1, 8))),
+    ]
+    ds = ModelInputs(model, inputs).guess_dynamic_shapes(auto="dim")
+    print(ds)
+
+::
+
+    >>> (({0: 'dim_0I0', 1: 'dim_0I1'}, {1: 'dim_1I1'}), {})