onnx-diagnostic 0.7.10__py3-none-any.whl → 0.7.11__py3-none-any.whl

onnx_diagnostic/__init__.py

@@ -3,5 +3,5 @@ Patches, Investigates onnx models.
 Functions, classes to dig into a model when this one is right, slow, wrong...
 """
 
-__version__ = "0.7.10"
+__version__ = "0.7.11"
 __author__ = "Xavier Dupré"

onnx_diagnostic/_command_lines_parser.py

@@ -474,7 +474,7 @@ def get_parser_validate() -> ArgumentParser:
     )
     parser.add_argument(
         "--runtime",
-        choices=["onnxruntime", "torch", "ref"],
+        choices=["onnxruntime", "torch", "ref", "orteval", "orteval10"],
         default="onnxruntime",
         help="onnx runtime to use, `onnxruntime` by default",
     )
@@ -542,6 +542,12 @@ def get_parser_validate() -> ArgumentParser:
         "the onnx exporter should use.",
         default="",
     )
+    parser.add_argument(
+        "--ort-logs",
+        default=False,
+        action=BooleanOptionalAction,
+        help="Enables onnxruntime logging when the session is created",
+    )
     return parser
 
 
@@ -601,6 +607,7 @@ def _cmd_validate(argv: List[Any]):
             repeat=args.repeat,
             warmup=args.warmup,
             inputs2=args.inputs2,
+            ort_logs=args.ort_logs,
             output_names=(
                 None if len(args.outnames.strip()) < 2 else args.outnames.strip().split(",")
             ),

onnx_diagnostic/helpers/cache_helper.py

@@ -4,11 +4,6 @@ import torch
 import transformers
 import transformers.cache_utils
 
-try:
-    from transformers.models.mamba.modeling_mamba import MambaCache
-except ImportError:
-    from transformers.cache_utils import MambaCache
-
 
 class CacheKeyValue:
     """
@@ -354,8 +349,15 @@ def make_encoder_decoder_cache(
     )
 
 
-def make_mamba_cache(key_value_pairs: List[Tuple[torch.Tensor, torch.Tensor]]) -> MambaCache:
+def make_mamba_cache(
+    key_value_pairs: List[Tuple[torch.Tensor, torch.Tensor]],
+) -> "MambaCache":  # noqa: F821
     "Creates a ``MambaCache``."
+    # import is moved here because this part is slow.
+    try:
+        from transformers.models.mamba.modeling_mamba import MambaCache
+    except ImportError:
+        from transformers.cache_utils import MambaCache
     dtype = key_value_pairs[0][0].dtype
 
     class _config:

onnx_diagnostic/tasks/__init__.py

@@ -5,6 +5,8 @@ from . import (
     fill_mask,
     image_classification,
     image_text_to_text,
+    image_to_video,
+    mask_generation,
     mixture_of_expert,
     object_detection,
     sentence_similarity,
@@ -14,7 +16,6 @@ from . import (
     text_to_image,
     text2text_generation,
     zero_shot_image_classification,
-    mask_generation,
 )
 
 __TASKS__ = [
@@ -23,6 +24,8 @@ __TASKS__ = [
     fill_mask,
     image_classification,
     image_text_to_text,
+    image_to_video,
+    mask_generation,
     mixture_of_expert,
     object_detection,
     sentence_similarity,
@@ -32,7 +35,6 @@ __TASKS__ = [
     text_to_image,
     text2text_generation,
     zero_shot_image_classification,
-    mask_generation,
 ]
 
 

onnx_diagnostic/tasks/image_to_video.py

@@ -0,0 +1,127 @@
+from typing import Any, Callable, Dict, Optional, Tuple
+import torch
+from ..helpers.config_helper import (
+    update_config,
+    check_hasattr,
+    default_num_hidden_layers as nhl,
+)
+
+__TASK__ = "image-to-video"
+
+
+def reduce_model_config(config: Any) -> Dict[str, Any]:
+    """Reduces a model size."""
+    if not hasattr(config, "num_hidden_layers") and not hasattr(config, "num_layers"):
+        # We cannot reduce.
+        return {}
+    check_hasattr(config, ("num_hidden_layers", "num_layers"))
+    kwargs = {}
+    if hasattr(config, "num_layers"):
+        kwargs["num_layers"] = min(config.num_layers, nhl())
+    if hasattr(config, "num_hidden_layers"):
+        kwargs["num_hidden_layers"] = min(config.num_hidden_layers, nhl())
+
+    update_config(config, kwargs)
+    return kwargs
+
+
+def get_inputs(
+    model: torch.nn.Module,
+    config: Optional[Any],
+    text_embed_dim: int,
+    latent_channels: int,
+    batch_size: int = 2,
+    image_height: int = 704,
+    image_width: int = 1280,
+    latent_frames: int = 1,
+    text_maxlen: int = 512,
+    add_second_input: int = 1,
+    **kwargs,  # unused
+):
+    """
+    Generates inputs for task ``image-to-video``.
+    """
+    assert (
+        "cls_cache" not in kwargs
+    ), f"Not yet implemented for cls_cache={kwargs['cls_cache']!r}."
+    latent_height = image_height // 8
+    latent_width = image_width // 8
+    dtype = torch.float32
+
+    inputs = dict(
+        hidden_states=torch.randn(
+            batch_size,
+            latent_channels,
+            latent_frames,
+            latent_height,
+            latent_width,
+            dtype=dtype,
+        ),
+        timestep=torch.tensor([1.0] * batch_size, dtype=dtype),
+        encoder_hidden_states=torch.randn(
+            batch_size, text_maxlen, text_embed_dim, dtype=dtype
+        ),
+        padding_mask=torch.ones(1, 1, image_height, image_width, dtype=dtype),
+        fps=torch.tensor([16] * batch_size, dtype=dtype),
+        condition_mask=torch.randn(
+            batch_size, 1, latent_frames, latent_height, latent_width, dtype=dtype
+        ),
+    )
+    shapes = dict(
+        hidden_states={
+            0: "batch_size",
+            2: "latent_frames",
+            3: "latent_height",
+            4: "latent_width",
+        },
+        timestep={0: "batch_size"},
+        encoder_hidden_states={0: "batch_size"},
+        padding_mask={0: "batch_size", 2: "height", 3: "width"},
+        fps={0: "batch_size"},
+        condition_mask={
+            0: "batch_size",
+            2: "latent_frames",
+            3: "latent_height",
+            4: "latent_width",
+        },
+    )
+    res = dict(inputs=inputs, dynamic_shapes=shapes)
+
+    if add_second_input:
+        assert (
+            add_second_input > 0
+        ), f"Not implemented for add_second_input={add_second_input}."
+        res["inputs2"] = get_inputs(
+            model=model,
+            config=config,
+            text_embed_dim=text_embed_dim,
+            latent_channels=latent_channels,
+            batch_size=batch_size,
+            image_height=image_height,
+            image_width=image_width,
+            latent_frames=latent_frames,
+            text_maxlen=text_maxlen,
+            add_second_input=0,
+            **kwargs,
+        )["inputs"]
+    return res
+
+
+def random_input_kwargs(config: Any) -> Tuple[Dict[str, Any], Callable]:
+    """
+    Inputs kwargs.
+
+    If the configuration is None, the function selects typical dimensions.
+    """
+    if config is not None:
+        check_hasattr(config, "in_channels", "text_embed_dim"),
+    kwargs = dict(
+        text_embed_dim=1024 if config is None else config.text_embed_dim,
+        latent_channels=16 if config is None else config.in_channels - 1,
+        batch_size=1,
+        image_height=8 * 50,
+        image_width=8 * 80,
+        latent_frames=1,
+        text_maxlen=512,
+    )
+    return kwargs, get_inputs

onnx_diagnostic/torch_export_patches/patches/patch_transformers.py

@@ -35,6 +35,9 @@ except ImportError:
 from ...ext_test_case import has_transformers
 from ...helpers.torch_helper import is_torchdynamo_exporting
 
+patch_is_initialized = pv.Version(transformers.__version__) > pv.Version("4.56.99")
+
+
 if patch_masking_utils:
     # Introduced in 4.52
     from transformers.masking_utils import (
@@ -213,6 +216,8 @@ if patch_DynamicLayer:
             new_shape[-2] = 0
             self.keys = torch.empty(new_shape, dtype=self.dtype, device=self.device)
             self.values = torch.empty(new_shape, dtype=self.dtype, device=self.device)
+            if patch_is_initialized:
+                self.is_initialized = True
 
 
 def _patch_make_causal_mask(

onnx_diagnostic/torch_models/hghub/hub_api.py

@@ -177,6 +177,51 @@ def task_from_arch(
     return data[arch]
 
 
+def _trygetattr(config, attname):
+    try:
+        return getattr(config, attname)
+    except AttributeError:
+        return None
+
+
+def architecture_from_config(config) -> Optional[str]:
+    """Guesses the architecture (class) of the model described by this config."""
+    if isinstance(config, dict):
+        if "_class_name" in config:
+            return config["_class_name"]
+        if "architecture" in config:
+            return config["architecture"]
+        if config.get("architectures", []):
+            return config["architectures"][0]
+    if hasattr(config, "_class_name"):
+        return config._class_name
+    if hasattr(config, "architecture"):
+        return config.architecture
+    if hasattr(config, "architectures") and config.architectures:
+        return config.architectures[0]
+    if hasattr(config, "__dict__"):
+        if "_class_name" in config.__dict__:
+            return config.__dict__["_class_name"]
+        if "architecture" in config.__dict__:
+            return config.__dict__["architecture"]
+        if config.__dict__.get("architectures", []):
+            return config.__dict__["architectures"][0]
+    return None
+
+
+def find_package_source(config) -> Optional[str]:
+    """Guesses the package the class models from."""
+    if isinstance(config, dict):
+        if "_diffusers_version" in config:
+            return "diffusers"
+    if hasattr(config, "_diffusers_version"):
+        return "diffusers"
+    if hasattr(config, "__dict__"):
+        if "_diffusers_version" in config.__dict__:
+            return "diffusers"
+    return "transformers"
+
+
 def task_from_id(
     model_id: str,
     default_value: Optional[str] = None,
@@ -202,28 +247,30 @@ def task_from_id(
             if not fall_back_to_pretrained:
                 raise
     config = get_pretrained_config(model_id, subfolder=subfolder)
-    try:
-        return config.pipeline_tag
-    except AttributeError:
-        guess = _guess_task_from_config(config)
-        if guess is not None:
-            return guess
-        data = load_architecture_task()
-        if model_id in data:
-            return data[model_id]
-        if type(config) is dict and "_class_name" in config:
-            return task_from_arch(config["_class_name"], default_value=default_value)
-        if not config.architectures or not config.architectures:
-            # Some hardcoded values until a better solution is found.
-            if model_id.startswith("google/bert_"):
-                return "fill-mask"
-        assert config.architectures is not None and len(config.architectures) == 1, (
-            f"Cannot return the task of {model_id!r}, pipeline_tag is not setup, "
-            f"architectures={config.architectures} in config={config}. "
-            f"The task can be added in "
-            f"``onnx_diagnostic.torch_models.hghub.hub_data.__data_arch__``."
-        )
-        return task_from_arch(config.architectures[0], default_value=default_value)
+    tag = _trygetattr(config, "pipeline_tag")
+    if tag is not None:
+        return tag
+
+    guess = _guess_task_from_config(config)
+    if guess is not None:
+        return guess
+    data = load_architecture_task()
+    if subfolder:
+        full_id = f"{model_id}//{subfolder}"
+        if full_id in data:
+            return data[full_id]
+    if model_id in data:
+        return data[model_id]
+    arch = architecture_from_config(config)
+    if arch is None:
+        if model_id.startswith("google/bert_"):
+            return "fill-mask"
+    assert arch is not None, (
+        f"Cannot return the task of {model_id!r}, pipeline_tag is not setup, "
+        f"config={config}. The task can be added in "
+        f"``onnx_diagnostic.torch_models.hghub.hub_data.__data_arch__``."
+    )
+    return task_from_arch(arch, default_value=default_value)
 
 
 def task_from_tags(tags: Union[str, List[str]]) -> str:

onnx_diagnostic/torch_models/hghub/hub_data.py

@@ -30,6 +30,7 @@ __data_arch__ = textwrap.dedent(
     ConvBertModel,feature-extraction
     ConvNextForImageClassification,image-classification
     ConvNextV2Model,image-feature-extraction
+    CosmosTransformer3DModel,image-to-video
     CvtModel,feature-extraction
     DPTModel,image-feature-extraction
     Data2VecAudioModel,feature-extraction
@@ -156,7 +157,8 @@ __data_arch__ = textwrap.dedent(
     YolosForObjectDetection,object-detection
     YolosModel,image-feature-extraction
     Alibaba-NLP/gte-large-en-v1.5,sentence-similarity
-    emilyalsentzer/Bio_ClinicalBERT,fill-mask"""
+    emilyalsentzer/Bio_ClinicalBERT,fill-mask
+    nvidia/Cosmos-Predict2-2B-Video2World//transformer,image-to-video"""
 )
 
 __data_tasks__ = [

onnx_diagnostic/torch_models/hghub/model_inputs.py

@@ -2,13 +2,21 @@ import copy
 import inspect
 import os
 import pprint
+import time
 from typing import Any, Dict, Optional, Tuple
 import torch
 import transformers
 from ...helpers.config_helper import update_config, build_diff_config
 from ...tasks import reduce_model_config, random_input_kwargs
-from .hub_api import task_from_arch, task_from_id, get_pretrained_config, download_code_modelid
-from .model_specific import HANDLED_MODELS, load_specific_model
+from .hub_api import (
+    task_from_arch,
+    task_from_id,
+    get_pretrained_config,
+    download_code_modelid,
+    architecture_from_config,
+    find_package_source,
+)
+from .model_specific import HANDLED_MODELS, load_specific_model, instantiate_specific_model
 
 
 def _code_needing_rewriting(model: Any) -> Any:
@@ -96,27 +104,18 @@ def get_untrained_model_with_inputs(
             model, task, config = load_specific_model(model_id, verbose=verbose)
 
     if model is None:
-        if hasattr(config, "architecture") and config.architecture:
-            archs = [config.architecture]
-        if type(config) is dict:
-            assert (
-                "_class_name" in config
-            ), f"Unable to get the architecture from config={config}"
-            archs = [config["_class_name"]]
-        else:
-            archs = config.architectures  # type: ignore
-        task = None
-        if archs is None:
-            task = task_from_id(model_id)
-        assert task is not None or (archs is not None and len(archs) == 1), (
+        arch = architecture_from_config(config)
+        if arch is None:
+            task = task_from_id(model_id, subfolder=subfolder)
+        assert task is not None or arch is not None, (
             f"Unable to determine the architecture for model {model_id!r}, "
-            f"architectures={archs!r}, conf={config}"
+            f"archs={arch!r}, conf={config}"
         )
         if verbose:
-            print(f"[get_untrained_model_with_inputs] architectures={archs!r}")
+            print(f"[get_untrained_model_with_inputs] architecture={arch!r}")
             print(f"[get_untrained_model_with_inputs] cls={config.__class__.__name__!r}")
         if task is None:
-            task = task_from_arch(archs[0], model_id=model_id, subfolder=subfolder)
+            task = task_from_arch(arch, model_id=model_id, subfolder=subfolder)
         if verbose:
             print(f"[get_untrained_model_with_inputs] task={task!r}")
 
@@ -170,36 +169,58 @@ def get_untrained_model_with_inputs(
                 f"{getattr(config, '_attn_implementation', '?')!r}"  # type: ignore[union-attr]
             )
 
-        if type(config) is dict and "_diffusers_version" in config:
+        if find_package_source(config) == "diffusers":
             import diffusers
 
             package_source = diffusers
         else:
             package_source = transformers
 
+        if verbose:
+            print(
+                f"[get_untrained_model_with_inputs] package_source={package_source.__name__} é"
+                f"from {package_source.__file__}"
+            )
         if use_pretrained:
+            begin = time.perf_counter()
+            if verbose:
+                print(
+                    f"[get_untrained_model_with_inputs] pretrained model_id {model_id!r}, "
+                    f"subfolder={subfolder!r}"
+                )
             model = transformers.AutoModel.from_pretrained(
-                model_id, trust_remote_code=True, **mkwargs
+                model_id, subfolder=subfolder, trust_remote_code=True, **mkwargs
             )
+            if verbose:
+                print(
+                    f"[get_untrained_model_with_inputs] -- done in "
+                    f"{time.perf_counter() - begin}s"
+                )
         else:
-            if archs is not None:
+            begin = time.perf_counter()
+            if verbose:
+                print(
+                    f"[get_untrained_model_with_inputs] instantiate model_id {model_id!r}, "
+                    f"subfolder={subfolder!r}"
+                )
+            if arch is not None:
                 try:
-                    cls_model = getattr(package_source, archs[0])
+                    cls_model = getattr(package_source, arch)
                 except AttributeError as e:
                     # The code of the models is not in transformers but in the
                     # repository of the model. We need to download it.
                     pyfiles = download_code_modelid(model_id, verbose=verbose)
                     if pyfiles:
-                        if "." in archs[0]:
-                            cls_name = archs[0]
+                        if "." in arch:
+                            cls_name = arch
                         else:
                             modeling = [_ for _ in pyfiles if "/modeling_" in _]
                             assert len(modeling) == 1, (
                                 f"Unable to guess the main file implemented class "
-                                f"{archs[0]!r} from {pyfiles}, found={modeling}."
+                                f"{arch!r} from {pyfiles}, found={modeling}."
                             )
                             last_name = os.path.splitext(os.path.split(modeling[0])[-1])[0]
-                            cls_name = f"{last_name}.{archs[0]}"
+                            cls_name = f"{last_name}.{arch}"
                         if verbose:
                             print(
                                 f"[get_untrained_model_with_inputs] "
@@ -217,7 +238,7 @@ def get_untrained_model_with_inputs(
                         )
                     else:
                         raise AttributeError(
-                            f"Unable to find class 'tranformers.{archs[0]}'. "
+                            f"Unable to find class 'tranformers.{arch}'. "
                             f"The code needs to be downloaded, config="
                             f"\n{pprint.pformat(config)}."
                         ) from e
@@ -225,20 +246,31 @@ def get_untrained_model_with_inputs(
                 assert same_as_pretrained and use_pretrained, (
                     f"Model {model_id!r} cannot be built, the model cannot be built. "
                     f"It must be downloaded. Use same_as_pretrained=True "
-                    f"and use_pretrained=True."
+                    f"and use_pretrained=True, arch={arch!r}, config={config}"
+                )
+            if verbose:
+                print(
+                    f"[get_untrained_model_with_inputs] -- done in "
+                    f"{time.perf_counter() - begin}s"
                 )
 
-            try:
-                if type(config) is dict:
-                    model = cls_model(**config)
-                else:
-                    model = cls_model(config)
-            except RuntimeError as e:
-                raise RuntimeError(
-                    f"Unable to instantiate class {cls_model.__name__} with\n{config}"
-                ) from e
+            seed = int(os.environ.get("SEED", "17"))
+            torch.manual_seed(seed)
+            model = instantiate_specific_model(cls_model, config)
+            if model is None:
+                try:
+                    if type(config) is dict:
+                        model = cls_model(**config)
+                    else:
+                        model = cls_model(config)
+                except RuntimeError as e:
+                    raise RuntimeError(
+                        f"Unable to instantiate class {cls_model.__name__} with\n{config}"
+                    ) from e
 
     # input kwargs
+    seed = int(os.environ.get("SEED", "17")) + 1
+    torch.manual_seed(seed)
     kwargs, fct = random_input_kwargs(config, task)  # type: ignore[arg-type]
     if verbose:
         print(f"[get_untrained_model_with_inputs] use fct={fct}")
@@ -250,7 +282,7 @@ def get_untrained_model_with_inputs(
 
     # This line is important. Some models may produce different
     # outputs even with the same inputs in training mode.
-    model.eval()
+    model.eval()  # type: ignore[union-attr]
     res = fct(model, config, add_second_input=add_second_input, **kwargs)
 
     res["input_kwargs"] = kwargs

onnx_diagnostic/torch_models/hghub/model_specific.py

@@ -1,6 +1,33 @@
 from typing import Any, Dict, Tuple
 
 
+def instantiate_specific_model(cls_model: type, config: Any) -> object:
+    """
+    Instantiates some model requiring some specific code.
+    """
+    if cls_model.__name__ == "CosmosTransformer3DModel":
+        return instantiate_CosmosTransformer3DModel(cls_model, config)
+    return None
+
+
+def instantiate_CosmosTransformer3DModel(cls_model: type, config: Any) -> object:
+    kwargs = dict(
+        in_channels=config.in_channels,
+        out_channels=config.out_channels,
+        attention_head_dim=config.attention_head_dim,
+        mlp_ratio=config.mlp_ratio,
+        num_layers=config.num_layers,
+        text_embed_dim=config.text_embed_dim,
+        adaln_lora_dim=config.adaln_lora_dim,
+        max_size=config.max_size,
+        patch_size=config.patch_size,
+        rope_scale=config.rope_scale,
+        concat_padding_mask=config.concat_padding_mask,
+        extra_pos_embed_type=config.extra_pos_embed_type,
+    )
+    return cls_model(**kwargs)
+
+
 class SpecificConfig:
     """Creates a specific configuration for the loaded model."""
 

onnx_diagnostic/torch_models/validate.py

@@ -7,8 +7,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 import time
 import numpy as np
 import onnx
-import onnxscript
-import onnxscript.rewriter.ort_fusions as ort_fusions
 import torch
 from ..export import CoupleInputsDynamicShapes
 from ..helpers import max_diff, string_type, string_diff
@@ -113,6 +111,7 @@ def _make_folder_name(
     dtype: Optional[Union[str, torch.dtype]] = None,
     device: Optional[Union[str, torch.device]] = None,
     subfolder: Optional[str] = None,
+    opset: Optional[int] = None,
 ) -> str:
     "Creates a filename unique based on the given options."
     els = [model_id.replace("/", "_")]
@@ -136,6 +135,8 @@ def _make_folder_name(
         else:
             raise AssertionError(f"unexpected value for device={device}, sdev={sdev!r}")
         els.append(sdev)
+    if opset is not None:
+        els.append(f"op{opset}")
     return "-".join(els)
 
 
@@ -246,6 +247,7 @@ def _quiet_or_not_quiet(
         summary[f"time_{suffix}_latency_std"] = a.std()
         summary[f"time_{suffix}_latency_min"] = a.min()
         summary[f"time_{suffix}_latency_min"] = a.max()
+        summary[f"time_{suffix}_n"] = len(a)
     return res
 
 
@@ -262,6 +264,16 @@ def shrink_config(cfg: Dict[str, Any]) -> Dict[str, Any]:
     return new_cfg
 
 
+def _preprocess_model_id(model_id, subfolder):
+    if subfolder or "//" not in model_id:
+        return model_id, subfolder
+    spl = model_id.split("//")
+    if spl[-1] in {"transformer", "vae"}:
+        # known subfolder
+        return "//".join(spl[:-1]), spl[-1]
+    return model_id, subfolder
+
+
 def validate_model(
     model_id: str,
     task: Optional[str] = None,
@@ -290,6 +302,7 @@ def validate_model(
     warmup: int = 0,
     inputs2: int = 1,
     output_names: Optional[List[str]] = None,
+    ort_logs: bool = False,
 ) -> Tuple[Dict[str, Union[int, float, str]], Dict[str, Any]]:
     """
     Validates a model.
@@ -334,13 +347,15 @@ def validate_model(
     :param subfolder: version or subfolders to uses when retrieving a model id
     :param opset: onnx opset to use for the conversion
     :param runtime: onnx runtime to use to check about discrepancies,
-        only if `do_run` is true
+        possible values ``onnxruntime``, ``torch``, ``orteval``,
+        ``orteval10``, ``ref`` only if `do_run` is true
     :param repeat: number of time to measure the model
     :param warmup: warmup the model first
     :param inputs2: checks that the second set of inputs is reunning as well,
         this ensures that the model does support dynamism, the value is used
         as an increment to the first set of values (added to dimensions)
     :param output_names: output names the onnx exporter should use
+    :param ort_logs: increases onnxruntime verbosity when creating the session
     :return: two dictionaries, one with some metrics,
         another one with whatever the function produces
 
@@ -361,8 +376,15 @@ def validate_model(
 
     The default runtime, :epkg:`onnxruntime` is used to validate a model and check the
     exported model returns the same outputs as the original one, otherwise,
-    :class:`onnx_diagnostic.reference.TorchOnnxEvaluator` is used.
+    :class:`onnx_diagnostic.reference.TorchOnnxEvaluator`
+    if ``runtime == 'torch'`` or
+    :class:`onnx_diagnostic.reference.OnnxruntimeEvaluator`
+    if ``runtime == 'orteval'`` or
+    :class:`onnx_diagnostic.reference.ExtendedReferenceEvaluator`
+    if ``runtime == 'ref'``,
+    ``orteval10`` increases the verbosity.
     """
+    model_id, subfolder = _preprocess_model_id(model_id, subfolder)
     if isinstance(patch, bool):
         patch_kwargs = (
             dict(patch_transformers=True, patch_diffusers=True, patch=True)
@@ -412,7 +434,13 @@ def validate_model(
     folder_name = None
     if dump_folder:
         folder_name = _make_folder_name(
-            model_id, exporter, optimization, dtype=dtype, device=device, subfolder=subfolder
+            model_id,
+            exporter,
+            optimization,
+            dtype=dtype,
+            device=device,
+            subfolder=subfolder,
+            opset=opset,
         )
         dump_folder = os.path.join(dump_folder, folder_name)
         if not os.path.exists(dump_folder):
@@ -743,6 +771,7 @@ def validate_model(
             repeat=repeat,
             warmup=warmup,
             inputs2=inputs2,
+            ort_logs=ort_logs,
         )
         summary.update(summary_valid)
 
@@ -837,15 +866,24 @@ def compute_statistics(onnx_filename: str) -> Dict[str, Union[float, int]]:
             raise NotImplementedError(f"Unexpected type={type(proto)}")
 
     counts: Dict[str, Union[float, int]] = {}
+    n_nodes = 0
+    n_nodes_nocst = 0
     for proto in node_iter(onx):
         if isinstance(proto, onnx.NodeProto):
             key = f"n_node_{proto.op_type}"
+            n_nodes += 1
+            if proto.op_type != "Constant":
+                n_nodes_nocst += 1
         else:
             key = f"n_node_initializer_{proto.data_type}"
 
         if key not in counts:
             counts[key] = 0
         counts[key] += 1
+
+    counts["n_node_nodes"] = n_nodes
+    counts["n_node_nodes_nocst"] = n_nodes_nocst
+    counts["n_node_functions"] = len(onx.functions)
     return counts
 
 
@@ -1134,6 +1172,7 @@ def validate_onnx_model(
     repeat: int = 1,
     warmup: int = 0,
     inputs2: int = 1,
+    ort_logs: bool = False,
 ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """
     Verifies that an onnx model produces the same
@@ -1146,12 +1185,13 @@ def validate_onnx_model(
     :param quiet: catch exception or not
     :param verbose: verbosity
     :param flavour: use a different version of the inputs
-    :param runtime: onnx runtime to use, onnxruntime or torch
+    :param runtime: onnx runtime to use, onnxruntime, torch, orteval, ref
     :param repeat: run that number of times the model
     :param warmup: warmup the model
     :param inputs2: to validate the model on the second input set
         to make sure the exported model supports dynamism, the value is
         used as an increment added to the first set of inputs (added to dimensions)
+    :param ort_logs: triggers the logs for onnxruntime
     :return: two dictionaries, one with some metrics,
         another one with whatever the function produces
     """
@@ -1193,23 +1233,71 @@ def validate_onnx_model(
             f"{providers}..., flavour={flavour!r}"
         )
 
-    if runtime != "onnxruntime":
+    if runtime == "onnxruntime":
+        if os.environ.get("DUMPORTOPT", "") in ("1", "true", "True"):
+            opts = onnxruntime.SessionOptions()
+            opts.optimized_model_filepath = f"{data['onnx_filename']}.rtopt.onnx"
+            if verbose:
+                print(
+                    f"[validate_onnx_model] saved optimized onnxruntime "
+                    f"in {opts.optimized_model_filepath!r}"
+                )
+            onnxruntime.InferenceSession(data["onnx_filename"], opts, providers=providers)
+            if verbose:
+                print("[validate_onnx_model] -- done")
+
+        if verbose:
+            print("[validate_onnx_model] runtime is onnxruntime")
+        sess_opts = onnxruntime.SessionOptions()
+        if ort_logs:
+            sess_opts.log_severity_level = 0
+            sess_opts.log_verbosity_level = 4
+        cls_runtime = lambda model, providers, _o=sess_opts: onnxruntime.InferenceSession(
+            (model.SerializeToString() if isinstance(model, onnx.ModelProto) else model),
+            _o,
+            providers=providers,
+        )
+    elif runtime == "torch":
         from ..reference import TorchOnnxEvaluator
 
-    cls_runtime = (
-        (
-            lambda model, providers: onnxruntime.InferenceSession(
-                (model.SerializeToString() if isinstance(model, onnx.ModelProto) else model),
-                providers=providers,
+        if verbose:
+            print("[validate_onnx_model] runtime is TorchOnnxEvaluator")
+        cls_runtime = (
+            lambda model, providers, _cls_=TorchOnnxEvaluator: _cls_(  # type: ignore[misc]
+                model, providers=providers, verbose=max(verbose - 1, 0)
             )
         )
-        if runtime == "onnxruntime"
-        else (
-            lambda model, providers, _cls_=TorchOnnxEvaluator: _cls_(  # type: ignore[misc]
+    elif runtime == "orteval":
+        from ..reference import OnnxruntimeEvaluator
+
+        if verbose:
+            print("[validate_onnx_model] runtime is OnnxruntimeEvaluator")
+        cls_runtime = (
+            lambda model, providers, _cls_=OnnxruntimeEvaluator: _cls_(  # type: ignore[misc]
                 model, providers=providers, verbose=max(verbose - 1, 0)
             )
         )
-    )
+    elif runtime == "orteval10":
+        from ..reference import OnnxruntimeEvaluator
+
+        if verbose:
+            print("[validate_onnx_model] runtime is OnnxruntimeEvaluator(verbose=10)")
+        cls_runtime = (
+            lambda model, providers, _cls_=OnnxruntimeEvaluator: _cls_(  # type: ignore[misc]
+                model, providers=providers, verbose=10
+            )
+        )
+    elif runtime == "ref":
+        from ..reference import ExtendedReferenceEvaluator
+
+        if verbose:
+            print("[validate_onnx_model] runtime is ExtendedReferenceEvaluator")
+        cls_runtime = lambda model, providers, _cls_=ExtendedReferenceEvaluator: _cls_(  # type: ignore[misc]
+            model, verbose=max(verbose - 1, 0)
+        )
+    else:
+        raise ValueError(f"Unexpecteed runtime={runtime!r}")
+
     sess = _quiet_or_not_quiet(
         quiet,
         _mk("create_onnx_ort"),
@@ -1390,6 +1478,8 @@ def call_torch_export_onnx(
         if optimization == "ir":
             label, f_optim = "export_onnx_opt_ir", (lambda epo=epo: epo.optimize())
         else:
+            import onnxscript
+            import onnxscript.rewriter.ort_fusions as ort_fusions
 
             def _os_ort_optim(epo):
                 onnxscript.optimizer.optimize_ir(epo.model)
@@ -1509,6 +1599,8 @@ def call_torch_export_custom(
         "default+onnxruntime+os_ort",
         None,
     }
+    if optimization == "none":
+        optimization = ""
     assert (
         optimization in available
     ), f"unexpected value for optimization={optimization}, available={available}"
@@ -1672,6 +1764,9 @@ def call_torch_export_custom(
         print("[call_torch_export_custom] done (export)")
 
     if os_ort:
+        import onnxscript
+        import onnxscript.rewriter.ort_fusions as ort_fusions
+
         if verbose:
             print("[call_torch_export_custom] conversion to IR...")
         begin = time.perf_counter()

{onnx_diagnostic-0.7.10.dist-info → onnx_diagnostic-0.7.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: onnx-diagnostic
-Version: 0.7.10
+Version: 0.7.11
 Summary: Investigate ONNX models
 Home-page: https://github.com/sdpython/onnx-diagnostic
 Author: Xavier Dupré

{onnx_diagnostic-0.7.10.dist-info → onnx_diagnostic-0.7.11.dist-info}/RECORD

@@ -1,6 +1,6 @@
-onnx_diagnostic/__init__.py,sha256=zdvFsRTL3vL-gFelvSYnpPCXNbg8EPwbC3qQ47KhLbw,174
+onnx_diagnostic/__init__.py,sha256=tyRANqD6rauk6F7FpFJN5K1FyB1baNeni92_ol7nrdU,174
 onnx_diagnostic/__main__.py,sha256=YmyV_Aq_ianDlHyKLHMa6h8YK3ZmFPpLVHLKjM91aCk,79
-onnx_diagnostic/_command_lines_parser.py,sha256=TVPlDjsWZd_Zb9DzN3zj0OGxd8nz_nUsjQyGkmyMNsA,32939
+onnx_diagnostic/_command_lines_parser.py,sha256=EljrcTEKF4TuSdougR3i3FL4_jVDG8xizrLsLIA2JRs,33185
 onnx_diagnostic/api.py,sha256=BhCl_yCd78N7TlVtPOHjeYv1QBEy39TjZ647rcHqLh0,345
 onnx_diagnostic/doc.py,sha256=t3RELgfooYnVMAi0JSpggWkQEgUsREz8NmRvn0TnLI8,2829
 onnx_diagnostic/ext_test_case.py,sha256=emfQGiQSz5FVDhyJ1Acsv_Tast7tWl426TjtpNqxDBU,43558
@@ -12,7 +12,7 @@ onnx_diagnostic/helpers/__init__.py,sha256=GJ2GT7cgnlIveVUwMZhuvUwidbTJaKv8CsSIO
 onnx_diagnostic/helpers/_log_helper.py,sha256=OTwQH0OIxs9B6nrSvR7MoxMimSw_8mU0mj133NvLk5o,16832
 onnx_diagnostic/helpers/args_helper.py,sha256=SRWnqC7EENg09RZlA50B_PcdiIhdbgA4C3ACfzl5nMs,4419
 onnx_diagnostic/helpers/bench_run.py,sha256=CGA6VMJZMH2gDhVueT9ypNm4PMcjGrrGFYp08nhWj9k,16539
-onnx_diagnostic/helpers/cache_helper.py,sha256=-2H4hMO5ZIINsaJS7mK9ETgv-kA_d-dlwT1TDp2Yjbo,24754
+onnx_diagnostic/helpers/cache_helper.py,sha256=zxjm0-3lHs0A7wLEejz2r2KPMPjkkva--8511MaSy74,24846
 onnx_diagnostic/helpers/config_helper.py,sha256=H2mOcMXfrcolFnt8EuqmRFkpQ3YdNRDfvm9ToI1vNH0,5618
 onnx_diagnostic/helpers/doc_helper.py,sha256=pl5MZd3_FaE8BqQnqoBuSBxoNCFcd2OJd3eITUSku5c,5897
 onnx_diagnostic/helpers/graph_helper.py,sha256=hevQT5a7_QuriVPQcbT5qe18n99Doyl5h3-qshx1-uk,14093
@@ -72,12 +72,13 @@ onnx_diagnostic/reference/torch_ops/reduce_ops.py,sha256=9gFfraPTQbe_ZEUNCUis1JS
 onnx_diagnostic/reference/torch_ops/sequence_ops.py,sha256=3EiVKpGfN4d1Iry4hgnr3MIJyEEKUrAIDgmRGsUXXa0,2297
 onnx_diagnostic/reference/torch_ops/shape_ops.py,sha256=pJrNR2UB4PlWl6cv4EDl1uGl8YTBUUMQkhJcsh5K4sA,4291
 onnx_diagnostic/reference/torch_ops/unary_ops.py,sha256=dwu6HPr4V_roxu85U3VLTtDLx5bfxKalT_-zlQxZ5wc,1850
-onnx_diagnostic/tasks/__init__.py,sha256=uWFP7HIr-VnxmXD5i_QAfXnLXc1HwUq2e8v9cKLqraQ,2492
+onnx_diagnostic/tasks/__init__.py,sha256=KHMH-YONqUQD3tT6N995wyZuF0R4NIZlIH8moumqmRc,2532
 onnx_diagnostic/tasks/automatic_speech_recognition.py,sha256=umZmjGW1gDUFkqvBJnQyaL7D7-HqiwlQpsq6Ip187Dg,7150
 onnx_diagnostic/tasks/feature_extraction.py,sha256=Zh9p_Q8FqEO2_aqI0cCiq8OXuM3WUZbwItlLOmLnNl8,5537
 onnx_diagnostic/tasks/fill_mask.py,sha256=5Gt6zlj0p6vuifox7Wmj-TpHXJvPS0CEH8evgdBHDNA,2640
 onnx_diagnostic/tasks/image_classification.py,sha256=nLpBBB1Gkog3Fk6pu2waiHcuQr4ILPptc9FhQ-pn460,4682
 onnx_diagnostic/tasks/image_text_to_text.py,sha256=wkFrUaEvQAW-D-jql2xSnae1XvQBl-sSbhmAmJ76qGo,17428
+onnx_diagnostic/tasks/image_to_video.py,sha256=SoF2cVIJr6P30Abp-FCuixFDh5RvTuNEOL36QthGY6U,3860
 onnx_diagnostic/tasks/mask_generation.py,sha256=fjdD3rd-O-mFL0hQy3la3JXKth_0bH2HL7Eelq-3Dbs,5057
 onnx_diagnostic/tasks/mixture_of_expert.py,sha256=al4tk1BrHidtRiHlAaiflWiJaAte0d5M8WcBioANG9k,2808
 onnx_diagnostic/tasks/object_detection.py,sha256=3FiT8ya5FCd9lwjQCRXhAwXspNwYTlAD3Gpk8aAcG5w,4279
@@ -99,27 +100,27 @@ onnx_diagnostic/torch_export_patches/eval/__init__.py,sha256=57x62uZNA80XiWgkG8F
 onnx_diagnostic/torch_export_patches/eval/model_cases.py,sha256=OU8-63VDhiWtQV3scBV9JyGXn8ds74OzY2-IOZkwg0A,26580
 onnx_diagnostic/torch_export_patches/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_export_patches/patches/patch_torch.py,sha256=TFjuw--sTYPCoVEaYlYLJuElx_CUynJR6s6ypoZtRWw,18956
-onnx_diagnostic/torch_export_patches/patches/patch_transformers.py,sha256=wXopyo0-6KmATOfqXMLEvxpe_jDRRIY8fWRjUjMlSkI,67776
+onnx_diagnostic/torch_export_patches/patches/patch_transformers.py,sha256=SsN-y2yoVaY3xRGDaIl0V449LcuwKAGBHPKm2JjQNhc,67942
 onnx_diagnostic/torch_export_patches/serialization/__init__.py,sha256=BHLdRPtNAtNPAS-bPKEj3-foGSPvwAbZXrHzGGPDLEw,1876
 onnx_diagnostic/torch_export_patches/serialization/diffusers_impl.py,sha256=drq3EH_yjcSuIWYsVeUWm8Cx6YCZFU6bP_1PLtPfY5I,945
 onnx_diagnostic/torch_export_patches/serialization/transformers_impl.py,sha256=mcmZGekzQlLgE_o3SdKlRgCx4ewwyyAuNWZ9CaN_zrI,9317
 onnx_diagnostic/torch_models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_models/llms.py,sha256=soyg4yC87ptGoeulJhKqw5opGmuLvH1pn_ZDXZ4Jr8E,90
-onnx_diagnostic/torch_models/validate.py,sha256=Qu9gW1AatgpmsWzXN3s-vVCKnKDYTV1wPO3wnUU44wU,67161
+onnx_diagnostic/torch_models/validate.py,sha256=3UJzjH89dpa_pyFoFG_fZ2IwOa25gtC7RxHjKX7c2PQ,70887
 onnx_diagnostic/torch_models/hghub/__init__.py,sha256=vi1Q7YHdddj1soiBN42MSvJdFqe2_KUoWafHISjwOu8,58
-onnx_diagnostic/torch_models/hghub/hub_api.py,sha256=Bvr-sTAhS6s6UCkt-KsY_7Mdai08-AQzvHrzbYCSuvk,13186
-onnx_diagnostic/torch_models/hghub/hub_data.py,sha256=jN2Y-96DRmj3hBCQT4ugCT6Q5rKv5y5TUi80G-95Zko,8610
+onnx_diagnostic/torch_models/hghub/hub_api.py,sha256=YYSX3pLsGCTwhMFSu-6ML4Bcy09EWmg1GgXSZ5eCQjA,14515
+onnx_diagnostic/torch_models/hghub/hub_data.py,sha256=8V_pAgACPLPsLRYUododg7MSL6str-T3tBEGY4OaeYQ,8724
 onnx_diagnostic/torch_models/hghub/hub_data_cached_configs.py,sha256=3yH1pQbCYNDmRMNUCwMFf5ELnAa35ubTKD2JRF5y9Ls,287515
-onnx_diagnostic/torch_models/hghub/model_inputs.py,sha256=-YX0guKGqj14cc8ZTco3QjCNXXBtf8inzwrsQdvQr6w,12559
-onnx_diagnostic/torch_models/hghub/model_specific.py,sha256=ZFajyL9MPZp7N6rveKB0IEAYeNKesbo0ItYoZIz90wc,1540
+onnx_diagnostic/torch_models/hghub/model_inputs.py,sha256=qg-_incL_nX9J1bit_nYV5diQN0Zqf7b10ZZfTikbjg,13701
+onnx_diagnostic/torch_models/hghub/model_specific.py,sha256=j50Nu7wddJMoqmD4QzMbNdFDUUgUmSBKRzPDH55TlUQ,2498
 onnx_diagnostic/torch_models/untrained/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_models/untrained/llm_phi2.py,sha256=JbGZmW41MPJcQgqaJc9R2G00nI79nI-lABN-ffA1lmY,4037
 onnx_diagnostic/torch_models/untrained/llm_tiny_llm.py,sha256=QXw_Bs2SzfeiQMf-tmtVl83SmVOL4-Um7Qy-f0E48QI,2507
 onnx_diagnostic/torch_onnx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnx_diagnostic/torch_onnx/runtime_info.py,sha256=1g9F_Jf9AAgYQU4stbsrFXwQl-30mWlQrFbQ7val8Ps,9268
 onnx_diagnostic/torch_onnx/sbs.py,sha256=fN799L_G1c2RKEuNcKt_MnQri5dwD4OzeCkBBFFoUBI,16865
-onnx_diagnostic-0.7.10.dist-info/licenses/LICENSE.txt,sha256=Vv6TXglX6Rc0d-f8aREhayhT-6PMQXEyOmI2NKlUCMc,1045
-onnx_diagnostic-0.7.10.dist-info/METADATA,sha256=5FswMlBjyOZNZ-pxujgExFBAiJ3rNd9DfPwWL0f9edw,7435
-onnx_diagnostic-0.7.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-onnx_diagnostic-0.7.10.dist-info/top_level.txt,sha256=KwNkXewmcobM3ZT1DJLVWH6ebJzA5qKg7cWqKfpGNT4,16
-onnx_diagnostic-0.7.10.dist-info/RECORD,,
+onnx_diagnostic-0.7.11.dist-info/licenses/LICENSE.txt,sha256=Vv6TXglX6Rc0d-f8aREhayhT-6PMQXEyOmI2NKlUCMc,1045
+onnx_diagnostic-0.7.11.dist-info/METADATA,sha256=vfCWZZUvnv_GKZxFRDvpKFrz5JU3LDmBH0WPK6uN__I,7435
+onnx_diagnostic-0.7.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+onnx_diagnostic-0.7.11.dist-info/top_level.txt,sha256=KwNkXewmcobM3ZT1DJLVWH6ebJzA5qKg7cWqKfpGNT4,16
+onnx_diagnostic-0.7.11.dist-info/RECORD,,