onnx-diagnostic 0.7.16__py3-none-any.whl → 0.8.1__py3-none-any.whl

onnx_diagnostic/helpers/log_helper.py

@@ -42,6 +42,8 @@ class CubeViewDef:
     :param name: name of the view, used mostly to debug
     :param plots: adds plot to the Excel sheet
     :param no_index: remove the index (but keeps the columns)
+    :param fix_aggregation_change: a column among the keys which changes aggregation value
+        for different dates
 
     Some examples of views. First example is an aggregated view
     for many metrics.
@@ -106,6 +108,7 @@ class CubeViewDef:
         name: Optional[str] = None,
         no_index: bool = False,
         plots: bool = False,
+        fix_aggregation_change: Optional[List["str"]] = None,
     ):
         self.key_index = key_index
         self.values = values
@@ -123,6 +126,7 @@ class CubeViewDef:
         self.name = name
         self.no_index = no_index
         self.plots = plots
+        self.fix_aggregation_change = fix_aggregation_change
 
     def __repr__(self) -> str:
         "usual"
@@ -750,6 +754,17 @@ class CubeLogs:
             f"values={sorted(self.values)}"
         )
 
+        if view_def.fix_aggregation_change and (
+            set(view_def.fix_aggregation_change) & set(self.keys_no_time)
+        ):
+            # before aggregation, let's fix some keys whose values changed over time
+            data_to_process = self._fix_aggregation_change(
+                self.data,
+                list(set(view_def.fix_aggregation_change) & set(self.keys_no_time)),
+            )
+        else:
+            data_to_process = self.data
+
         # aggregation
         if key_agg:
             final_stack = True
@@ -763,7 +778,7 @@ class CubeLogs:
                 print(f"[CubeLogs.view] aggregation of {set_key_agg}")
                 print(f"[CubeLogs.view] groupby {keys_no_agg}")
 
-            data_red = self.data[[*keys_no_agg, *values]]
+            data_red = data_to_process[[*keys_no_agg, *values]]
             assert set(key_index) <= set(data_red.columns), (
                 f"view_def.name={view_def.name!r}, "
                 f"nnable to find {set(key_index) - set(data_red.columns)}, "
@@ -792,7 +807,7 @@ class CubeLogs:
             key_index = self._filter_column(view_def.key_index, self.keys_time)
             if verbose:
                 print(f"[CubeLogs.view] no aggregation, index={key_index}")
-            data = self.data[[*self.keys_time, *values]]
+            data = data_to_process[[*self.keys_time, *values]]
             set_all_keys = set(self.keys_time)
             final_stack = False
 
@@ -829,7 +844,7 @@ class CubeLogs:
             key_columns = sorted(set_key_columns)
             unique = set()
 
-        _md = lambda s: {k: v for k, v in self.values_for_key.items() if k in s}  # noqa: E731
+        # md = lambda s: {k: v for k, v in self.values_for_key.items() if k in s}  # noqa: E731
         all_cols = set(key_columns) | set(key_index) | set(key_agg) | unique
         assert all_cols == set(self.keys_time), (
             f"view_def.name={view_def.name!r}, "
@@ -892,7 +907,7 @@ class CubeLogs:
             f"key={sorted(key_columns)}, key_agg={key_agg}, values={sorted(values)}, "
             f"columns={sorted(data.columns)}, ignored={view_def.ignore_columns}, "
             f"not unique={set(data.columns) - unique}"
-            f"\n--\n{not_unique.head()}"
+            f"\n--\n{not_unique.head(10)}"
         )
 
         # pivot
@@ -961,6 +976,70 @@ class CubeLogs:
             print(f"[CubeLogs.view] -- done view {view_def.name!r}")
         return (piv, view_def) if return_view_def else piv
 
+    def _fix_aggregation_change(
+        self,
+        data: pandas.DataFrame,
+        columns_to_fix: Union[str, List[str]],
+        overwrite_or_merge: bool = True,
+    ) -> pandas.DataFrame:
+        """
+        Fixes columns used to aggregate values because their meaning changed over time.
+
+        :param data: data to fix
+        :param columns_to_fix: list of columns to fix
+        :param overwrite_or_merge: if True, overwrite all values by the concatenation
+            of all existing values, if merge, merges existing values found
+            and grouped by the other keys
+        :return: fixed data
+        """
+        if not isinstance(columns_to_fix, str):
+            for c in columns_to_fix:
+                data = self._fix_aggregation_change(data, c)
+            return data
+        # Let's process one column.
+        keys = set(self.keys_time) - {columns_to_fix}
+        select = data[self.keys_time]
+        select_agg = select.groupby(list(keys)).count()
+        assert select_agg[columns_to_fix].max() <= 1, (
+            f"Column {columns_to_fix!r} has two distinct values at least for one date\n"
+            f"{select_agg[select_agg[columns_to_fix] > 1]}"
+        )
+
+        # unique value (to fill NaN)
+        unique = "-".join(sorted(set(data[columns_to_fix].dropna())))
+
+        keys = set(self.keys_no_time) - {columns_to_fix}
+        select = data[self.keys_no_time]
+        select_agg = select.groupby(list(keys), as_index=True).apply(
+            lambda x: "-".join(sorted(set(x[columns_to_fix].dropna()))), include_groups=False
+        )
+        select_agg = select_agg.to_frame(name=columns_to_fix)
+        res = pandas.merge(
+            data.drop([columns_to_fix], axis=1),
+            select_agg,
+            how="left",
+            left_on=list(keys),
+            right_index=True,
+        )
+        val = f"?{unique}?"
+        res[columns_to_fix] = res[columns_to_fix].fillna(val).replace("", val)
+        assert (
+            data.shape == res.shape
+            and sorted(data.columns) == sorted(res.columns)
+            and sorted(data.index) == sorted(res.index)
+        ), (
+            f"Shape should match, data.shape={data.shape}, res.shape={res.shape}, "
+            f"lost={set(data.columns) - set(res.columns)}, "
+            f"added={set(res.columns) - set(data.columns)}"
+        )
+        res = res[data.columns]
+        assert data.columns.equals(res.columns) and data.index.equals(res.index), (
+            f"Columns or index mismatch "
+            f"data.columns.equals(res.columns)={data.columns.equals(res.columns)}, "
+            f"data.index.equals(res.columns)={data.index.equals(res.columns)}, "
+        )
+        return res
+
     def _dropna(
         self,
         data: pandas.DataFrame,
@@ -1090,7 +1169,8 @@ class CubeLogs:
             assuming they should remain stale
         :param sbs: configurations to compare side-by-side, this adds two tabs,
             one gathering raw data about the two configurations, the other one
-            is aggregated by metrics
+            is aggregated by metrics, example:
+            ``=dict(CFA=dict(exporter="E1", opt="O"), CFB=dict(exporter="E2", opt="O"))``
         """
         if verbose:
             print(f"[CubeLogs.to_excel] create Excel file {output}, shape={self.shape}")
@@ -1532,6 +1612,7 @@ class CubeLogsPerformance(CubeLogs):
             "n_node_initializer_small",
             "n_node_layer_normalization",
             "n_node_layer_normalization23",
+            "n_node_random",
             "n_node_reshape",
             "n_node_rotary_embedding",
             "n_node_rotary_embedding23",
@@ -1723,6 +1804,16 @@ class CubeLogsPerformance(CubeLogs):
                     + gdf(df, "op_onnx__InstanceNormlization", 0)
                     + gdf(df, "op_onnx__GroupNormalization", 0),
                 ),
+                n_node_random=lambda df: gpreserve(
+                    df,
+                    "time_latency_eager",
+                    gdf(df, "op_onnx__RandomNormal", 0)
+                    + gdf(df, "op_onnx__RandomNormalLike", 0)
+                    + gdf(df, "op_onnx__RandomUniform", 0)
+                    + gdf(df, "op_onnx__RandomUniformLike", 0)
+                    + gdf(df, "op_onnx__Multinomial", 0)
+                    + gdf(df, "op_onnx__Bernoulli", 0),
+                ),
                 n_node_attention=lambda df: gpreserve(
                     df,
                     "time_latency_eager",
@@ -1886,6 +1977,7 @@ class CubeLogsPerformance(CubeLogs):
         * **cmd:** command lines
         * **raw-short:** raw data without all the unused columns
         """
+        fix_aggregation_change = ["model_speedup_input_set", "model_test_with"]
         fs = ["suite", "model_suite", "task", "model_name", "model_task"]
         index_cols = self._filter_column(fs, self.keys_time)
         assert index_cols, (
@@ -1984,6 +2076,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="agg-suite",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "agg-all": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2014,6 +2107,7 @@ class CubeLogsPerformance(CubeLogs):
                 name="agg-all",
                 order=order,
                 plots=True,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "disc": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2023,6 +2117,7 @@ class CubeLogsPerformance(CubeLogs):
                 f_highlight=f_disc,
                 name="disc",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "speedup": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2032,6 +2127,7 @@ class CubeLogsPerformance(CubeLogs):
                 f_highlight=f_speedup,
                 name="speedup",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "counts": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2048,6 +2144,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="peak-gpu",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "time": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2058,6 +2155,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="time",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "time_export": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2066,6 +2164,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="time_export",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "err": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2076,6 +2175,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="err",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "bucket-speedup": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2085,6 +2185,7 @@ class CubeLogsPerformance(CubeLogs):
                 name="bucket-speedup",
                 f_highlight=f_bucket,
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "onnx": lambda: CubeViewDef(
                 key_index=index_cols,
@@ -2103,6 +2204,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="onnx",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             ),
             "raw-short": lambda: CubeViewDef(
                 key_index=self.keys_time,
@@ -2111,6 +2213,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="raw-short",
                 no_index=True,
+                fix_aggregation_change=fix_aggregation_change,
             ),
         }
 
@@ -2123,6 +2226,7 @@ class CubeLogsPerformance(CubeLogs):
                 keep_columns_in_index=["suite"],
                 name="cmd",
                 order=order,
+                fix_aggregation_change=fix_aggregation_change,
             )
 
         assert name in implemented_views or name in {"cmd"}, (

onnx_diagnostic/helpers/memory_peak.py

@@ -47,6 +47,8 @@ class Monitor:
 
     @property
     def delta_avg(self):
+        if self.n_measures == 0:
+            return 0
         return self.average / self.n_measures - self.begin
 
     def __repr__(self):

onnx_diagnostic/helpers/mini_onnx_builder.py

@@ -52,7 +52,7 @@ def proto_from_array(
 
     tensor = TensorProto()
     tensor.dims.extend(arr_cpu.shape)
-    tensor.name = name
+    tensor.name = name or ""
     itype = dtype_to_tensor_dtype(arr_cpu.dtype)
     assert not hasattr(TensorProto, "INT4") or itype not in {
         TensorProto.INT4,

onnx_diagnostic/helpers/model_builder_helper.py

@@ -1,11 +1,13 @@
+import copy
 import importlib.util
 import os
+import re
 import requests
 import sys
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 from urllib.parse import urlparse
-from onnx import ModelProto, TensorProto
+from onnx import ModelProto, TensorProto, load as load_model
 
 CACHE_SUBDIR = "onnx-diagnostic"
 
@@ -337,3 +339,131 @@ def create_model_builder(
     # onnx_model.make_genai_config(hf_name, extra_kwargs, output_dir)
     # onnx_model.save_processing(hf_name, extra_kwargs, output_dir)
     return onnx_model
+
+
+def find_names_pattern(names: List[str]) -> str:
+    """
+    Finds a repeatable patterns in a list of names.
+    It tries to locate the figures.
+
+    .. runpython::
+        :showcode:
+
+        from onnx_diagnostic.helpers.model_builder_helper import find_names_pattern
+        pattern = find_names_pattern(["past_key_values_key_0", "past_key_values_key_1"])
+        print(pattern)
+    """
+    patterns = [re.sub(r"(\d+)", r"%d", t) for t in names]
+    unique = set(patterns)
+    assert (
+        len(unique) == 1
+    ), f"Unable to guess a pattern from {names} which led to the unique patterns {unique}"
+    return patterns[0]
+
+
+def make_genai_config(
+    config,
+    onnx_filename: str,
+) -> Dict:
+    """
+    Creates genai config file for a model.
+
+    :param config: configuration from transformers
+    :param onnx_filename: onnx configuration
+    :return: configuration
+    """
+    onx = load_model(onnx_filename, load_external_data=False)
+    config = copy.deepcopy(config)
+    defaults = {
+        "bos_token_id": None,
+        "do_sample": False,
+        "eos_token_id": None,
+        "pad_token_id": None,
+        "temperature": 1.0,
+        "top_k": 50,
+        "top_p": 1.0,
+    }
+    for key, default_val in defaults.items():
+        if not hasattr(config, key):
+            setattr(config, key, default_val)
+
+    bos_token_id = (
+        config.bos_token_id
+        if hasattr(config, "bos_token_id") and config.bos_token_id is not None
+        else 1
+    )
+    eos_token_id = config.eos_token_id
+    pad_token_id = (
+        config.pad_token_id
+        if hasattr(config, "pad_token_id") and config.pad_token_id is not None
+        else (
+            config.eos_token_id[0]
+            if isinstance(config.eos_token_id, list)
+            else config.eos_token_id
+        )
+    )
+    input_names = [i.name for i in onx.graph.input]
+    output_names = [i.name for i in onx.graph.output]
+    past_key_values = [s for s in input_names if s.startswith("past_key_value")]
+    first = [i for i in onx.graph.input if i.name == past_key_values[0]][0]  # noqa: RUF015
+    shape = tuple(d.dim_value or d.dim_param for d in first.type.tensor_type.shape.dim)
+    return {
+        "model": {
+            "bos_token_id": bos_token_id,
+            "context_length": config.max_position_embeddings,
+            "decoder": {
+                "session_options": {
+                    "log_id": "onnxruntime-genai",
+                    "provider_options": [],
+                },
+                "filename": os.path.split(onnx_filename)[-1],
+                "head_size": shape[-1],
+                "hidden_size": config.hidden_size,
+                "inputs": {
+                    "input_ids": input_names[0],
+                    "attention_mask": input_names[1],
+                    "past_key_names": find_names_pattern(input_names[2::2]),
+                    "past_value_names": find_names_pattern(input_names[3::2]),
+                },
+                "outputs": {
+                    "logits": output_names[0],
+                    "present_key_names": find_names_pattern(output_names[1::2]),
+                    "present_value_names": find_names_pattern(output_names[2::2]),
+                },
+                "num_attention_heads": config.num_attention_heads,
+                "num_hidden_layers": len(past_key_values) // 2,
+                "num_key_value_heads": shape[1],
+            },
+            "eos_token_id": eos_token_id,
+            "pad_token_id": pad_token_id,
+            "type": config.model_type,
+            # if "For" in self.model_type else len(self.model_type)].lower(),
+            "vocab_size": config.vocab_size,
+        },
+        "search": {
+            "diversity_penalty": (
+                config.diversity_penalty if hasattr(config, "diversity_penalty") else 0.0
+            ),
+            "do_sample": config.do_sample if hasattr(config, "do_sample") else False,
+            "early_stopping": True,
+            "length_penalty": (
+                config.length_penalty if hasattr(config, "length_penalty") else 1.0
+            ),
+            "max_length": config.max_position_embeddings,
+            "min_length": 0,
+            "no_repeat_ngram_size": (
+                config.no_repeat_ngram_size if hasattr(config, "no_repeat_ngram_size") else 0
+            ),
+            "num_beams": config.num_beams if hasattr(config, "num_beams") else 1,
+            "num_return_sequences": (
+                config.num_return_sequences if hasattr(config, "num_return_sequences") else 1
+            ),
+            "past_present_share_buffer": False,
+            "repetition_penalty": (
+                config.repetition_penalty if hasattr(config, "repetition_penalty") else 1.0
+            ),
+            "temperature": config.temperature if hasattr(config, "temperature") else 1.0,
+            "top_k": config.top_k if hasattr(config, "top_k") else 50,
+            "top_p": config.top_p if hasattr(config, "top_p") else 1.0,
+        },
+    }

onnx_diagnostic/helpers/onnx_helper.py

@@ -331,7 +331,7 @@ def onnx_dtype_name(itype: int, exc: bool = True) -> str:
         print(onnx_dtype_name(7))
     """
     for k in dir(TensorProto):
-        if "FLOAT" in k or "INT" in k or "TEXT" in k or "BOOL" in k:
+        if k.upper() == k and k != "EXTERNAL":
             v = getattr(TensorProto, k)
             if v == itype:
                 return k

onnx_diagnostic/helpers/ort_session.py

@@ -135,6 +135,10 @@ class _InferenceSession:
         self.sess = sess
         self.input_names = [i.name for i in sess.get_inputs()]
         self.output_names = [i.name for i in sess.get_outputs()]
+        self.input_shapes = [i.shape for i in sess.get_inputs()]
+        self.output_shapes = [i.shape for i in sess.get_outputs()]
+        self.input_types = [i.type for i in sess.get_inputs()]
+        self.output_types = [i.type for i in sess.get_outputs()]
         self.torch = torch
         self.nvtx = nvtx
         self.run_options = onnxruntime.RunOptions()