onnx-diagnostic 0.6.3__py3-none-any.whl → 0.7.1__py3-none-any.whl

onnx_diagnostic/__init__.py

@@ -3,5 +3,5 @@ Patches, Investigates onnx models.
 Functions, classes to dig into a model when this one is right, slow, wrong...
 """
 
-__version__ = "0.6.3"
+__version__ = "0.7.1"
 __author__ = "Xavier Dupré"

onnx_diagnostic/_command_lines_parser.py

@@ -5,19 +5,18 @@ import re
 import sys
 import textwrap
 import onnx
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional, Union
 from argparse import ArgumentParser, RawTextHelpFormatter, BooleanOptionalAction
-from textwrap import dedent
 
 
 def get_parser_lighten() -> ArgumentParser:
     parser = ArgumentParser(
         prog="lighten",
-        description=dedent(
+        description=textwrap.dedent(
+            """
+            Removes the weights from a heavy model, stores statistics to restore
+            random weights.
             """
-        Removes the weights from a heavy model, stores statistics to restore
-        random weights.
-        """
         ),
         epilog="This is mostly used to write unit tests without adding "
         "a big onnx file to the repository.",
@@ -70,11 +69,11 @@ def _cmd_lighten(argv: List[Any]):
 def get_parser_unlighten() -> ArgumentParser:
     parser = ArgumentParser(
         prog="unlighten",
-        description=dedent(
+        description=textwrap.dedent(
+            """
+            Restores random weights for a model reduces with command lighten,
+            the command expects to find a file nearby with extension '.stats'.
             """
-        Restores random weights for a model reduces with command lighten,
-        the command expects to find a file nearby with extension '.stats'.
-        """
         ),
         epilog="This is mostly used to write unit tests without adding "
         "a big onnx file to the repository.",
@@ -120,11 +119,7 @@ def _cmd_unlighten(argv: List[Any]):
 def get_parser_print() -> ArgumentParser:
     parser = ArgumentParser(
         prog="print",
-        description=dedent(
-            """
-        Prints the model on the standard output.
-        """
-        ),
+        description="Prints the model on the standard output.",
         epilog="To show a model.",
         formatter_class=RawTextHelpFormatter,
     )
@@ -171,11 +166,11 @@ def _cmd_print(argv: List[Any]):
 def get_parser_find() -> ArgumentParser:
     parser = ArgumentParser(
         prog="find",
-        description=dedent(
+        description=textwrap.dedent(
+            """
+            Look into a model and search for a set of names,
+            tells which node is consuming or producing it.
             """
-        Look into a model and search for a set of names,
-        tells which node is consuming or producing it.
-        """
         ),
         epilog="Enables Some quick validation.",
     )
@@ -191,8 +186,8 @@ def get_parser_find() -> ArgumentParser:
         "--names",
         type=str,
         required=False,
-        help="names to look at comma separated values, if 'SHADOW', "
-        "search for shadowing names",
+        help="Names to look at comma separated values, if 'SHADOW', "
+        "search for shadowing names.",
     )
     parser.add_argument(
         "-v",
@@ -206,7 +201,7 @@ def get_parser_find() -> ArgumentParser:
         "--v2",
         default=False,
         action=BooleanOptionalAction,
-        help="use enumerate_results instead of onnx_find",
+        help="Uses enumerate_results instead of onnx_find.",
     )
     return parser
 
@@ -235,12 +230,13 @@ def _cmd_find(argv: List[Any]):
 def get_parser_config() -> ArgumentParser:
     parser = ArgumentParser(
         prog="config",
-        description=dedent(
+        description=textwrap.dedent(
+            """
+            Prints out a configuration for a model id,
+            prints the associated task as well.
             """
-        Prints out a configuration for a model id,
-        prints the associated task as well.
-        """
         ),
+        formatter_class=RawTextHelpFormatter,
         epilog="",
     )
     parser.add_argument(
@@ -248,29 +244,29 @@ def get_parser_config() -> ArgumentParser:
         "--mid",
         type=str,
         required=True,
-        help="model id, usually <author>/<name>",
+        help="model id, usually `<author>/<name>`",
     )
     parser.add_argument(
         "-t",
         "--task",
         default=False,
         action=BooleanOptionalAction,
-        help="displays the task as well",
+        help="Displays the task as well.",
     )
     parser.add_argument(
         "-c",
         "--cached",
         default=True,
         action=BooleanOptionalAction,
-        help="uses cached configuration, only available for some of them, "
-        "mostly for unit test purposes",
+        help="Uses cached configuration, only available for some of them,\n"
+        "mostly for unit test purposes.",
     )
     parser.add_argument(
         "--mop",
         metavar="KEY=VALUE",
         nargs="*",
         help="Additional model options, use to change some parameters of the model, "
-        "example: --mop attn_implementation=eager",
+        "example:\n  --mop attn_implementation=sdpa or --mop attn_implementation=eager",
         action=_ParseDict,
     )
     return parser
@@ -291,6 +287,14 @@ def _cmd_config(argv: List[Any]):
         print(f"task: {task_from_id(args.mid)}")
 
 
+def _parse_json(value: str) -> Union[str, Dict[str, Any]]:
+    assert isinstance(value, str), f"value should be string but value={value!r}"
+    if value and value[0] == "{" and value[-1] == "}":
+        # a dictionary
+        return json.loads(value.replace("'", '"'))
+    return value
+
+
 class _ParseDict(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         d = getattr(namespace, self.dest) or {}
@@ -314,22 +318,40 @@ class _ParseDict(argparse.Action):
                     continue
                 except (TypeError, ValueError):
                     pass
-                d[key] = value
+                d[key] = _parse_json(value)
 
         setattr(namespace, self.dest, d)
 
 
 def get_parser_validate() -> ArgumentParser:
     parser = ArgumentParser(
-        prog="test",
-        description=dedent(
+        prog="validate",
+        description=textwrap.dedent(
+            """
+            Prints out dummy inputs for a particular task or a model id.
+            If both mid and task are empty, the command line displays the list
+            of supported tasks.
+            """
+        ),
+        epilog=textwrap.dedent(
+            """
+            If the model id is specified, one untrained version of it is instantiated.
+            Examples:
+
+            python -m onnx_diagnostic validate -m microsoft/Phi-4-mini-reasoning \\
+                --run -v 1 -o dump_test --no-quiet --repeat 2 --warmup 2 \\
+                --dtype float16 --device cuda --patch --export onnx-dynamo --opt ir
+
+            python -m onnx_diagnostic validate -m microsoft/Phi-4-mini-reasoning \\
+                --run -v 1 -o dump_test --no-quiet --repeat 2 --warmup 2 \\
+                --dtype float16 --device cuda --patch --export custom --opt default
+
+            python -m onnx_diagnostic validate -m microsoft/Phi-4-mini-reasoning \\
+                --run -v 1 -o dump_test --no-quiet --repeat 2 --warmup 2 \\
+                --dtype float16 --device cuda --export modelbuilder
             """
-        Prints out dummy inputs for a particular task or a model id.
-        If both mid and task are empty, the command line displays the list
-        of supported tasks.
-        """
         ),
-        epilog="If the model id is specified, one untrained version of it is instantiated.",
+        formatter_class=RawTextHelpFormatter,
     )
     parser.add_argument("-m", "--mid", type=str, help="model id, usually <author>/<name>")
     parser.add_argument("-t", "--task", default=None, help="force the task to use")
@@ -340,55 +362,67 @@ def get_parser_validate() -> ArgumentParser:
         "--run",
         default=False,
         action=BooleanOptionalAction,
-        help="runs the model to check it runs",
+        help="Runs the model to check it runs.",
     )
     parser.add_argument(
         "-q",
         "--quiet",
         default=False,
         action=BooleanOptionalAction,
-        help="catches exception, report them in the summary",
+        help="Catches exception, reports them in the summary.",
     )
     parser.add_argument(
         "--patch",
         default=True,
         action=BooleanOptionalAction,
-        help="applies patches before exporting",
+        help="Applies patches before exporting.",
     )
     parser.add_argument(
         "--rewrite",
         default=True,
         action=BooleanOptionalAction,
-        help="applies rewrite before exporting",
+        help="Applies rewrite before exporting.",
     )
     parser.add_argument(
         "--stop-if-static",
         default=0,
         type=int,
-        help="raises an exception if a dynamic dimension becomes static",
+        help="Raises an exception if a dynamic dimension becomes static.",
+    )
+    parser.add_argument(
+        "--same-as-trained",
+        default=False,
+        action=BooleanOptionalAction,
+        help="Validates a model identical to the trained model but not trained.",
     )
     parser.add_argument(
         "--trained",
         default=False,
         action=BooleanOptionalAction,
-        help="validate the trained model (requires downloading)",
+        help="Validates the trained model (requires downloading).",
+    )
+    parser.add_argument(
+        "--inputs2",
+        default=True,
+        action=BooleanOptionalAction,
+        help="Validates the model on a second set of inputs\n"
+        "to check the exported model supports dynamism.",
     )
     parser.add_argument(
         "--runtime",
         choices=["onnxruntime", "torch", "ref"],
         default="onnxruntime",
-        help="onnx runtime to use, onnxruntime by default",
+        help="onnx runtime to use, `onnxruntime` by default",
     )
     parser.add_argument(
         "-o",
         "--dump-folder",
-        help="if not empty, a folder is created to dumps statistics, "
-        "exported program, onnx...",
+        help="A folder is created to dumps statistics,\nexported program, onnx...",
     )
     parser.add_argument(
         "--drop",
-        help="drops the following inputs names, it should be a list "
-        "with comma separated values",
+        help="Drops the following inputs names, it should be a list\n"
+        "with comma separated values.",
     )
     parser.add_argument(
         "--opset",
@@ -398,24 +432,25 @@ def get_parser_validate() -> ArgumentParser:
     )
     parser.add_argument(
         "--subfolder",
-        help="subfolder where to find the model and the configuration",
+        help="Subfolder where to find the model and the configuration.",
     )
     parser.add_argument(
         "--ortfusiontype",
         required=False,
-        help="applies onnxruntime fusion, this parameter should contain the "
-        "model type or multiple values separated by `|`. `ALL` can be used "
-        "to run them all",
+        help="Applies onnxruntime fusion, this parameter should contain the\n"
+        "model type or multiple values separated by `|`. `ALL` can be used\n"
+        "to run them all.",
     )
     parser.add_argument("-v", "--verbose", default=0, type=int, help="verbosity")
-    parser.add_argument("--dtype", help="changes dtype if necessary")
-    parser.add_argument("--device", help="changes the device if necessary")
+    parser.add_argument("--dtype", help="Changes dtype if necessary.")
+    parser.add_argument("--device", help="Changes the device if necessary.")
     parser.add_argument(
         "--iop",
         metavar="KEY=VALUE",
         nargs="*",
-        help="Additional input options, use to change the default "
-        "inputs use to export, example: --iop cls_cache=SlidingWindowCache",
+        help="Additional input options, use to change the default"
+        "inputs use to export, example:\n  --iop cls_cache=SlidingWindowCache"
+        "\n  --iop cls_cache=StaticCache",
         action=_ParseDict,
     )
     parser.add_argument(
@@ -423,7 +458,8 @@ def get_parser_validate() -> ArgumentParser:
         metavar="KEY=VALUE",
         nargs="*",
         help="Additional model options, use to change some parameters of the model, "
-        "example: --mop attn_implementation=eager",
+        "example:\n  --mop attn_implementation=sdpa --mop attn_implementation=eager\n  "
+        "--mop \"rope_scaling={'rope_type': 'dynamic', 'factor': 10.0}\"",
         action=_ParseDict,
     )
     parser.add_argument(
@@ -440,7 +476,7 @@ def get_parser_validate() -> ArgumentParser:
 
 def _cmd_validate(argv: List[Any]):
     from .helpers import string_type
-    from .torch_models.test_helper import get_inputs_for_task, validate_model
+    from .torch_models.validate import get_inputs_for_task, validate_model
     from .tasks import supported_tasks
 
     parser = get_parser_validate()
@@ -474,7 +510,8 @@ def _cmd_validate(argv: List[Any]):
             do_run=args.run,
             verbose=args.verbose,
             quiet=args.quiet,
-            trained=args.trained,
+            same_as_pretrained=args.same_as_trained,
+            use_pretrained=args.trained,
             dtype=args.dtype,
             device=args.device,
             patch=args.patch,
@@ -492,6 +529,7 @@ def _cmd_validate(argv: List[Any]):
             runtime=args.runtime,
             repeat=args.repeat,
             warmup=args.warmup,
+            inputs2=args.inputs2,
         )
         print("")
         print("-- summary --")
@@ -502,11 +540,7 @@ def _cmd_validate(argv: List[Any]):
 def get_parser_stats() -> ArgumentParser:
     parser = ArgumentParser(
         prog="stats",
-        description=dedent(
-            """
-        Prints out statistics on an ONNX model.
-        """
-        ),
+        description="Prints out statistics on an ONNX model.",
         epilog="",
     )
     parser.add_argument(
@@ -553,8 +587,8 @@ def get_parser_stats() -> ArgumentParser:
         required=False,
         default="",
         type=str,
-        help="keeps only tensors whose name verifies "
-        "this regular expression, empty = no filter",
+        help="Keeps only tensors whose name verifies "
+        "this regular expression, empty = no filter.",
     )
     return parser
 
@@ -599,6 +633,161 @@ def _cmd_stats(argv: List[Any]):
         print("done.")
 
 
+def get_parser_agg() -> ArgumentParser:
+    parser = ArgumentParser(
+        prog="agg",
+        description=textwrap.dedent(
+            """
+            Aggregates statistics coming from benchmarks.
+            Every run is a row. Every row is indexed by some keys,
+            and produces values. Every row has a date.
+            """
+        ),
+        epilog=textwrap.dedent(
+            """
+            examples:\n
+
+                python -m onnx_diagnostic agg test_agg.xlsx raw/*.zip -v 1
+            """
+        ),
+        formatter_class=RawTextHelpFormatter,
+    )
+    parser.add_argument("output", help="output excel file")
+    parser.add_argument(
+        "inputs",
+        nargs="+",
+        help="input csv or zip files, at least 1, it can be a name, or search path",
+    )
+    parser.add_argument(
+        "--filter", default="rawdata_.*.csv", help="filter for input files inside zip files"
+    )
+    parser.add_argument(
+        "--recent",
+        default=True,
+        action=BooleanOptionalAction,
+        help="Keeps only the most recent experiment for the same of keys.",
+    )
+    parser.add_argument(
+        "--keep-last-date",
+        default=False,
+        action=BooleanOptionalAction,
+        help="Rewrite all dates to the last one to simplifies the analysis, "
+        "this assume changing the date does not add ambiguity, if any, option "
+        "--recent should be added.",
+    )
+    parser.add_argument(
+        "--raw",
+        default=True,
+        action=BooleanOptionalAction,
+        help="Keeps the raw data in a sheet.",
+    )
+    parser.add_argument("-t", "--time", default="DATE", help="Date or time column")
+    parser.add_argument(
+        "-k",
+        "--keys",
+        default="^version_.*,^model_.*,device,opt_patterns,suite,memory_peak,"
+        "machine,exporter,dynamic,rtopt,dtype,device,architecture",
+        help="List of columns to consider as keys, "
+        "multiple values are separated by `,`\n"
+        "regular expressions are allowed",
+    )
+    parser.add_argument(
+        "--drop-keys",
+        default="",
+        help="Drops keys from the given list. Something it is faster "
+        "to remove one than to select all the remaining ones.",
+    )
+    parser.add_argument(
+        "-w",
+        "--values",
+        default="^time_.*,^disc.*,^ERR_.*,CMD,^ITER.*,^onnx_.*,^op_onnx_.*,^peak_gpu_.*",
+        help="List of columns to consider as values, "
+        "multiple values are separated by `,`\n"
+        "regular expressions are allowed",
+    )
+    parser.add_argument(
+        "-i", "--ignored", default="^version_.*", help="List of columns to ignore"
+    )
+    parser.add_argument(
+        "-f",
+        "--formula",
+        default="speedup,bucket[speedup],ERR1,n_models,n_model_eager,"
+        "n_model_running,n_model_acc01,n_model_acc001,n_model_dynamic,"
+        "n_model_pass,n_model_faster,"
+        "n_model_faster2x,n_model_faster3x,n_model_faster4x,n_node_attention,"
+        "peak_gpu_torch,peak_gpu_nvidia,n_node_control_flow,"
+        "n_node_constant,n_node_shape,n_node_expand,"
+        "n_node_function,n_node_initializer,n_node_scatter,"
+        "time_export_unbiased",
+        help="Columns to compute after the aggregation was done.",
+    )
+    parser.add_argument(
+        "--views",
+        default="agg-suite,agg-all,disc,speedup,time,time_export,err,cmd,"
+        "bucket-speedup,raw-short,counts,peak-gpu",
+        help="Views to add to the output files.",
+    )
+    parser.add_argument(
+        "--csv",
+        default="raw-short",
+        help="Views to dump as csv files.",
+    )
+    parser.add_argument("-v", "--verbose", type=int, default=0, help="verbosity")
+    return parser
+
+
+def _cmd_agg(argv: List[Any]):
+    from .helpers.log_helper import CubeLogsPerformance, open_dataframe, enumerate_csv_files
+
+    parser = get_parser_agg()
+    args = parser.parse_args(argv[1:])
+    reg = re.compile(args.filter)
+
+    csv = list(
+        enumerate_csv_files(
+            args.inputs, verbose=args.verbose, filtering=lambda name: bool(reg.search(name))
+        )
+    )
+    assert csv, f"No csv files in {args.inputs}, csv={csv}"
+    if args.verbose:
+        from tqdm import tqdm
+
+        loop = tqdm(csv)
+    else:
+        loop = csv
+    dfs = []
+    for c in loop:
+        df = open_dataframe(c)
+        assert (
+            args.time in df.columns
+        ), f"Missing time column {args.time!r} in {c!r}\n{df.head()}\n{sorted(df.columns)}"
+        dfs.append(df)
+
+    drop_keys = set(args.drop_keys.split(","))
+    cube = CubeLogsPerformance(
+        dfs,
+        time=args.time,
+        keys=[a for a in args.keys.split(",") if a and a not in drop_keys],
+        values=[a for a in args.values.split(",") if a],
+        ignored=[a for a in args.ignored.split(",") if a],
+        recent=args.recent,
+        formulas={k: k for k in args.formula.split(",")},
+        keep_last_date=args.keep_last_date,
+    )
+    cube.load(verbose=max(args.verbose - 1, 0))
+    if args.verbose:
+        print(f"Dumps final file into {args.output!r}")
+    cube.to_excel(
+        args.output,
+        {k: k for k in args.views.split(",")},
+        verbose=args.verbose,
+        csv=args.csv.split(","),
+        raw=args.raw,
+    )
+    if args.verbose:
+        print(f"Wrote {args.output!r}")
+
+
 def get_main_parser() -> ArgumentParser:
     parser = ArgumentParser(
         prog="onnx_diagnostic",
@@ -606,22 +795,32 @@ def get_main_parser() -> ArgumentParser:
         formatter_class=RawTextHelpFormatter,
         epilog=textwrap.dedent(
             """
-        Type 'python -m onnx_diagnostic <cmd> --help'
-        to get help for a specific command.
-
-        config     - prints a configuration for a model id
-        find       - find node consuming or producing a result
-        lighten    - makes an onnx model lighter by removing the weights,
-        unlighten  - restores an onnx model produces by the previous experiment
-        print      - prints the model on standard output
-        validate   - validate a model
-        stats      - produces statistics on a model
-        """
+            Type 'python -m onnx_diagnostic <cmd> --help'
+            to get help for a specific command.
+
+            agg        - aggregates statistics from multiple files
+            config     - prints a configuration for a model id
+            find       - find node consuming or producing a result
+            lighten    - makes an onnx model lighter by removing the weights,
+            print      - prints the model on standard output
+            stats      - produces statistics on a model
+            unlighten  - restores an onnx model produces by the previous experiment
+            validate   - validate a model
+            """
         ),
     )
     parser.add_argument(
         "cmd",
-        choices=["config", "find", "lighten", "print", "stats", "unlighten", "validate"],
+        choices=[
+            "agg",
+            "config",
+            "find",
+            "lighten",
+            "print",
+            "stats",
+            "unlighten",
+            "validate",
+        ],
         help="Selects a command.",
     )
     return parser
@@ -636,6 +835,7 @@ def main(argv: Optional[List[Any]] = None):
         config=_cmd_config,
         validate=_cmd_validate,
         stats=_cmd_stats,
+        agg=_cmd_agg,
     )
 
     if argv is None:
@@ -657,6 +857,7 @@ def main(argv: Optional[List[Any]] = None):
                 config=get_parser_config,
                 validate=get_parser_validate,
                 stats=get_parser_stats,
+                agg=get_parser_agg,
             )
             cmd = argv[0]
             if cmd not in parsers:

onnx_diagnostic/doc.py

@@ -2,6 +2,28 @@ from typing import Optional
 import numpy as np
 
 
+def get_latest_pypi_version(package_name="onnx-diagnostic") -> str:
+    """Returns the latest published version."""
+
+    import requests
+
+    url = f"https://pypi.org/pypi/{package_name}/json"
+    response = requests.get(url)
+
+    assert response.status_code == 200, f"Unable to retrieve the version response={response}"
+    data = response.json()
+    version = data["info"]["version"]
+    return version
+
+
+def update_version_package(version: str, package_name="onnx-diagnostic") -> str:
+    "Adds dev if the major version is different from the latest published one."
+    released = get_latest_pypi_version(package_name)
+    shorten_r = ".".join(released.split(".")[:2])
+    shorten_v = ".".join(version.split(".")[:2])
+    return version if shorten_r == shorten_v else f"{shorten_v}.dev"
+
+
 def reset_torch_transformers(gallery_conf, fname):
     "Resets torch dynamo for :epkg:`sphinx-gallery`."
     import matplotlib.pyplot as plt