lemonade-sdk 8.0.2__tar.gz → 8.0.4__tar.gz

{lemonade_sdk-8.0.2/src/lemonade_sdk.egg-info → lemonade_sdk-8.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lemonade-sdk
-Version: 8.0.2
+Version: 8.0.4
 Summary: Lemonade SDK: Your LLM Aide for Validation and Deployment
 Author-email: lemonade@amd.com
 Requires-Python: >=3.10, <3.12
@@ -26,45 +26,49 @@ Requires-Dist: openai>=1.81.0
 Requires-Dist: transformers<=4.51.3
 Requires-Dist: jinja2
 Requires-Dist: tabulate
-Requires-Dist: huggingface-hub==0.30.2
+Requires-Dist: sentencepiece
+Requires-Dist: huggingface-hub==0.33.0
+Provides-Extra: oga-hybrid
+Requires-Dist: onnx==1.16.1; extra == "oga-hybrid"
+Requires-Dist: numpy==1.26.4; extra == "oga-hybrid"
+Requires-Dist: protobuf>=6.30.1; extra == "oga-hybrid"
+Provides-Extra: oga-cpu
+Requires-Dist: onnxruntime-genai==0.8.2; extra == "oga-cpu"
+Requires-Dist: onnxruntime>=1.22.0; extra == "oga-cpu"
+Provides-Extra: dev
+Requires-Dist: torch>=2.6.0; extra == "dev"
+Requires-Dist: accelerate; extra == "dev"
+Requires-Dist: datasets; extra == "dev"
+Requires-Dist: pandas>=1.5.3; extra == "dev"
+Requires-Dist: matplotlib; extra == "dev"
+Requires-Dist: human-eval-windows==1.0.4; extra == "dev"
+Requires-Dist: lm-eval[api]; extra == "dev"
 Provides-Extra: oga-hybrid-minimal
-Requires-Dist: onnx==1.16.1; extra == "oga-hybrid-minimal"
-Requires-Dist: numpy==1.26.4; extra == "oga-hybrid-minimal"
-Requires-Dist: protobuf>=6.30.1; extra == "oga-hybrid-minimal"
+Requires-Dist: lemonade-sdk[oga-hybrid]; extra == "oga-hybrid-minimal"
 Provides-Extra: oga-cpu-minimal
-Requires-Dist: onnxruntime-genai==0.6.0; extra == "oga-cpu-minimal"
-Requires-Dist: onnxruntime<1.22.0,>=1.10.1; extra == "oga-cpu-minimal"
+Requires-Dist: lemonade-sdk[oga-cpu]; extra == "oga-cpu-minimal"
 Provides-Extra: llm
-Requires-Dist: torch>=2.6.0; extra == "llm"
-Requires-Dist: accelerate; extra == "llm"
-Requires-Dist: sentencepiece; extra == "llm"
-Requires-Dist: datasets; extra == "llm"
-Requires-Dist: pandas>=1.5.3; extra == "llm"
-Requires-Dist: matplotlib; extra == "llm"
-Requires-Dist: human-eval-windows==1.0.4; extra == "llm"
-Requires-Dist: lm-eval[api]; extra == "llm"
+Requires-Dist: lemonade-sdk[dev]; extra == "llm"
 Provides-Extra: llm-oga-cpu
-Requires-Dist: lemonade-sdk[oga-cpu-minimal]; extra == "llm-oga-cpu"
-Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-cpu"
+Requires-Dist: lemonade-sdk[dev,oga-cpu]; extra == "llm-oga-cpu"
 Provides-Extra: llm-oga-igpu
 Requires-Dist: onnxruntime-genai-directml==0.6.0; extra == "llm-oga-igpu"
 Requires-Dist: onnxruntime-directml<1.22.0,>=1.19.0; extra == "llm-oga-igpu"
 Requires-Dist: transformers<4.45.0; extra == "llm-oga-igpu"
-Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-igpu"
+Requires-Dist: lemonade-sdk[dev]; extra == "llm-oga-igpu"
 Provides-Extra: llm-oga-cuda
-Requires-Dist: onnxruntime-genai-cuda==0.6.0; extra == "llm-oga-cuda"
-Requires-Dist: onnxruntime-gpu<1.22.0,>=1.19.1; extra == "llm-oga-cuda"
-Requires-Dist: transformers<4.45.0; extra == "llm-oga-cuda"
-Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-cuda"
+Requires-Dist: onnxruntime-genai-cuda==0.8.2; extra == "llm-oga-cuda"
+Requires-Dist: onnxruntime-gpu>=1.22.0; extra == "llm-oga-cuda"
+Requires-Dist: transformers<=4.51.3; extra == "llm-oga-cuda"
+Requires-Dist: lemonade-sdk[dev]; extra == "llm-oga-cuda"
 Provides-Extra: llm-oga-npu
 Requires-Dist: onnx==1.16.0; extra == "llm-oga-npu"
 Requires-Dist: onnxruntime==1.18.0; extra == "llm-oga-npu"
 Requires-Dist: numpy==1.26.4; extra == "llm-oga-npu"
 Requires-Dist: protobuf>=6.30.1; extra == "llm-oga-npu"
-Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-npu"
+Requires-Dist: lemonade-sdk[dev]; extra == "llm-oga-npu"
 Provides-Extra: llm-oga-hybrid
-Requires-Dist: lemonade-sdk[oga-hybrid-minimal]; extra == "llm-oga-hybrid"
-Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-hybrid"
+Requires-Dist: lemonade-sdk[dev,oga-hybrid]; extra == "llm-oga-hybrid"
 Provides-Extra: llm-oga-unified
 Requires-Dist: lemonade-sdk[llm-oga-hybrid]; extra == "llm-oga-unified"
 Dynamic: author-email
@@ -78,7 +82,7 @@ Dynamic: summary
 
 [![Lemonade tests](https://github.com/lemonade-sdk/lemonade/actions/workflows/test_lemonade.yml/badge.svg)](https://github.com/lemonade-sdk/lemonade/tree/main/test "Check out our tests")
 [![OS - Windows | Linux](https://img.shields.io/badge/OS-windows%20%7C%20linux-blue)](docs/README.md#installation "Check out our instructions")
-[![Made with Python](https://img.shields.io/badge/Python-3.8,3.10-blue?logo=python&logoColor=white)](docs/README.md#installation "Check out our instructions")
+[![Made with Python](https://img.shields.io/badge/Python-3.10-blue?logo=python&logoColor=white)](docs/README.md#installation "Check out our instructions")
 
 ## 🍋 Lemonade SDK: Quickly serve, benchmark and deploy LLMs
 
@@ -93,8 +97,8 @@ The [Lemonade SDK](./docs/README.md) makes it easy to run Large Language Models
 The [Lemonade SDK](./docs/README.md) is comprised of the following:
 
 - 🌐 **[Lemonade Server](https://lemonade-server.ai/docs)**: A local LLM server for running ONNX and GGUF models using the OpenAI API standard. Install and enable your applications with NPU and GPU acceleration in minutes.
-- 🐍 **Lemonade API**: High-level Python API to directly integrate Lemonade LLMs into Python applications.
-- 🖥️ **Lemonade CLI**: The `lemonade` CLI lets you mix-and-match LLMs (ONNX, GGUF, SafeTensors) with measurement tools to characterize your models on your hardware. The available tools are:
+- 🐍 **[Lemonade API](./docs/lemonade_api.md)**: High-level Python API to directly integrate Lemonade LLMs into Python applications.
+- 🖥️ **[Lemonade CLI](./docs/dev_cli/README.md)**: The `lemonade` CLI lets you mix-and-match LLMs (ONNX, GGUF, SafeTensors) with measurement tools to characterize your models on your hardware. The available tools are:
   - Prompting with templates.
   - Measuring accuracy with a variety of tests.
   - Benchmarking to get the time-to-first-token and tokens per second.
@@ -149,14 +153,7 @@ Maximum LLM performance requires the right hardware accelerator with the right i
   </tbody>
 </table>
 
-
-
-#### Inference Engines Overview
-| Engine | Description |
-| :--- | :--- |
-| **OnnxRuntime GenAI (OGA)** | Microsoft engine that runs `.onnx` models and enables hardware vendors to provide their own execution providers (EPs) to support specialized hardware, such as neural processing units (NPUs). |
-| **llamacpp** | Community-driven engine with strong GPU acceleration, support for thousands of `.gguf` models, and advanced features such as vision-language models (VLMs) and mixture-of-experts (MoEs). |
-| **Hugging Face (HF)** | Hugging Face's `transformers` library can run the original `.safetensors` trained weights for models on Meta's PyTorch engine, which provides a source of truth for accuracy measurement. |
+To learn more about the supported hardware and software, visit the documentation [here](./docs/README.md#software-and-hardware-overview).
 
 ## Integrate Lemonade Server with Your Application
 

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/README.md

@@ -1,6 +1,6 @@
 [![Lemonade tests](https://github.com/lemonade-sdk/lemonade/actions/workflows/test_lemonade.yml/badge.svg)](https://github.com/lemonade-sdk/lemonade/tree/main/test "Check out our tests")
 [![OS - Windows | Linux](https://img.shields.io/badge/OS-windows%20%7C%20linux-blue)](docs/README.md#installation "Check out our instructions")
-[![Made with Python](https://img.shields.io/badge/Python-3.8,3.10-blue?logo=python&logoColor=white)](docs/README.md#installation "Check out our instructions")
+[![Made with Python](https://img.shields.io/badge/Python-3.10-blue?logo=python&logoColor=white)](docs/README.md#installation "Check out our instructions")
 
 ## 🍋 Lemonade SDK: Quickly serve, benchmark and deploy LLMs
 
@@ -15,8 +15,8 @@ The [Lemonade SDK](./docs/README.md) makes it easy to run Large Language Models
 The [Lemonade SDK](./docs/README.md) is comprised of the following:
 
 - 🌐 **[Lemonade Server](https://lemonade-server.ai/docs)**: A local LLM server for running ONNX and GGUF models using the OpenAI API standard. Install and enable your applications with NPU and GPU acceleration in minutes.
-- 🐍 **Lemonade API**: High-level Python API to directly integrate Lemonade LLMs into Python applications.
-- 🖥️ **Lemonade CLI**: The `lemonade` CLI lets you mix-and-match LLMs (ONNX, GGUF, SafeTensors) with measurement tools to characterize your models on your hardware. The available tools are:
+- 🐍 **[Lemonade API](./docs/lemonade_api.md)**: High-level Python API to directly integrate Lemonade LLMs into Python applications.
+- 🖥️ **[Lemonade CLI](./docs/dev_cli/README.md)**: The `lemonade` CLI lets you mix-and-match LLMs (ONNX, GGUF, SafeTensors) with measurement tools to characterize your models on your hardware. The available tools are:
   - Prompting with templates.
   - Measuring accuracy with a variety of tests.
   - Benchmarking to get the time-to-first-token and tokens per second.
@@ -71,14 +71,7 @@ Maximum LLM performance requires the right hardware accelerator with the right i
   </tbody>
 </table>
 
-
-
-#### Inference Engines Overview
-| Engine | Description |
-| :--- | :--- |
-| **OnnxRuntime GenAI (OGA)** | Microsoft engine that runs `.onnx` models and enables hardware vendors to provide their own execution providers (EPs) to support specialized hardware, such as neural processing units (NPUs). |
-| **llamacpp** | Community-driven engine with strong GPU acceleration, support for thousands of `.gguf` models, and advanced features such as vision-language models (VLMs) and mixture-of-experts (MoEs). |
-| **Hugging Face (HF)** | Hugging Face's `transformers` library can run the original `.safetensors` trained weights for models on Meta's PyTorch engine, which provides a source of truth for accuracy measurement. |
+To learn more about the supported hardware and software, visit the documentation [here](./docs/README.md#software-and-hardware-overview).
 
 ## Integrate Lemonade Server with Your Application
 

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/setup.py

@@ -48,29 +48,30 @@ setup(
         "transformers<=4.51.3",
         "jinja2",
         "tabulate",
-        # huggingface-hub==0.31.0 introduces a new transfer protocol that was causing us issues
-        "huggingface-hub==0.30.2",
+        "sentencepiece",
+        "huggingface-hub==0.33.0",
     ],
     extras_require={
-        # The -minimal extras are meant to deploy specific backends into end-user
+        # The non-dev extras are meant to deploy specific backends into end-user
         # applications, without including developer-focused tools
-        "oga-hybrid-minimal": [
+        "oga-hybrid": [
             # Note: `lemonade-install --ryzenai hybrid` is necessary
             # to complete installation
             "onnx==1.16.1",
             "numpy==1.26.4",
             "protobuf>=6.30.1",
         ],
-        "oga-cpu-minimal": [
-            "onnxruntime-genai==0.6.0",
-            "onnxruntime >=1.10.1,<1.22.0",
+        "oga-cpu": [
+            "onnxruntime-genai==0.8.2",
+            "onnxruntime >=1.22.0",
         ],
-        "llm": [
+        # Developer-focused tools for benchmarking, accuracy testing, and
+        # model preparation (ONNX export, quantization, device-specifc optimization, etc.)
+        "dev": [
             # Minimal dependencies for developers to use all features of
             # Lemonade SDK, including building and optimizing models
             "torch>=2.6.0",
             "accelerate",
-            "sentencepiece",
             "datasets",
             "pandas>=1.5.3",
             "matplotlib",
@@ -79,36 +80,35 @@ setup(
             "human-eval-windows==1.0.4",
             "lm-eval[api]",
         ],
-        "llm-oga-cpu": [
-            "lemonade-sdk[oga-cpu-minimal]",
-            "lemonade-sdk[llm]",
-        ],
+        # Keep backwards compatibility for old extras names
+        "oga-hybrid-minimal": ["lemonade-sdk[oga-hybrid]"],
+        "oga-cpu-minimal": ["lemonade-sdk[oga-cpu]"],
+        "llm": ["lemonade-sdk[dev]"],
+        "llm-oga-cpu": ["lemonade-sdk[dev,oga-cpu]"],
+        # The following extras are deprecated and/or not commonly used
         "llm-oga-igpu": [
             "onnxruntime-genai-directml==0.6.0",
             "onnxruntime-directml>=1.19.0,<1.22.0",
             "transformers<4.45.0",
-            "lemonade-sdk[llm]",
+            "lemonade-sdk[dev]",
         ],
         "llm-oga-cuda": [
-            "onnxruntime-genai-cuda==0.6.0",
-            "onnxruntime-gpu >=1.19.1,<1.22.0",
-            "transformers<4.45.0",
-            "lemonade-sdk[llm]",
+            "onnxruntime-genai-cuda==0.8.2",
+            "onnxruntime-gpu >=1.22.0",
+            "transformers<=4.51.3",
+            "lemonade-sdk[dev]",
         ],
         "llm-oga-npu": [
             "onnx==1.16.0",
+            # NPU requires specific onnxruntime version for Ryzen AI compatibility
+            # This may conflict with other OGA extras that require >=1.22.0
             "onnxruntime==1.18.0",
             "numpy==1.26.4",
             "protobuf>=6.30.1",
-            "lemonade-sdk[llm]",
-        ],
-        "llm-oga-hybrid": [
-            "lemonade-sdk[oga-hybrid-minimal]",
-            "lemonade-sdk[llm]",
-        ],
-        "llm-oga-unified": [
-            "lemonade-sdk[llm-oga-hybrid]",
+            "lemonade-sdk[dev]",
         ],
+        "llm-oga-hybrid": ["lemonade-sdk[dev,oga-hybrid]"],
+        "llm-oga-unified": ["lemonade-sdk[llm-oga-hybrid]"],
     },
     classifiers=[],
     entry_points={

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/cli.py

@@ -90,9 +90,9 @@ https://github.com/lemonade-sdk/lemonade/blob/main/docs/README.md""",
     )
 
     profiler_instances = [
-        profiler(global_args[profiler.unique_name])
+        profiler(global_args[profiler.unique_name.replace("-", "_")])
         for profiler in profilers
-        if global_args.get(profiler.unique_name, None) is not None
+        if global_args.get(profiler.unique_name.replace("-", "_"), None) is not None
     ]
 
     if len(evaluation_tools) > 0:

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/profilers/profiler.py

@@ -48,7 +48,10 @@ class Profiler(abc.ABC):
         This method is called so that the profiler can create its output files.
         The state is passed so that build info can be gathered and stats can be written.
         The timestamp can be used for filename in current working directory.
-        The start times contain a list of tools and start times.
+        The start times parameter is a dict with the keys being the tools names and
+        the values being the time the tool started.  There is an initial "warmup" key
+        that has a start time before the first tool and a "cool down" key that contains the
+        time when the last tool ended.
         """
 
 

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/humaneval.py

@@ -24,7 +24,7 @@ class AccuracyHumaneval(Tool):
     - pass@10: Percentage of problems solved within 10 generation attempts
     - pass@100: Percentage of problems solved within 100 generation attempts
 
-    See docs/lemonade/humaneval_accuracy.md for more details
+    See docs/dev_cli/humaneval_accuracy.md for more details
     """
 
     unique_name = "accuracy-humaneval"

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/mmlu.py

@@ -27,7 +27,7 @@ def min_handle_none(*args: int):
 
 class AccuracyMMLU(Tool):
     """
-    See docs/lemonade/mmlu_accuracy.md for more details
+    See docs/dev_cli/mmlu_accuracy.md for more details
     """
 
     unique_name = "accuracy-mmlu"

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/oga/load.py

@@ -1,12 +1,6 @@
 # onnxruntime_genai is not lint-friendly yet and PyLint can't
 # find any of the class methods
 # pylint: disable=no-member
-#
-# Model builder constraints:
-#   11/10/24 Need transformers <4.45.0 OR onnxruntime-genai 0.5.0 (which must be built from source)
-#   (transformers v4.45 changes the format of the tokenizer.json file which will be supported in
-#   onnxruntime-genai 0.5)
-#
 
 import argparse
 import os
@@ -51,8 +45,8 @@ def import_error_heler(e: Exception):
     """
     raise ImportError(
         f"{e}\n Please install lemonade-sdk with "
-        "one of the llm-oga extras, for example:\n"
-        "pip install lemonade-sdk[llm-oga-cpu]\n"
+        "one of the oga extras, for example:\n"
+        "pip install lemonade-sdk[dev,oga-cpu]\n"
         "See https://lemonade_server.ai/install_options.html for details"
     )
 
@@ -64,7 +58,7 @@ class OgaLoad(FirstTool):
     Input: path to a checkpoint.
         Supported choices for cpu and igpu from HF model repository:
             LLM models on Huggingface supported by model_builder.  See documentation
-            (https://github.com/lemonade-sdk/lemonade/blob/main/docs/ort_genai_igpu.md)
+            (https://github.com/lemonade-sdk/lemonade/blob/main/docs/dev_cli/ort_genai_igpu.md)
             for supported models.
         Supported choices for npu from HF model repository:
             Models on Hugging Face that follow the "amd/**-onnx-ryzen-strix" pattern

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/perplexity.py

@@ -17,7 +17,7 @@ class AccuracyPerplexity(Tool):
 
     Output state produced: None
 
-    See docs/lemonade/perplexity.md for more details.
+    See docs/dev_cli/perplexity.md for more details.
     """
 
     unique_name = "accuracy-perplexity"
@@ -63,7 +63,7 @@ class AccuracyPerplexity(Tool):
             # try-except will allow a few more LLMs to work
             max_length = 2048
         # Set stride to half of the maximum input length for overlapping window processing
-        # Refer to docs/perplexity.md for more information on sliding window
+        # Refer to docs/dev_cli/perplexity.md for more information on sliding window
         stride = max_length // 2
         # Determine the total sequence length of the tokenized input
         seq_len = encodings.input_ids.size(1)

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/prompt.py

@@ -176,12 +176,21 @@ class LLMPrompt(Tool):
 
         input_ids = tokenizer(prompt, return_tensors="pt").input_ids
         if isinstance(input_ids, (list, str)):
-            # OGA models return a list of tokens
+            # OGA models return a list of tokens (older versions)
             # Our llama.cpp adapter returns a string
             len_tokens_in = len(input_ids)
-        else:
+        elif hasattr(input_ids, "shape"):
             # HF models return a 2-D tensor
-            len_tokens_in = input_ids.shape[1]
+            # OGA models with newer versions may return numpy arrays
+            if len(input_ids.shape) == 1:
+                # 1-D array from newer OGA versions
+                len_tokens_in = len(input_ids)
+            else:
+                # 2-D tensor from HF models
+                len_tokens_in = input_ids.shape[1]
+        else:
+            # Fallback: try to get length directly
+            len_tokens_in = len(input_ids)
 
         len_tokens_out = []
         response_texts = []
@@ -202,9 +211,15 @@ class LLMPrompt(Tool):
                 random_seed += 1
 
             # Flatten the input and response
-            input_ids_array = (
-                input_ids if isinstance(input_ids, (list, str)) else input_ids[0]
-            )
+            if isinstance(input_ids, (list, str)):
+                input_ids_array = input_ids
+            elif hasattr(input_ids, "shape") and len(input_ids.shape) == 1:
+                # 1-D array from newer OGA versions - already flat
+                input_ids_array = input_ids
+            else:
+                # 2-D tensor from HF models - take first row
+                input_ids_array = input_ids[0]
+
             response_array = response if isinstance(response, str) else response[0]
 
             # Separate the prompt from the response

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/quark/quark_load.py

@@ -18,7 +18,7 @@ class QuarkLoad(Tool):
     Output:
         - state of the loaded model
 
-    See docs/quark.md for more details.
+    See docs/dev_cli/quark.md for more details.
     """
 
     unique_name = "quark-load"

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/quark/quark_quantize.py

@@ -25,7 +25,7 @@ class QuarkQuantize(Tool):
     Output:
         - Modifies `state` with quantized and optionally exported model.
 
-    See docs/quark.md for more details.
+    See docs/dev_cli/quark.md for more details.
     """
 
     unique_name = "quark-quantize"
@@ -94,7 +94,7 @@ class QuarkQuantize(Tool):
             help="Number of samples for calibration.",
         )
 
-        # See docs/quark.md for more details.
+        # See docs/dev_cli/quark.md for more details.
         parser.add_argument(
             "--quant-scheme",
             type=str,

{lemonade_sdk-8.0.2 → lemonade_sdk-8.0.4}/src/lemonade/tools/report/table.py

@@ -74,6 +74,7 @@ class SimpleStat(TableColumn):
         align="center",
         omit_if_lean=False,
         wrap=None,
+        stat_fn=None,
     ):
         self.column_header = column_header
         self.stat = stat
@@ -81,6 +82,7 @@ class SimpleStat(TableColumn):
         self.align = align
         self.omit_if_lean = omit_if_lean
         self.wrap = wrap or self.default_wrap
+        self.stat_fn = stat_fn
 
     def get_str(self, build_stats, lean=False):
         if lean and self.omit_if_lean:
@@ -88,6 +90,8 @@ class SimpleStat(TableColumn):
         data = build_stats.get(self.stat, None)
         if data is None:
             return ""
+        if self.stat_fn:
+            data = self.stat_fn(data)
         cell_str = "\n".join(
             [_wrap(f"{x:{self.format_str}}", self.wrap) for x in _to_list(data)]
         )
@@ -233,6 +237,47 @@ class AdditionalStat(TableColumn):
         return "\n".join(cell_entry)
 
 
+class DictListStat(TableColumn):
+    """
+    A statistic that is a list of dicts and values from a given list of keys will be
+    pulled out of each dict and placed in the cell
+    """
+
+    def __init__(
+        self,
+        column_header,
+        statistic_name,
+        key_format_list,
+        align="center",
+        omit_if_lean=False,
+        wrap=None,
+    ):
+        self.column_header = column_header
+        self.statistic_name = statistic_name
+        self.key_format_list = key_format_list
+        self.align = align
+        self.omit_if_lean = omit_if_lean
+        self.wrap = wrap or self.default_wrap
+
+    def get_str(self, build_stats, lean=False):
+        if lean and self.omit_if_lean:
+            return None
+        stat = build_stats.get(self.statistic_name, None)
+        if not stat:
+            return ""
+        cell_entry = []
+        for stat_dict in stat:
+            line = [
+                format_str.format(stat_dict[key])
+                for key, format_str in self.key_format_list
+            ]
+            cell_entry.append(" ".join(line))
+        return "\n".join(cell_entry)
+
+    def get_keys(self):
+        return [self.statistic_name]
+
+
 ################################################################################
 # ABSTRACT BASE CLASS FOR DEFINING A TABLE
 ################################################################################
@@ -350,6 +395,28 @@ class Table(ABC):
                     headers.append(column.column_header)
                     col_align += (column.align,)
 
+        # Stat column headers
+        stat_columns = self.table_descriptor.get("stat_columns", [])
+        stat_columns_include = []
+        for column in stat_columns:
+            # Check to see that at least one build has data for the column
+            keep_column = False
+            if not (self.lean and column.omit_if_lean):
+                keys = column.get_keys()
+                for build_stats in self.all_stats:
+                    found = [(key in build_stats) for key in keys]
+                    if any(found):
+                        keep_column = True
+                        headers.append(column.column_header)
+                        col_align += (column.align,)
+                        break
+            stat_columns_include.append(keep_column)
+        stat_columns = [
+            column
+            for column, include in zip(stat_columns, stat_columns_include)
+            if include
+        ]
+
         # Final headers
         last_columns = self.table_descriptor.get("last_columns", [])
         for column in last_columns:
@@ -386,6 +453,12 @@ class Table(ABC):
                     if entry_str is not None:
                         row.append(entry_str)
 
+            # Per stat columns
+            for entry in stat_columns:
+                entry_str = entry.get_str(build_stats, self.lean)
+                if entry_str is not None:
+                    row.append(entry_str)
+
             # Final columns
             for entry in last_columns:
                 entry_str = entry.get_str(build_stats, self.lean)
@@ -514,6 +587,12 @@ class LemonadePerfTable(Table):
                     Keys.STD_DEV_TOKENS_PER_SECOND,
                     ".2f",
                 ),
+                SimpleStat(
+                    _wrap("Total Generated Tokens", 9),
+                    Keys.RESPONSE_TOKENS,
+                    "d",
+                    stat_fn=sum,
+                ),
                 SimpleStat(
                     _wrap("Memory Used (GB)", 8), Keys.MAX_MEMORY_USED_GBYTE, ".3f"
                 ),
@@ -537,6 +616,7 @@ class LemonadePerfTable(Table):
                 )
             ],
         },
+        "stat_columns": [],
         "last_columns": [
             SimpleStat(
                 "System Info",