nsight-python 0.9.5__py3-none-any.whl → 1.0.0__py3-none-any.whl

nsight/__init__.py

@@ -1,12 +1,16 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+from importlib.metadata import version
+
 from nsight import analyze
 from nsight.annotation import annotate
-from nsight.utils import col_panel, row_panel
+from nsight.collection.ncu import try_init_injection
+from nsight.utils import VerbosityLevel, col_panel, row_panel
 
-# Versioning Scheme: major.minor.build
-__version__ = "0.9.5"
+# Load injection library and symbols for NCU attach (if available)
+try_init_injection()
 
+__version__ = version("nsight-python")
 
-__all__ = ["analyze", "annotate"]
+__all__ = ["analyze", "annotate", "VerbosityLevel"]

nsight/analyze.py

@@ -7,14 +7,16 @@ import os
 import tempfile
 import warnings
 from collections.abc import Callable, Iterable, Sequence
-from typing import Any, Literal, overload
+from typing import Any, Literal, Mapping, overload
 
 import matplotlib
 import matplotlib.figure
 import numpy as np
+from numpy.typing import NDArray
 
 import nsight.collection as collection
 import nsight.visualization as visualization
+from nsight.utils import VerbosityLevel
 
 
 # Overload 1: When used without parentheses: @kernel
@@ -31,15 +33,24 @@ def kernel(
     *,
     configs: Iterable[Any] | None = None,
     runs: int = 1,
-    derive_metric: Callable[..., float] | None = None,
+    derive_metric: (
+        Callable[
+            ..., float | tuple[float, str] | Mapping[str, float | tuple[float, str]]
+        ]
+        | None
+    ) = None,
     normalize_against: str | None = None,
-    output: Literal["quiet", "progress", "verbose"] = "progress",
+    verbosity: VerbosityLevel = VerbosityLevel.INFO,
     metrics: Sequence[str] = ["gpu__time_duration.sum"],
     ignore_kernel_list: Sequence[str] | None = None,
     clock_control: Literal["base", "none"] = "none",
     cache_control: Literal["all", "none"] = "all",
     replay_mode: Literal["kernel", "range"] = "kernel",
-    thermal_control: bool = True,
+    thermal_mode: Literal["auto", "manual", "off"] = "auto",
+    thermal_wait: int | None = None,
+    thermal_cont: int | None = None,
+    thermal_timeout: int | None = None,
+    thermal_device: int | None = None,
     combine_kernel_metrics: Callable[[float, float], float] | None = None,
     output_prefix: str | None = None,
     output_csv: bool = False,
@@ -52,15 +63,24 @@ def kernel(
     *,
     configs: Iterable[Any] | None = None,
     runs: int = 1,
-    derive_metric: Callable[..., float] | None = None,
+    derive_metric: (
+        Callable[
+            ..., float | tuple[float, str] | Mapping[str, float | tuple[float, str]]
+        ]
+        | None
+    ) = None,
     normalize_against: str | None = None,
-    output: Literal["quiet", "progress", "verbose"] = "progress",
+    verbosity: VerbosityLevel = VerbosityLevel.INFO,
     metrics: Sequence[str] = ["gpu__time_duration.sum"],
     ignore_kernel_list: Sequence[str] | None = None,
     clock_control: Literal["base", "none"] = "none",
     cache_control: Literal["all", "none"] = "all",
     replay_mode: Literal["kernel", "range"] = "kernel",
-    thermal_control: bool = True,
+    thermal_mode: Literal["auto", "manual", "off"] = "auto",
+    thermal_wait: int | None = None,
+    thermal_cont: int | None = None,
+    thermal_timeout: int | None = None,
+    thermal_device: int | None = None,
     combine_kernel_metrics: Callable[[float, float], float] | None = None,
     output_prefix: str | None = None,
     output_csv: bool = False,
@@ -86,6 +106,13 @@ def kernel(
         - ``**kwargs``: Original function keyword arguments
         - Returns ``ProfileResults`` object containing profiling data
 
+    .. note::
+        Configs are passed to the decorated function as positional arguments.
+        Both positional and keyword-only parameters are supported in the function
+        signature. If the function has ``*args`` or ``**kwargs``, they are tolerated
+        but will always be empty during profiling. Parameters with default values
+        may be omitted from configs — missing values will be filled from the defaults.
+
 
     Parameters:
         configs: An iterable of configurations to run the function with. Each configuration can be either:
@@ -98,6 +125,10 @@ def kernel(
 
                    configs = [1, 2, 3, 4]
 
+            If configs provide fewer values than the function has parameters,
+            the remaining parameters must have default values, which will be used
+            automatically.
+
             If the configs are not provided at decoration time, they must be provided when calling the decorated function.
         runs:  Number of times each configuration should be executed.
         derive_metric:
@@ -105,15 +136,25 @@ def kernel(
             This can be used to compute derived metrics like TFLOPs that cannot
             be captured by ncu directly. The function takes the metric values and
             the arguments of the profile-decorated function and returns the new
-            metric. The parameter order requirements for the custom function:
+            metrics. Return value can be either:
+
+            - A single value (float/int): Will be added as a new metric with the function name as the metric name. For lambda functions, the metric name will be ``"<lambda>"``.
+            - A tuple (value, unit): A single metric value with its unit specified. The value will be added with the function name as the metric name, and the unit will be stored in the ``Unit`` column of the dataframe. Example: ``(tflops_value, "TFLOPS")``.
+            - A dictionary: Keys will be used as metric names, values can be either:
+                - Scalar values (float/int): Metric values without units. ``np.nan`` will be added to the ``Unit`` column and a warning will be issued.
+                - Tuples (value, unit): Metric values with units specified. The unit will be stored in the ``Unit`` column of the dataframe. Example: ``{"TFLOPS": (tflops_value, "TFLOPS"), "ArithIntensity": (intensity_value, "FLOP/byte")}``.
+
+            The parameter order requirements for the custom function:
 
             - First several arguments: Must exactly match the order of metrics declared in the @kernel decorator. These arguments will receive the actual measured values of those metrics.
             - Remaining arguments: Must exactly match the signature of the decorated function. In other words, the original function's parameters are passed in order.
 
+            **Note on units**: When units are not provided (returning scalar values instead of tuples), ``np.nan`` will be added to the ``Unit`` column in the dataframe and a warning will be issued. To avoid warnings and ensure proper unit tracking, return tuples ``(value, unit)`` for all derived metrics.
+
             See the examples for concrete use cases.
         normalize_against:
-            Annotation name to normalize metrics against.
-            This is useful to compute relative metrics like speedup.
+            Annotation name to normalize metrics against (computes current / baseline).
+            Combine with ``derive_metric`` to compute speedup (reciprocal of normalized value).
         metrics: The metrics to collect. By default, kernel runtimes in nanoseconds
             are collected. Default: ``["gpu__time_duration.sum"]``. To see the available
             metrics on your system, use the command: ``ncu --query-metrics``.
@@ -147,12 +188,56 @@ def kernel(
 
             Default: ``"kernel"``
 
-        thermal_control : Toggles whether to enable thermal control. Default: ``True``
-        output: Controls the verbosity level of the output.
+        thermal_mode: Controls GPU thermal management mode. Default: ``"auto"``
+
+            Monitors GPU thermal headroom (T.Limit) and pauses profiling when GPU gets
+            too hot, resuming after cooling. Prevents thermal throttling from affecting
+            profiling accuracy.
+
+            **Modes:**
+
+            - ``"auto"``: Adaptive mode - automatically adjusts thermal_cont threshold based on GPU heating behavior.
+            - ``"manual"``: Fixed mode - uses specified thermal_wait and thermal_cont thresholds without adaptation.
+            - ``"off"``: Thermal control disabled - no monitoring or pausing.
+
+        thermal_wait: Thermal headroom threshold (T.Limit in °C) that triggers cooling pause.
+
+            When GPU thermal headroom drops to this value, profiling pauses and waits
+            for the GPU to cool down. Lower values allow the GPU to run hotter before
+            pausing (more aggressive, faster profiling, higher throttling risk).
+
+            If ``None``, uses default value (10°C). Applied in both "auto" and "manual" modes.
+
+        thermal_cont: Thermal headroom threshold (T.Limit in °C) to resume profiling after cooling.
+
+            After a cooling pause, profiling resumes once GPU thermal headroom reaches
+            this value. Higher values mean the GPU cools more before resuming (safer,
+            fewer cooling cycles, but slower overall profiling time).
+
+            If ``None``, uses default value (20°C). In "auto" mode, this value adapts based on
+            workload characteristics. In "manual" mode, it remains fixed.
 
-            - ``"quiet"``: Suppresses all output.
-            - ``"progress"``: Shows a progress bar along with details about profiling and data extraction progress.
-            - ``"verbose"``: Displays the progress bar, configuration-specific logs, and profiler logs.
+            **Requirement:** Must be greater than thermal_wait.
+
+        thermal_timeout: Maximum wait time in seconds for GPU to cool down.
+            Default: 180 seconds
+
+        thermal_device: CUDA device ordinal to monitor for thermal throttling.
+
+            If set, monitor that CUDA device. If ``None``, monitor the current
+            CUDA device context. Device numbering follows CUDA runtime ordinals and
+            honors ``CUDA_VISIBLE_DEVICES``.
+
+            Default: ``None``
+
+        verbosity: Controls the verbosity level of the output. Selecting level N
+            enables output at all levels with value <= N. Default: ``VerbosityLevel.INFO``.
+
+            - ``VerbosityLevel.SILENT``: Suppresses all output.
+            - ``VerbosityLevel.INFO``: Shows a progress bar, profiling completion messages,
+              and report file paths.
+            - ``VerbosityLevel.DEBUG``: Adds configuration-specific logs, the full NCU CLI
+              command, and enables NCU verbose output.
 
         output_prefix: When specified, all intermediate profiler files are created with this prefix.
             For example, if `output_prefix="/home/user/run1_"`, the profiler will generate:
@@ -175,8 +260,8 @@ def kernel(
 
                 - ``Annotation``: Name of the annotated region being profiled
                 - ``Value``: Raw metric values collected by the profiler
-                - ``Metric``: The metrics being collected (e.g., ``gpu__time_duration.sum``)
-                - ``Transformed``: Name of the function used to transform the metrics (specified via ``derive_metric``), or ``False`` if no transformation was applied. For lambda functions, this shows ``"<lambda>"``
+                - ``Metric``: The metrics being collected (e.g., ``gpu__time_duration.sum``) and the metrics being derived
+                - ``Unit``: Unit of measurement for the metric value (e.g., ``ns`` for nanoseconds)
                 - ``Kernel``: Name of the GPU kernel(s) launched
                 - ``GPU``: GPU device name
                 - ``Host``: Host machine name
@@ -197,8 +282,8 @@ def kernel(
                 - ``CI95_Upper``: Upper bound of the 95% confidence interval
                 - ``RelativeStdDevPct``: Standard deviation as a percentage of the mean
                 - ``StableMeasurement``: Boolean indicating if the measurement is stable (low variance). The measurement is stable if ``RelativeStdDevPct`` < 2 % .
-                - ``Metric``: The metrics being collected
-                - ``Transformed``: Name of the function used to transform the metrics (specified via ``derive_metric``), or ``False`` if no transformation was applied. For lambda functions, this shows ``"<lambda>"``
+                - ``Metric``: The metrics being collected and the metrics being derived
+                - ``Unit``: Unit of measurement for the metric value (e.g., ``ns`` for nanoseconds)
                 - ``Kernel``: Name of the GPU kernel(s) launched
                 - ``GPU``: GPU device name
                 - ``Host``: Host machine name
@@ -208,33 +293,37 @@ def kernel(
     # Strip whitespace
     metrics = [m.strip() for m in metrics]
 
+    # Validate thermal parameters if both are provided
+    if thermal_wait is not None and thermal_cont is not None:
+        if thermal_cont <= thermal_wait:
+            raise ValueError(
+                f"Invalid thermal control parameters: thermal_cont ({thermal_cont}°C) "
+                f"must be greater than thermal_wait ({thermal_wait}°C). "
+                f"thermal_cont is the target thermal headroom after cooling, "
+                f"thermal_wait is the threshold that triggers cooling."
+            )
+
     def _create_profiler() -> collection.core.NsightProfiler:
         """Helper to create the profiler with the given settings."""
-        if output not in ("quiet", "progress", "verbose"):
-            raise ValueError("output must be 'quiet', 'progress' or 'verbose'")
-
-        output_progress = output == "progress" or output == "verbose"
-        output_detailed = output == "verbose"
-
         # Create the output paths needed for the ncu report, ncu logs and the CSVs
         prefix = output_prefix
-        if "NSPY_NCU_PROFILE" not in os.environ:
-            if prefix is None:
-                prefix = tempfile.mkdtemp(prefix="nspy_")
-                prefix = os.path.join(
-                    prefix, ""
-                )  # Adds a trailing forward/backward slash
-            else:
-                os.makedirs(os.path.dirname(prefix) or ".", exist_ok=True)
+        if prefix is None:
+            prefix = tempfile.mkdtemp(prefix="nspy_")
+            prefix = os.path.join(prefix, "")  # Adds a trailing forward/backward slash
+        else:
+            os.makedirs(os.path.dirname(prefix) or ".", exist_ok=True)
 
         settings = collection.core.ProfileSettings(
             configs=configs,
             runs=runs,
-            output_progress=output_progress,
-            output_detailed=output_detailed,
+            verbosity=verbosity,
             derive_metric=derive_metric,
             normalize_against=normalize_against,
-            thermal_control=thermal_control,
+            thermal_mode=thermal_mode,
+            thermal_wait=thermal_wait,
+            thermal_cont=thermal_cont,
+            thermal_timeout=thermal_timeout,
+            thermal_device=thermal_device,
             output_prefix=prefix,
             output_csv=output_csv,
         )
@@ -259,29 +348,63 @@ def kernel(
         return profiler(_func)  # type: ignore[return-value]
 
 
-def _validate_metric(result: collection.core.ProfileResults) -> None:
+def _legalize_metric(result: collection.core.ProfileResults, metric: str | None) -> str:
     """
-    Check if ProfileResults contains only a single metric.
+    Legalize the metric parameter for plotting.
+
+    This function ensures that the provided metric parameter is valid for plotting.
+    If the metric is None, it verifies there is exactly one metric available and
+    returns that metric's name. If a metric is specified, it validates that the
+    metric exists in the profile results and returns itself.
 
     Args:
-        result: ProfileResults object
+        result: ProfileResults object containing profiling data.
+        metric: The name of the metric to plot, or None to use the single metric
+            if only one exists.
+
+    Returns:
+        The validated metric name for plotting.
 
     Raises:
-        ValueError: If multiple metrics are detected
+        ValueError: Raised in the following cases:
+            1. When `metric` is None and multiple metrics are found.
+            2. When `metric` is None and no metrics are found.
+            3. When `metric` is specified but not present in the results.
     """
     df = result.to_dataframe()
 
-    # Check for multiple metrics in "Metric" column
-    unique_metrics = df["Metric"].unique()
-    if len(unique_metrics) > 1:
-        raise ValueError(
-            f"Cannot visualize {len(unique_metrics)} > 1 metrics with the "
-            "@nsight.analyze.plot decorator."
-        )
+    # Extract unique metric names from the DataFrame
+    unique_metrics: NDArray[Any] = df["Metric"].unique()
+    n_metrics = len(unique_metrics)
+
+    if metric is None:
+        # Case: No metric specified - ensure there is exactly one metric
+        if n_metrics > 1:
+            raise ValueError(
+                f"Cannot visualize multiple metrics ({n_metrics} > 1) with the "
+                "@nsight.analyze.plot decorator. Please specify a metric to plot. "
+                f"The metrics found are: {unique_metrics.tolist()}"
+            )
+        elif n_metrics == 0:
+            raise ValueError(
+                "No metrics found in the profile results. Please ensure that the "
+                "profiler is configured correctly."
+            )
+        else:  # n_metrics == 1
+            return str(unique_metrics[0])
+    else:
+        # Case: Metric specified - ensure it exists in the results
+        if metric not in unique_metrics:
+            raise ValueError(
+                f"Metric '{metric}' not found in the profile results. Please check "
+                f"the metric name and try again. The metrics found are: {unique_metrics.tolist()}."
+            )
+        return metric
 
 
 def plot(
     filename: str = "plot.png",
+    metric: str | None = None,
     *,
     title: str = "Nsight Analyze Kernel Plot Results",
     ylabel: str | None = None,
@@ -320,6 +443,10 @@ def plot(
         def my_func(...):
 
     Args:
+        metric: The specific metric to plot (e.g., ``gpu__time_duration.sum``).
+            If None (default), the function will plot the single metric found in
+            the ``ProfileResults`` object containing profiling data, if only one exists,
+            otherwise it will raise an error if multiple metrics are present. Default: ``None``
         filename: Filename to save the plot. Default: ``'plot'``
         title: Title for the plot. Default: ``'Nsight Analyze Kernel Plot Results'``
         ylabel: Label for the y-axis in the generated plot.
@@ -357,29 +484,29 @@ def plot(
         def wrapper(*args: Any, **kwargs: Any) -> Any:
             result = func(*args, **kwargs)
 
-            if "NSPY_NCU_PROFILE" not in os.environ:
-                # Check for multiple metrics or complex data structures
-                _validate_metric(result)
-
-                visualization.visualize(
-                    result.to_dataframe(),
-                    row_panels=row_panels,
-                    col_panels=col_panels,
-                    x_keys=x_keys,
-                    print_data=print_data,
-                    title=title,
-                    filename=filename,
-                    ylabel=ylabel or "",
-                    annotate_points=annotate_points,
-                    show_avg=show_avg,
-                    show_geomean=show_geomean,
-                    plot_type=plot_type,
-                    plot_width=plot_width,
-                    plot_height=plot_height,
-                    variant_fields=variant_fields,
-                    variant_annotations=variant_annotations,
-                    plot_callback=plot_callback,
-                )
+            # Legalize the user-provided metric for plotting
+            legalized_metric = _legalize_metric(result, metric)
+
+            visualization.visualize(
+                result.to_dataframe(),
+                metric=legalized_metric,
+                row_panels=row_panels,
+                col_panels=col_panels,
+                x_keys=x_keys,
+                print_data=print_data,
+                title=title,
+                filename=filename,
+                ylabel=ylabel or "",
+                annotate_points=annotate_points,
+                show_avg=show_avg,
+                show_geomean=show_geomean,
+                plot_type=plot_type,
+                plot_width=plot_width,
+                plot_height=plot_height,
+                variant_fields=variant_fields,
+                variant_annotations=variant_annotations,
+                plot_callback=plot_callback,
+            )
             return result
 
         return wrapper

nsight/annotation.py

@@ -129,6 +129,10 @@ class annotate(nvtx.annotate):  # type: ignore[misc]
         Nested annotations are currently not supported. However, since each
         annotation is expected to contain a single kernel launch by default,
         nested annotations should not be necessary in typical usage scenarios.
+        Entering and exiting the annotation context pushes and pops an NVTX
+        range, so calling ``nvtx.pop_range()`` inside the annotation context
+        without a corresponding push would pop the annotation context's own
+        range instead, resulting in incorrect profiling results.
 
     """