aiqtoolkit 1.2.0rc4__py3-none-any.whl → 1.2rc9__py3-none-any.whl

aiq/profiler/calc/calculations.py

@@ -1,288 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-import numpy as np
-
-from aiq.profiler.calc.data_models import FitConfig
-from aiq.profiler.calc.data_models import GPUEstimates
-from aiq.profiler.calc.data_models import LinearFitResult
-
-logger = logging.getLogger(__name__)
-
-
-def compute_slope(concurrencies: list[float],
-                  time_metrics: list[float],
-                  fit_config: FitConfig | None = None) -> LinearFitResult:
-    """
-    Concurrency is the independent variable (x-axis) and time metric (which can be runtime or latency)
-    is the dependent variable (y-axis). This function computes the slope of the linear relationship
-    between concurrency and time metric.
-
-    Args:
-        concurrencies: List of concurrency values (x-axis)
-        time_metrics: List of time metric values (y-axis)
-        fit_config: Configuration for outlier detection and fit validation
-
-    Returns:
-        LinearFitResult containing slope, intercept, R-squared, and outliers removed
-
-    Raises:
-        ValueError: If the relationship is not linear (R² < min_r_squared)
-    """
-    # Use default config if none provided
-    if fit_config is None:
-        fit_config = FitConfig()
-
-    # Convert to numpy arrays for calculations
-    x = np.array(concurrencies)
-    y = np.array(time_metrics)
-
-    # Validate input
-    if len(x) != len(y):
-        raise ValueError("Concurrencies and time_metrics must have the same length")
-    if len(x) < 2:
-        raise ValueError("Need at least 2 points for linear regression")
-
-    outliers_removed = []
-
-    # Remove outliers if requested
-    if fit_config.remove_outliers and len(x) > 4:  # Need at least 4 points for outlier detection
-        x_clean, y_clean, removed_concurrencies = _remove_outliers(x, y, fit_config)
-        x, y = x_clean, y_clean
-        outliers_removed = removed_concurrencies
-
-    # Calculate linear regression using least squares
-    n = len(x)
-    sum_x = x.sum()
-    sum_y = y.sum()
-    sum_xy = (x * y).sum()
-    sum_x2 = (x**2).sum()
-
-    # Calculate slope and intercept
-    slope = (n * sum_xy - sum_x * sum_y) / (n * sum_x2 - sum_x**2)
-    intercept = (sum_y - slope * sum_x) / n
-
-    # Calculate R-squared
-    y_pred = slope * x + intercept
-    ss_res = ((y - y_pred)**2).sum()
-    ss_tot = ((y - y.mean())**2).sum()
-    r_squared = 1 - (ss_res / ss_tot) if ss_tot != 0 else 0.0
-
-    # Validate linearity
-    if r_squared < fit_config.min_r_squared:
-        raise ValueError(f"Poor linear fit detected (R² = {r_squared:.3f} < {fit_config.min_r_squared}). "
-                         f"The relationship may not be linear. Consider using non-linear regression.")
-
-    return LinearFitResult(slope=slope, intercept=intercept, r_squared=r_squared, outliers_removed=outliers_removed)
-
-
-def _remove_outliers(x: np.ndarray, y: np.ndarray, fit_config: FitConfig) -> tuple[np.ndarray, np.ndarray, list[int]]:
-    """
-    Remove outliers using the Interquartile Range (IQR) method.
-    For small concurrency range (≤ threshold points), also checks raw y-values for extreme outliers.
-
-    Args:
-        x: Input x values (concurrencies)
-        y: Input y values (time metrics)
-        fit_config: Configuration for outlier detection
-
-    Returns:
-        Tuple of (cleaned_x, cleaned_y, list_of_removed_concurrencies)
-    """
-    # if the number of concurrency points is less removing outliers can be challenging
-    # as extreme outliers can skew the results.
-    # We use a threshold to check for extreme outliers in raw y-values first.
-    n = len(x)
-    all_removed_concurrencies = []
-
-    # For smaller concurrency ranges, check for extreme outliers in raw y-values first
-    if n <= fit_config.small_concurrency_range_threshold:
-        # Calculate IQR on raw y-values
-        y_q1 = np.percentile(y, 25)
-        y_q3 = np.percentile(y, 75)
-        y_iqr = y_q3 - y_q1
-
-        # Use a more aggressive threshold for small datasets
-        y_lower_bound = y_q1 - fit_config.extreme_outlier_threshold * y_iqr  # More aggressive than 1.5
-        y_upper_bound = y_q3 + fit_config.extreme_outlier_threshold * y_iqr
-
-        # Find extreme outliers in raw values
-        extreme_outlier_mask = (y >= y_lower_bound) & (y <= y_upper_bound)
-        extreme_outliers_removed = np.sum(~extreme_outlier_mask)
-
-        if extreme_outliers_removed > 0:
-            extreme_removed_concurrencies = x[~extreme_outlier_mask].tolist()
-            all_removed_concurrencies.extend(extreme_removed_concurrencies)
-            logger.info("Removed %d extreme outliers from raw values: concurrencies %s",
-                        extreme_outliers_removed,
-                        extreme_removed_concurrencies)
-            # Continue with residual-based detection on the cleaned data
-            x = x[extreme_outlier_mask]
-            y = y[extreme_outlier_mask]
-            n = len(x)
-
-    # Standard residual-based outlier detection
-    # Calculate residuals from a simple linear fit
-    if n == 0:
-        raise ValueError("No data points remaining after outlier removal. Cannot compute linear fit.")
-
-    sum_x = x.sum()
-    sum_y = y.sum()
-    sum_xy = (x * y).sum()
-    sum_x2 = (x**2).sum()
-
-    slope = (n * sum_xy - sum_x * sum_y) / (n * sum_x2 - sum_x**2)
-    intercept = (sum_y - slope * sum_x) / n
-
-    # Calculate residuals
-    y_pred = slope * x + intercept
-    residuals = y - y_pred
-
-    # Use IQR method to detect outliers
-    q1 = np.percentile(residuals, 25)
-    q3 = np.percentile(residuals, 75)
-    iqr = q3 - q1
-
-    # Define outlier bounds (1.5 * IQR rule)
-    lower_bound = q1 - fit_config.conservative_outlier_threshold * iqr
-    upper_bound = q3 + fit_config.conservative_outlier_threshold * iqr
-
-    # Find non-outlier indices
-    non_outlier_mask = (residuals >= lower_bound) & (residuals <= upper_bound)
-
-    outliers_removed = np.sum(~non_outlier_mask)
-    residual_removed_concurrencies = x[~non_outlier_mask].tolist()
-    all_removed_concurrencies.extend(residual_removed_concurrencies)
-
-    # Add debugging for small datasets
-    if len(x) <= fit_config.small_concurrency_range_threshold:
-        logger.debug("Outlier detection for small dataset (n=%d):", len(x))
-        logger.debug("  Data points: %s", list(zip(x, y)))
-        logger.debug("  Residuals: %s", residuals.tolist())
-        logger.debug("  Q1=%.3f, Q3=%.3f, IQR=%.3f", q1, q3, iqr)
-        logger.debug("  Bounds: [%.3f, %.3f]", lower_bound, upper_bound)
-        logger.info("  Outliers removed: %d (concurrencies: %s)", outliers_removed, residual_removed_concurrencies)
-
-    return x[non_outlier_mask], y[non_outlier_mask], all_removed_concurrencies
-
-
-def calc_gpu_estimate_based_on_slope(target_time_metric: float,
-                                     target_users: int,
-                                     test_gpu_count: int,
-                                     observed_slope: float,
-                                     observed_intercept: float = 0.0) -> float:
-    """
-    Calculate the GPU estimate based on the slope of the time metric.
-
-    This function uses the linear relationship between concurrency and time metrics
-    to estimate the required GPU count for a target user load.
-
-    Args:
-        target_time_metric: Target time metric (latency or runtime) in seconds
-        observed_slope: Slope from linear regression of time vs concurrency
-        target_users: Target number of concurrent users
-        test_gpu_count: Number of GPUs used in the test
-        observed_intercept: Y-intercept from linear regression (default: 0.0)
-
-    Returns:
-        Estimated number of GPUs required
-
-    Raises:
-        ValueError: If target_time_metric is less than or equal to intercept
-    """
-    if target_time_metric <= observed_intercept:
-        raise ValueError(f"Target time metric ({target_time_metric}) must be greater than "
-                         f"the intercept ({observed_intercept}) for valid GPU estimation.")
-
-    # Calculate the concurrency that would achieve the target time metric
-    # Using the linear equation: time = slope * concurrency + intercept
-    # Solving for concurrency: concurrency = (time - intercept) / slope
-    calculated_concurrency = (target_time_metric - observed_intercept) / observed_slope
-    logger.info("Calculated concurrency: %f for target time metric: %f, observed intercept: %f, observed slope: %f",
-                calculated_concurrency,
-                target_time_metric,
-                observed_intercept,
-                observed_slope)
-
-    if calculated_concurrency <= 0:
-        raise ValueError(f"Calculated target concurrency ({calculated_concurrency}) is not positive. "
-                         f"This suggests the slope or intercept values may be invalid.")
-
-    # Estimate GPUs using the ratio of target users to target concurrency
-    # scaled by the test GPU count
-    gpu_estimate = (target_users / calculated_concurrency) * test_gpu_count
-
-    return gpu_estimate
-
-
-def calc_gpu_estimate_for_single_concurrency(target_llm_latency: float,
-                                             target_workflow_runtime: float,
-                                             target_users: int,
-                                             test_concurrency: int,
-                                             test_gpu_count: int,
-                                             observed_latency: float,
-                                             observed_runtime: float) -> GPUEstimates:
-    """
-    ROUGH ESTIMATE: Calculate GPU count estimate for a single concurrency level.
-
-    This is a simplified estimate that assumes linear scaling and should be used
-    as a baseline only. For more accurate estimates, use slope-based estimation
-    with multiple concurrency levels.
-
-    Formula based on the target latency:
-        G_required = (U_target / C_test) * (L_obs / L_target) * G_test
-
-    Formula based on the target runtime:
-        G_required = (U_target / C_test) * (R_obs / R_target) * G_test
-
-    where:
-        - U_target: Target number of users
-        - C_test: Test concurrency level
-        - L_obs: Observed LLM latency
-        - L_target: Target LLM latency
-        - R_obs: Observed workflow runtime
-        - R_target: Target workflow runtime
-        - G_test: Test GPU count
-
-    WARNING: This is a rough estimate that:
-    - Assumes perfect linear scaling (rarely true in practice)
-    - Doesn't account for GPU utilization inefficiencies
-    - May underestimate GPU requirements for high concurrency
-    - Should be validated against slope-based estimates
-    """
-    use_latency = target_llm_latency > 0
-    use_runtime = target_workflow_runtime > 0
-
-    # If observed latency or runtime exceeds the target, return empty estimates
-    if use_latency and observed_latency > target_llm_latency:
-        return GPUEstimates()
-
-    if use_runtime and observed_runtime > target_workflow_runtime:
-        return GPUEstimates()
-
-    # Calculate multipliers (how much faster we need to be)
-    llm_latency_multiplier = observed_latency / target_llm_latency if use_latency else 1.0
-    wf_runtime_multiplier = observed_runtime / target_workflow_runtime if use_runtime else 1.0
-
-    # Calculate GPU estimates using the corrected formula
-    gpu_estimate_by_wf_runtime = (target_users /
-                                  test_concurrency) * wf_runtime_multiplier * test_gpu_count if use_runtime else None
-    gpu_estimate_by_llm_latency = (target_users /
-                                   test_concurrency) * llm_latency_multiplier * test_gpu_count if use_latency else None
-
-    return GPUEstimates(gpu_estimate_by_wf_runtime=gpu_estimate_by_wf_runtime,
-                        gpu_estimate_by_llm_latency=gpu_estimate_by_llm_latency)

aiq/profiler/calc/data_models.py

@@ -1,188 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import typing
-from pathlib import Path
-
-from pydantic import BaseModel
-from pydantic import Field
-
-
-class FitConfig(BaseModel):
-    """
-    Configuration parameters for linear fit and outlier detection.
-    """
-    # Threshold for small concurrency range (≤ 8 points) to check for extreme outliers in raw y-values first
-    small_concurrency_range_threshold: int = 8
-
-    # Extreme outlier threshold is 2.0 times the IQR, extreme outliers are removed
-    extreme_outlier_threshold: float = 2.0
-
-    # Conservative outlier threshold is 1.5 times the IQR, conservative outliers are removed
-    conservative_outlier_threshold: float = 1.5
-
-    # Minimum R-squared value required for a valid linear fit
-    min_r_squared: float = 0.7
-
-    # Whether to remove outliers during linear fit calculation
-    remove_outliers: bool = True
-
-
-class CalcRunnerConfig(BaseModel):
-    """
-    Parameters used for a calc runner.
-    """
-    # base config and endpoints (if remote)- not needed in offline mode
-    config_file: Path | None = None
-    # endpoint to use for the workflow, if not provided the workflow is run locally
-    endpoint: str | None = None
-    # timeout for the workflow
-    endpoint_timeout: int = 300
-
-    # if true workflow is not run, instead results from previous runs are used to estimate the
-    # GPU count
-    offline_mode: bool = False
-
-    # number of passes at each concurrency, if 0 the dataset is adjusted to a multiple of the
-    # concurrency
-    num_passes: int = 0
-    # concurrency values to test
-    concurrencies: list[int] = [1, 2, 4, 8]
-
-    # Targets for GPU estimation
-    target_llm_latency_p95: float = 0
-    target_workflow_runtime_p95: float = 0
-    target_users: int = 0
-
-    # Test setup information needed for GPU estimation
-    test_gpu_count: int = 0
-
-    # output directory for results
-    output_dir: Path | None = None
-    # if true, the job is stored in a new subdirectory of the output directory
-    append_job: bool = False
-    # if true, the data is plotted
-    plot_data: bool = True
-
-    # Configuration for linear fit and outlier detection
-    fit_config: FitConfig = Field(default_factory=FitConfig)
-
-
-# Sizing metrics are gathered from the evaluation runs and used as input by the calculator.
-class SizingMetricPerItem(BaseModel):
-    """
-    Sizing metrics per dataset entry item.
-    """
-    # LLM latency
-    llm_latency: float
-    # workflow runtime
-    workflow_runtime: float
-
-
-class SizingMetricsAlerts(BaseModel):
-    """
-    Sizing metrics alerts.
-    """
-    # if true, the workflow was interrupted that concurrency cannot be used
-    workflow_interrupted: bool = False
-
-
-class SizingMetrics(BaseModel):
-    """
-    Sizing metrics for a single concurrency.
-    """
-    # alerts associated with the sizing metrics
-    alerts: SizingMetricsAlerts = Field(default_factory=SizingMetricsAlerts)
-
-    # p95 LLM latency
-    llm_latency_p95: float = 0.0
-    # p95 workflow runtime
-    workflow_runtime_p95: float = 0.0
-    # total workflow runtime
-    total_runtime: float = 0.0
-    # per item metrics, key is the dataset entry id
-    per_item_metrics: dict[typing.Any, SizingMetricPerItem] = {}
-
-
-class LinearFitResult(BaseModel):
-    """
-    Result of linear regression including slope, intercept, and quality metrics.
-    """
-    slope: float
-    intercept: float
-    r_squared: float
-    outliers_removed: list[int]
-
-
-class FitResults(BaseModel):
-    """
-    Linear fit results for both LLM latency and workflow runtime analysis.
-    """
-    llm_latency_fit: LinearFitResult | None = None
-    wf_runtime_fit: LinearFitResult | None = None
-
-
-# GPU estimates are generated by the calculator.
-class GPUEstimates(BaseModel):
-    """
-    GPU estimates.
-    """
-    # GPU estimate based on the workflow runtime
-    gpu_estimate_by_wf_runtime: float | None = None
-    # GPU estimate based on the LLM latency
-    gpu_estimate_by_llm_latency: float | None = None
-
-
-# Calc runner alerts are generated by the calculator.
-class CalcAlerts(BaseModel):
-    """
-    Calc runner alerts.
-    """
-    # if true, the run was identified as an outlier by the workflow runtime linear fit
-    outlier_workflow_runtime: bool = False
-    # if true, the run was identified as an outlier by the LLM latency linear fit
-    outlier_llm_latency: bool = False
-
-    # number of items that are greater than the target latency
-    num_items_greater_than_target_latency: int = 0
-    # number of items that are greater than the target runtime
-    num_items_greater_than_target_runtime: int = 0
-
-
-class CalcData(BaseModel):
-    """
-    Output of the calc runner per concurrency.
-    """
-    # ROUGH GPU estimates per concurrency: these are not used for the final GPU estimation
-    # they are only available for information purposes
-    gpu_estimates: GPUEstimates = Field(default_factory=GPUEstimates)
-    # Calc runner alerts
-    alerts: CalcAlerts = Field(default_factory=CalcAlerts)
-    # Sizing metrics
-    sizing_metrics: SizingMetrics = Field(default_factory=SizingMetrics)
-
-
-class CalcRunnerOutput(BaseModel):
-    """
-    Output of the calc runner.
-    """
-    # GPU estimates based on the slope of the time vs concurrency, calculated online or offline
-    gpu_estimates: GPUEstimates = Field(default_factory=GPUEstimates)
-
-    # Linear fit results for analysis and debugging
-    fit_results: FitResults = Field(default_factory=FitResults)
-
-    # Per-concurrency data (GPU estimates, out-of-range runs, and sizing metrics)
-    calc_data: dict[int, CalcData] = {}