nemo-evaluator-launcher 0.1.0rc2__py3-none-any.whl

nemo_evaluator_launcher/common/helpers.py

@@ -0,0 +1,157 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import copy
+import datetime
+from typing import Optional
+
+from omegaconf import DictConfig, OmegaConf
+
+from nemo_evaluator_launcher.common.logging_utils import logger
+
+
+def get_eval_factory_command(
+    cfg: DictConfig, user_task_config: DictConfig, task_definition: dict
+) -> str:
+    overrides = copy.deepcopy(dict(cfg.evaluation.get("overrides", {})))
+    overrides.update(dict(user_task_config.get("overrides", {})))
+    # NOTE(dfridman): Temporary fix to make sure that the overrides arg is not split into multiple lines.
+    # Consider passing a JSON object on Eval Factory side
+    overrides = {
+        k: (v.strip("\n") if isinstance(v, str) else v) for k, v in overrides.items()
+    }
+    overrides_str = ",".join([f"{k}={v}" for k, v in overrides.items()])
+    model_url = get_endpoint_url(cfg, user_task_config, task_definition)
+    command = f"""nv_eval run_eval \
+    --model_id {get_served_model_name(cfg)} \
+    --model_type {task_definition["endpoint_type"]} \
+    --eval_type {task_definition["task"]} \
+    --model_url {model_url} \
+    --api_key_name API_KEY \
+    --output_dir /results"""
+    if overrides_str:
+        command = f"{command} --overrides {overrides_str}"
+    return command
+
+
+def get_endpoint_url(
+    cfg: DictConfig, user_task_config: DictConfig, task_definition: dict
+) -> str:
+    def apply_url_override(url: str) -> str:
+        """Apply user URL override if provided."""
+        override_url = user_task_config.get("overrides", {}).get(
+            "config.target.api_endpoint.url"
+        )
+        return override_url if override_url is not None else url
+
+    if cfg.deployment.type == "none":
+        # For deployment: none, use target URL regardless of executor type
+        if OmegaConf.is_missing(cfg.target.api_endpoint, "url"):
+            raise ValueError(
+                "API endpoint URL is not set. Add `target.api_endpoint.url` to your config "
+                "OR override via CLI"
+            )
+        return apply_url_override(cfg.target.api_endpoint.url)
+
+    elif (
+        hasattr(cfg, "target")
+        and hasattr(cfg.target, "api_endpoint")
+        and hasattr(cfg.target.api_endpoint, "url")
+        and not OmegaConf.is_missing(cfg.target.api_endpoint, "url")
+    ):
+        # For Lepton executor with dynamically set target URL
+        return apply_url_override(cfg.target.api_endpoint.url)
+
+    else:
+        # Local executor - use localhost
+        task_endpoint_type = task_definition["endpoint_type"]
+        endpoint_uri = cfg.deployment.endpoints[task_endpoint_type]
+        endpoint_url = f"http://127.0.0.1:{cfg.deployment.port}{endpoint_uri}"
+        return endpoint_url
+
+
+def get_health_url(cfg: DictConfig, endpoint_url: str) -> str:
+    if cfg.deployment.type == "none":
+        logger.warning("Using endpoint URL as health URL", will_be_used=endpoint_url)
+        return endpoint_url  # TODO(public release) is using model url as health url OK?
+    health_uri = cfg.deployment.endpoints["health"]
+    health_url = f"http://127.0.0.1:{cfg.deployment.port}{health_uri}"
+    return health_url
+
+
+def get_served_model_name(cfg: DictConfig) -> str:
+    if cfg.deployment.type == "none":
+        return str(cfg.target.api_endpoint.model_id)
+    else:
+        return str(cfg.deployment.served_model_name)
+
+
+def get_api_key_name(cfg: DictConfig) -> str | None:
+    res = cfg.get("target", {}).get("api_endpoint", {}).get("api_key_name", None)
+    return str(res) if res else None
+
+
+def get_timestamp_string(include_microseconds: bool = True) -> str:
+    """Get timestamp in format YYYYmmdd_HHMMSS_ffffff."""
+    dt = datetime.datetime.now()
+    fmt = "%Y%m%d_%H%M%S"
+    if include_microseconds:
+        fmt += "_%f"
+    dts = datetime.datetime.strftime(dt, fmt)
+    return dts
+
+
+def get_eval_factory_dataset_size_from_run_config(run_config: dict) -> Optional[int]:
+    config = run_config["config"]
+    limit_samples = config["params"].get("limit_samples", None)
+    if limit_samples is not None:
+        return int(limit_samples)
+
+    # TODO(dfridman): Move `dataset_size` values to the corresponding `framework.yaml` in Eval Factory.
+    dataset_sizes = {
+        ("lm-evaluation-harness", "ifeval"): 541,
+        ("simple_evals", "gpqa_diamond"): 198,
+        ("simple_evals", "gpqa_diamond_nemo"): 198,
+        ("simple_evals", "AA_math_test_500"): 500,
+        ("simple_evals", "math_test_500_nemo"): 500,
+        ("simple_evals", "aime_2024_nemo"): 30,
+        ("simple_evals", "AA_AIME_2024"): 30,
+        ("simple_evals", "aime_2025_nemo"): 30,
+        ("simple_evals", "AIME_2025"): 30,
+        ("simple_evals", "humaneval"): 164,
+        ("simple_evals", "mmlu"): 14042,
+        ("simple_evals", "mmlu_pro"): 12032,
+        ("bigcode-evaluation-harness", "mbpp"): 500,
+        ("bigcode-evaluation-harness", "humaneval"): 164,
+        ("livecodebench", "livecodebench_0724_0125"): 315,
+        ("livecodebench", "livecodebench_0824_0225"): 279,
+        ("hle", "hle"): 2684,
+        ("scicode", "aa_scicode"): 338,
+    }
+    dataset_size = dataset_sizes.get((run_config["framework_name"], config["type"]))
+    if dataset_size is None:
+        return None
+    else:
+        dataset_size = int(dataset_size)
+
+    downsampling_ratio = (
+        config["params"].get("extra", {}).get("downsampling_ratio", None)
+    )
+    if downsampling_ratio is not None:
+        dataset_size = int(round(dataset_size * downsampling_ratio))
+
+    n_samples = int(config["params"].get("extra", {}).get("n_samples", 1))
+    dataset_size *= n_samples
+    return dataset_size

nemo_evaluator_launcher/common/logging_utils.py

@@ -0,0 +1,349 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Logging configuration module for nemo-evaluator-launcher.
+
+This module provides a centralized logging configuration using structlog that outputs
+to both stderr and a log file. All modules should import and use the logger from this
+module to ensure consistent logging behavior across the application.
+
+LOGGING POLICY:
+==============
+All logging in this project MUST go through this module. This is enforced by a pre-commit
+hook that checks for violations.
+
+DO NOT:
+- import structlog directly
+- import logging directly
+- call structlog.get_logger() directly
+- call logging.getLogger() directly
+
+DO:
+- from nemo_evaluator_launcher.common.logging_utils import logger
+- from nemo_evaluator_launcher.common.logging_utils import get_logger
+
+Examples:
+    # Correct
+    from nemo_evaluator_launcher.common.logging_utils import logger
+    logger.info("User logged in", user_id="12345")
+
+    # Incorrect
+    import structlog
+    logger = structlog.get_logger()
+    logger.info("User logged in")
+"""
+
+import json
+import logging
+import logging.config
+import os
+import pathlib
+import sys
+from datetime import datetime
+from pprint import pformat
+from typing import Any, Dict
+
+import structlog
+
+# If this env var is set, it will override a more standard "LOG_LEVEL". If
+# both are unset, default would be used.
+_LOG_LEVEL_ENV_VAR = "NEMO_EVALUATOR_LOG_LEVEL"
+_DEFAULT_LOG_LEVEL = "WARNING"
+_SENSITIVE_KEY_SUBSTRINGS = {
+    # Keep minimal, broad substrings (normalized: lowercased, no spaces/_/-)
+    "authorization",  # covers proxy-authorization, etc.
+    "apikey",  # covers api_key, api-key, x-api-key, nvidia_api_key, ...
+    "accesskey",  # covers access_key / access-key
+    "privatekey",
+    "token",  # covers access_token, id_token, refresh_token, *_token
+    "secret",  # covers openai_client_secret, aws_secret_access_key, *_secret
+    "password",
+    "pwd",  # common shorthand
+    "passwd",  # common variant
+}
+
+
+def _mask(val: object) -> str:
+    s = str(val)
+    if len(s) <= 10:
+        return "[REDACTED]"
+    return f"{s[:2]}…{s[-2:]}"
+
+
+def _normalize(name: object) -> str:
+    if not isinstance(name, str):
+        return ""
+    s = name.lower()
+    # drop spaces, hyphens, underscores
+    return s.replace(" ", "").replace("-", "").replace("_", "")
+
+
+def _is_sensitive_key(key: object) -> bool:
+    k = _normalize(key)
+    return any(substr in k for substr in _SENSITIVE_KEY_SUBSTRINGS)
+
+
+def _redact_mapping(m: dict) -> dict:
+    red = {}
+    for k, v in m.items():
+        if _is_sensitive_key(k):
+            red[k] = _mask(v)
+        elif isinstance(v, dict):
+            red[k] = _redact_mapping(v)
+        else:
+            red[k] = v
+    return red
+
+
+def redact_processor(_: Any, __: str, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+    if os.getenv("LOG_DISABLE_REDACTION", "").lower() in {"1", "true", "yes"}:
+        return event_dict
+    return _redact_mapping(event_dict)
+
+
+def _ensure_log_dir() -> pathlib.Path:
+    """Ensure the log directory exists and return its path."""
+    log_dir = pathlib.Path.home() / ".nemo-evaluator" / "logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    return log_dir
+
+
+def _get_env_log_level() -> str:
+    """Get log level from environment variable, translating single letters to full names.
+
+    Translates:
+    - D -> DEBUG
+    - I -> INFO
+    - W -> WARNING
+    - E -> ERROR
+    - F -> CRITICAL
+
+    Returns:
+        Uppercase log level string, defaults to WARNING if not set or invalid.
+    """
+    env_level = os.getenv(_LOG_LEVEL_ENV_VAR, os.getenv("LOG_LEVEL"))
+    # If empty or unset, default
+    if not env_level:
+        env_level = _DEFAULT_LOG_LEVEL
+    env_level = env_level.upper()
+
+    # Translate single letters to full level names
+    level_map = {
+        "D": "DEBUG",
+        "I": "INFO",
+        "W": "WARNING",
+        "E": "ERROR",
+        "F": "CRITICAL",
+    }
+
+    return level_map.get(env_level, env_level)
+
+
+def custom_timestamper(_: Any, __: Any, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """Add ISO UTC timestamp with milliseconds to event_dict['timestamp']."""
+    now = datetime.now()
+    event_dict["timestamp"] = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
+    return event_dict
+
+
+class MainConsoleRenderer:
+    """Custom console renderer for [L TIMESTAMP] message with color by level."""
+
+    LEVEL_MAP = {
+        "debug": ("D", "\033[90m"),  # grey
+        "info": ("I", "\033[32m"),  # green
+        "warning": ("W", "\033[33m"),  # yellow
+        "warn": ("W", "\033[33m"),  # yellow
+        "error": ("E", "\033[31m"),  # red
+        "critical": ("F", "\033[41m"),  # red background
+        "fatal": ("F", "\033[41m"),  # alias for critical
+    }
+    RESET = "\033[0m"
+
+    def __init__(self, colors: bool = True):
+        self.colors = colors
+
+    def __call__(
+        self, logger: Any, method_name: str, event_dict: Dict[str, Any]
+    ) -> str:
+        timestamp = event_dict.get("timestamp", "")
+        message = event_dict.get("event", "")
+        level = event_dict.get("level", method_name).lower()
+        letter, color = self.LEVEL_MAP.get(level, ("?", ""))
+        prefix = f"[{letter} {timestamp}]"
+        if self.colors and color:
+            prefix = f"{color}{prefix}{self.RESET}"
+
+        # Build the output with message and key-value pairs
+        output_parts = [prefix]
+
+        # Make the main message bold
+        if self.colors:
+            message = f"\033[1m{message}\033[0m"  # bold
+        output_parts.append(message)
+
+        # Add key-value pairs (excluding internal structlog keys)
+        kv_pairs = []
+        for key, value in event_dict.items():
+            if key not in ["timestamp", "event", "level"]:
+                # Pretty-format complex values (dict/list) as JSON on new lines
+                pretty_value = None
+                if isinstance(value, (dict, list)):
+                    try:
+                        pretty_value = json.dumps(
+                            value, ensure_ascii=False, sort_keys=True, indent=2
+                        )
+                    except Exception:
+                        pretty_value = pformat(value, width=100, compact=False)
+                elif not isinstance(value, (str, int, float, bool, type(None))):
+                    # Fall back to reasonably readable representation for other complex types
+                    pretty_value = pformat(value, width=100, compact=False)
+
+                rendered_value = (
+                    pretty_value if pretty_value is not None else str(value)
+                )
+
+                # If multiline, place value on a new line for readability
+                if "\n" in rendered_value:
+                    if self.colors:
+                        kv_pairs.append(
+                            f"\033[35m{key}\033[0m=\n\033[36m{rendered_value}\033[0m"
+                        )
+                    else:
+                        kv_pairs.append(f"{key}=\n{rendered_value}")
+                else:
+                    if self.colors:
+                        # Format: magenta key + equals + cyan value
+                        kv_pairs.append(
+                            f"\033[35m{key}\033[0m=\033[36m{rendered_value}\033[0m"
+                        )
+                    else:
+                        # No colors for plain output
+                        kv_pairs.append(f"{key}={rendered_value}")
+
+        if kv_pairs:
+            # If any kv is multiline, join with newlines; otherwise keep single line
+            if any("\n" in kv for kv in kv_pairs):
+                kv_text = "\n".join(kv_pairs)
+            else:
+                kv_text = " ".join(kv_pairs)
+            if self.colors:
+                kv_text = f"\033[35m{kv_text}{self.RESET}"  # magenta
+            output_parts.append(kv_text)
+
+        return " ".join(output_parts)
+
+
+def _configure_structlog() -> None:
+    """Configure structlog for both console and file output."""
+    log_dir = _ensure_log_dir()
+    log_file = log_dir / "main.log"
+    json_log_file = log_dir / "main.log.json"
+
+    shared_processors = [
+        structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+        redact_processor,
+        custom_timestamper,
+        structlog.stdlib.add_log_level,
+        structlog.processors.StackInfoRenderer(),
+        structlog.dev.set_exc_info,
+        structlog.processors.format_exc_info,
+        structlog.processors.UnicodeDecoder(),
+    ]
+
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "formatters": {
+                # Formatter for colored console output
+                "colored": {
+                    "()": "structlog.stdlib.ProcessorFormatter",
+                    "processors": [
+                        *shared_processors,
+                        MainConsoleRenderer(colors=True),
+                    ],
+                },
+                # Formatter for plain file output
+                "plain": {
+                    "()": "structlog.stdlib.ProcessorFormatter",
+                    "processors": [
+                        *shared_processors,
+                        MainConsoleRenderer(colors=False),
+                    ],
+                },
+                # Formatter for JSON file output
+                "json": {
+                    "()": "structlog.stdlib.ProcessorFormatter",
+                    "processors": [
+                        *shared_processors,
+                        structlog.processors.JSONRenderer(),
+                    ],
+                },
+            },
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "level": _get_env_log_level(),
+                    "formatter": "colored",
+                    "stream": sys.stderr,
+                },
+                "file": {
+                    "class": "logging.handlers.WatchedFileHandler",
+                    "level": "DEBUG",
+                    "filename": log_file,
+                    "formatter": "plain",
+                },
+                "json_file": {
+                    "class": "logging.handlers.WatchedFileHandler",
+                    "level": "DEBUG",
+                    "filename": json_log_file,
+                    "formatter": "json",
+                },
+            },
+            "loggers": {
+                "": {
+                    "handlers": ["console", "file", "json_file"],
+                    "level": "DEBUG",
+                    "propagate": True,
+                },
+            },
+        }
+    )
+
+    structlog.configure(
+        processors=[
+            structlog.stdlib.filter_by_level,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+    structlog.get_logger().debug("Logger configured", config=structlog.get_config())
+
+
+# Configure logging on module import
+_configure_structlog()
+
+
+def get_logger(name: str | None = None) -> Any:
+    """Get a configured structlog logger."""
+    return structlog.get_logger(name)
+
+
+# Export the root logger for convenience
+logger = get_logger()