crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/common/hierarchical_logger.py

@@ -1,4 +1,7 @@
 import logging
+import logging.config
+import yaml
+import os
 import sys
 import time
 from typing import Any, Callable, List, Optional
@@ -34,22 +37,41 @@ class HierarchicalLogger(object):
     def indent(self) -> str:
         return "  " * len(self.start_times)
 
-    def track_begin(self, x: Any) -> None:
-        self.logger.info(self.indent() + str(x) + " {")
+    def track_begin(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x) + " {", **kwargs)
         sys.stdout.flush()
         self.start_times.append(time.time())
 
-    def track_end(self) -> None:
+    def track_end(self, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
         t = time.time() - self.start_times.pop()
-        self.logger.info(self.indent() + "} [%s]" % (format_time(t)))
+        self.logger.info(self.indent() + "} [%s]" % (format_time(t)), **kwargs)
         sys.stdout.flush()
 
-    def log(self, x: Any) -> None:
-        self.logger.info(self.indent() + str(x))
+    def log(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.info(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
-    def warn(self, x: Any) -> None:
-        self.logger.warning(self.indent() + str(x))
+    def debug(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.debug(self.indent() + str(x), **kwargs)
+        sys.stdout.flush()
+
+    def warn(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.warning(self.indent() + str(x), **kwargs)
+        sys.stdout.flush()
+
+    def error(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.error(self.indent() + str(x), **kwargs)
+        sys.stdout.flush()
+
+    def exception(self, x: Any, **kwargs) -> None:
+        kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+        self.logger.exception(self.indent() + str(x), **kwargs)
         sys.stdout.flush()
 
 
@@ -69,23 +91,41 @@ singleton = HierarchicalLogger()
 # Exposed public methods
 
 
-def hlog(x: Any) -> None:
-    singleton.log(x)
+def hdebug(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.debug(x, **kwargs)
+
+
+def hlog(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.log(x, **kwargs)
 
 
-def hwarn(x: Any) -> None:
-    singleton.warn(x)
+def hwarn(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.warn(x, **kwargs)
+
+
+def herror(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.error(x, **kwargs)
+
+
+def hexception(x: Any, **kwargs) -> None:
+    kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+    singleton.exception(x, **kwargs)
 
 
 class htrack_block:
-    def __init__(self, x: Any) -> None:
+    def __init__(self, x: Any, stacklevel=1) -> None:
+        self._stacklevel = stacklevel + 1
         self.x = x
 
     def __enter__(self) -> None:
-        singleton.track_begin(self.x)
+        singleton.track_begin(self.x, stacklevel=self._stacklevel)
 
     def __exit__(self, tpe: Any, value: Any, callback: Any) -> None:
-        singleton.track_end()
+        singleton.track_end(stacklevel=self._stacklevel)
 
 
 class htrack:
@@ -116,34 +156,63 @@ class htrack:
                     description = description.replace("$" + k, str(v))
             else:
                 description = ""
-            with htrack_block(parent + fn.__name__ + description):
+            with htrack_block(parent + fn.__name__ + description, stacklevel=2):
                 return fn(*args, **kwargs)
 
         return wrapper
 
 
-def setup_default_logging():
+def setup_default_logging(config_path: Optional[str] = None):
     """
-    Setup a default logger to STDOUT for HELM via Python logging
-    """
-    formatter = ColoredFormatter(
-        "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S",
-        reset=True,
-        log_colors={
-            "DEBUG": "cyan",
-            "INFO": "green",
-            "WARNING": "yellow",
-            "ERROR": "red",
-            "CRITICAL": "red,bg_white",
-        },
-        secondary_log_colors={},
-        style="%",
-    )
+    Setup Python logging for HELM
 
+    Priority:
+    1. External config file (YAML or JSON).
+    2. ENV var LOG_LEVEL.
+    3. a default logger to STDOUT
+    """
     logger = logging.getLogger("helm")
-    logger.setLevel(logging.INFO)
     logger.propagate = False
+
+    if config_path and os.path.exists(config_path):
+        with open(config_path, "r") as f:
+            config = yaml.safe_load(f)
+        logging.config.dictConfig(config)
+        hdebug("setup custom HELM logging")
+        return
+
+    log_level = (os.getenv("HELM_LOG_LEVEL") or os.getenv("LOG_LEVEL") or "INFO").upper()
+    try:
+        logger.setLevel(getattr(logging, log_level))
+    except AttributeError:
+        logger.setLevel(logging.INFO)
+
+    # Set formatter
+    formatter: Optional[logging.Formatter] = None
+    if sys.stdout.isatty():
+        try:
+            formatter = ColoredFormatter(
+                "%(bold_black)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
+                datefmt="%Y-%m-%dT%H:%M:%S",
+                reset=True,
+                log_colors={
+                    "DEBUG": "cyan",
+                    "INFO": "green",
+                    "WARNING": "yellow",
+                    "ERROR": "red",
+                    "CRITICAL": "red,bg_white",
+                },
+                style="%",
+            )
+        except ImportError:
+            pass
+
+    if formatter is None:
+        # fallback
+        formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+
+    # Add default stdout handler
     handler = logging.StreamHandler(sys.stdout)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
+    hdebug("setup default HELM logging")

helm/common/object_spec.py

@@ -55,14 +55,23 @@ def inject_object_spec_args(
     This is loosely based on instance (constant) bindings and provider bindings in Guice dependency injection.
 
     Example:
-
-    class MyClass:
-        def __init__(a: int, b: int, c: int, d: int = 0):
-            pass
-
-    old_object_spec = ObjectSpec(class_name="MyClass", args={"a": 11})
-    new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
-    # new_object_spec is now ObjectSpec(class_name="MyClass", args={"a": 11, "b": 12, "c": 13})
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> import sys, types
+        >>> # Given a custom class with hashable arguments
+        >>> class MyClass:
+        ...     def __init__(a: int, b: int, c: int, d: int = 0):
+        ...         pass
+        >>> #
+        >>> # <boilerplate>: make a dummy module for MyClass to make this doctest exectuable
+        >>> sys.modules["my_module"] = type("MyModule", (types.ModuleType,), {"MyClass": MyClass})("my_module")
+        >>> # </boilerplate>
+        >>> #
+        >>> # Define new style and old style object specs
+        >>> old_object_spec = ObjectSpec(class_name="my_module.MyClass", args={"a": 11})
+        >>> new_object_spec = inject_object_spec_args(old_object_spec, {"b": 12}, {"c": lambda: 13})
+        >>> # new_object_spec is now
+        >>> print(new_object_spec)
+        ObjectSpec(class_name='my_module.MyClass', args={'a': 11, 'b': 12, 'c': 13})
     """
     cls = get_class_by_name(spec.class_name)
     init_signature = inspect.signature(cls.__init__)
@@ -93,6 +102,12 @@ def parse_object_spec(description: str) -> ObjectSpec:
         <class_name>:<key>=<value>,<key>=<value>
     Usually, the description is something that's succinct and can be typed on the command-line.
     Here, value defaults to string.
+
+    Example:
+        >>> from helm.common.object_spec import *  # NOQA
+        >>> description = 'mscoco:model=huggingface_stable-diffusion-v1-4'
+        >>> parse_object_spec(description)
+        ObjectSpec(class_name='mscoco', args={'model': 'huggingface_stable-diffusion-v1-4'})
     """
 
     def parse_arg(arg: str) -> Tuple[str, Any]:

helm/common/optional_dependencies.py

@@ -9,7 +9,7 @@ def handle_module_not_found_error(e: ModuleNotFoundError, suggestions: Optional[
     # TODO: Ask user to install more specific optional dependencies
     # e.g. crfm-helm[plots] or crfm-helm[server]
     suggested_commands = " or ".join(
-        [f"`pip install crfm-helm[{suggestion}]`" for suggestion in (suggestions or []) + ["all"]]
+        [f'`pip install "crfm-helm[{suggestion}]"`' for suggestion in (suggestions or []) + ["all"]]
     )
     raise OptionalDependencyNotInstalled(
         f"Optional dependency {e.name} is not installed. Please run {suggested_commands} to install it."

helm/common/test_general.py

@@ -1,5 +1,8 @@
 import shutil
 import os
+
+import pytest
+
 from helm.common.general import (
     ensure_file_downloaded,
     format_tags,
@@ -12,6 +15,7 @@ from helm.common.general import (
 
 
 def test_ensure_file_downloaded():
+    pytest.skip("Skipping download tests because these downloads are not reliable and may be throttled")
     ensure_file_downloaded("https://ftp.gnu.org/gnu/tar/tar-1.34.tar.gz", "test-tar", unpack=True, unpack_type="untar")
     assert os.path.isdir("test-tar")
     shutil.rmtree("test-tar")

helm/common/test_logging.py

@@ -0,0 +1,94 @@
+import sys
+import tempfile
+import textwrap
+import pathlib
+from helm.benchmark import run
+from typing import List, Optional
+
+
+class ArgvContext:
+    """
+    Helper to assign a temporary value to sys.argv and then restore it
+    """
+
+    def __init__(self, argv: Optional[List[str]]):
+        self.argv = argv
+        self._original_argv: Optional[List[str]] = None
+
+    def __enter__(self):
+        self._original_argv = sys.argv[:]
+        sys.argv = self.argv or []
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        assert self._original_argv is not None  # Satisfies mypy
+        sys.argv = self._original_argv
+
+
+def test_run_with_custom_logging_config():
+    # Setup temporary directory
+    with tempfile.TemporaryDirectory(prefix="helm_test_") as tmp_dir_str:
+        tmp_dir = pathlib.Path(tmp_dir_str)
+        log_path = tmp_dir / "test.log"
+        log_config_path = tmp_dir / "test_config.yaml"
+
+        # Write custom YAML log config to file
+        log_config_text = textwrap.dedent(
+            f"""
+            version: 1
+            disable_existing_loggers: false
+            formatters:
+              simple:
+                datefmt: '%Y-%m-%dT%H:%M:%S'
+                format: '%(asctime)s %(levelname)s %(name)s %(message)s'
+            handlers:
+              file:
+                class: logging.FileHandler
+                filename: {log_path}
+                formatter: simple
+                level: DEBUG
+                mode: w
+            loggers:
+              helm:
+                handlers:
+                - file
+                level: DEBUG
+                propagate: false
+            """
+        ).strip()
+
+        log_config_path.write_text(log_config_text)
+
+        # Simulate command-line arguments
+        argv = [
+            "run.py",  # Fake script name
+            "--run-entries",
+            "mmlu:subject=philosophy,model=openai/gpt2",
+            "-m",
+            "1",
+            "--suite",
+            "my-suite",
+            "--dry-run",
+            "--log-config",
+            str(log_config_path),
+        ]
+
+        # Call main
+        with ArgvContext(argv):
+            run.main()
+
+        # Check log file contents
+        assert log_path.exists(), "Log file was not created"
+        log_contents = log_path.read_text()
+
+        # Test that log file was written to disk as requested
+        print("Log Contents")
+        print("------------")
+        print(log_contents)
+
+        assert (
+            "mscoco" in log_contents or "huggingface" in log_contents or "dry-run" in log_contents
+        ), "Expected log content not found in log file:\n"
+
+
+if __name__ == "__main__":
+    test_run_with_custom_logging_config()