hydra-ax-sweeper 1.4.0.dev4__tar.gz → 1.4.0.dev6__tar.gz

{hydra_ax_sweeper-1.4.0.dev4 → hydra_ax_sweeper-1.4.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydra-ax-sweeper
-Version: 1.4.0.dev4
+Version: 1.4.0.dev6
 Summary: Hydra Ax Sweeper plugin
 Home-page: https://github.com/facebookresearch/hydra/
 Author: Omry Yadan, Shagun Sodhani

{hydra_ax_sweeper-1.4.0.dev4 → hydra_ax_sweeper-1.4.0.dev6}/hydra_ax_sweeper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydra-ax-sweeper
-Version: 1.4.0.dev4
+Version: 1.4.0.dev6
 Summary: Hydra Ax Sweeper plugin
 Home-page: https://github.com/facebookresearch/hydra/
 Author: Omry Yadan, Shagun Sodhani

{hydra_ax_sweeper-1.4.0.dev4 → hydra_ax_sweeper-1.4.0.dev6}/hydra_plugins/hydra_ax_sweeper/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 
-__version__ = "1.4.0.dev4"
+__version__ = "1.4.0.dev6"

{hydra_ax_sweeper-1.4.0.dev4 → hydra_ax_sweeper-1.4.0.dev6}/hydra_plugins/hydra_ax_sweeper/_core.py

@@ -34,6 +34,8 @@ log = logging.getLogger(__name__)
 AxRangeParameterType = Literal["float", "int"]
 AxChoiceParameterType = Literal["float", "int", "str", "bool"]
 AxParameterConfig = Union[RangeParameterConfig, ChoiceParameterConfig]
+AxMetricValue = Union[float, Tuple[float, float]]
+AxRawData = Mapping[str, AxMetricValue]
 
 
 @dataclass
@@ -126,6 +128,30 @@ def create_ax_parameter_config(param: Dict[Any, Any]) -> AxParameterConfig:
     )
 
 
+def create_ax_raw_data(value: Any, objective_name: str, is_noisy: bool) -> AxRawData:
+    def normalize_metric(metric_value: Any) -> AxMetricValue:
+        assert isinstance(metric_value, (int, float, tuple))
+        if isinstance(metric_value, (int, float)):
+            mean = float(metric_value)
+            return mean if is_noisy else (mean, 0.0)
+
+        assert len(metric_value) == 2
+        mean, sem = metric_value
+        assert isinstance(mean, (int, float))
+        assert sem is None or isinstance(sem, (int, float))
+        return (float(mean), float("nan") if sem is None else float(sem))
+
+    assert isinstance(value, (int, float, tuple, dict))
+    if isinstance(value, dict):
+        raw_data: Dict[str, AxMetricValue] = {}
+        for metric_name, metric_value in value.items():
+            assert isinstance(metric_name, str)
+            raw_data[metric_name] = normalize_metric(metric_value)
+        return raw_data
+
+    return {objective_name: normalize_metric(value)}
+
+
 def get_one_batch_of_trials(
     ax_client: Client,
     num_max_trials_to_do: int,
@@ -258,19 +284,13 @@ class CoreAxSweeper(Sweeper):
                 # Alternatively, the task function can return a dict whose values
                 # represent multiple metrics, where each key is the name of the metric
                 # and the item can be an int, float or tuple.
-                assert isinstance(val, (int, float, tuple, dict))
-                # is_noisy specifies how Ax should behave when not given an error value.
-                # if true (default), the error of each measurement is inferred by Ax.
-                # if false, the error of each measurement is set to 0.
-                if isinstance(val, (int, float)):
-                    if self.is_noisy:
-                        val = (val, None)  # specify unknown noise
-                    else:
-                        val = (val, 0)  # specify no noise
-                if isinstance(val, tuple):
-                    val = {self.experiment.objective_name: val}
+                # is_noisy specifies how Ax should behave when not given an
+                # error value: true means unknown error, false means zero error.
+                raw_data = create_ax_raw_data(
+                    val, self.experiment.objective_name, self.is_noisy
+                )
                 ax_client.complete_trial(
-                    trial_index=batch[idx].trial_index, raw_data=val
+                    trial_index=batch[idx].trial_index, raw_data=raw_data
                 )
 
     def get_best_point(

{hydra_ax_sweeper-1.4.0.dev4 → hydra_ax_sweeper-1.4.0.dev6}/tests/test_ax_sweeper_plugin.py

@@ -316,6 +316,23 @@ def test_command_line_log_interval_configures_ax_log_range() -> None:
     assert ax_parameter.scaling == "log"
 
 
+def test_create_ax_raw_data() -> None:
+    from hydra_plugins.hydra_ax_sweeper._core import create_ax_raw_data
+
+    assert create_ax_raw_data(1, "loss", is_noisy=True) == {"loss": 1.0}
+    assert create_ax_raw_data(1, "loss", is_noisy=False) == {"loss": (1.0, 0.0)}
+    assert create_ax_raw_data({"loss": 1, "accuracy": (0.5, 0.1)}, "loss", True) == {
+        "loss": 1.0,
+        "accuracy": (0.5, 0.1),
+    }
+
+    raw_data = create_ax_raw_data((1, None), "loss", is_noisy=True)
+    loss = raw_data["loss"]
+    assert isinstance(loss, tuple)
+    sem = loss[1]
+    assert math.isnan(sem)
+
+
 def test_ax_logging_from_hydra_app(tmpdir: Path) -> None:
     cmd = [
         "tests/apps/polynomial.py",