hydra-nevergrad-sweeper 1.3.0.dev0__tar.gz → 1.4.0.dev4__tar.gz

hydra_nevergrad_sweeper-1.4.0.dev4/MANIFEST.in

@@ -0,0 +1,3 @@
+global-exclude *.pyc
+global-exclude __pycache__
+recursive-include hydra_plugins/* *.yaml py.typed

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/PKG-INFO

@@ -1,19 +1,32 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: hydra-nevergrad-sweeper
-Version: 1.3.0.dev0
+Version: 1.4.0.dev4
 Summary: Hydra Nevergrad Sweeper plugin
 Home-page: https://github.com/facebookresearch/hydra/
 Author: Jeremy Rapin, Omry Yadan, Jieru Hu
 Author-email: jrapin@fb.com, omry@fb.com, jieru@fb.com
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
+License: MIT
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 4 - Beta
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+Requires-Dist: hydra-core>=1.1.0.dev7
+Requires-Dist: nevergrad>=1.0.12
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Hydra Nevergrad Sweeper plugin
 

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/hydra_nevergrad_sweeper.egg-info/PKG-INFO

@@ -1,19 +1,32 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: hydra-nevergrad-sweeper
-Version: 1.3.0.dev0
+Version: 1.4.0.dev4
 Summary: Hydra Nevergrad Sweeper plugin
 Home-page: https://github.com/facebookresearch/hydra/
 Author: Jeremy Rapin, Omry Yadan, Jieru Hu
 Author-email: jrapin@fb.com, omry@fb.com, jieru@fb.com
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
+License: MIT
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 4 - Beta
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+Requires-Dist: hydra-core>=1.1.0.dev7
+Requires-Dist: nevergrad>=1.0.12
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Hydra Nevergrad Sweeper plugin
 

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/hydra_nevergrad_sweeper.egg-info/SOURCES.txt

@@ -10,4 +10,6 @@ hydra_nevergrad_sweeper.egg-info/top_level.txt
 hydra_plugins/hydra_nevergrad_sweeper/__init__.py
 hydra_plugins/hydra_nevergrad_sweeper/_impl.py
 hydra_plugins/hydra_nevergrad_sweeper/config.py
-hydra_plugins/hydra_nevergrad_sweeper/nevergrad_sweeper.py
+hydra_plugins/hydra_nevergrad_sweeper/nevergrad_sweeper.py
+hydra_plugins/hydra_nevergrad_sweeper/py.typed
+tests/test_nevergrad_sweeper_plugin.py

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/hydra_nevergrad_sweeper.egg-info/requires.txt

@@ -1,2 +1,2 @@
 hydra-core>=1.1.0.dev7
-nevergrad>=0.4.3.post9
+nevergrad>=1.0.12

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/hydra_plugins/hydra_nevergrad_sweeper/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 
-__version__ = "1.3.0.dev0"
+__version__ = "1.4.0.dev4"

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/hydra_plugins/hydra_nevergrad_sweeper/_impl.py

@@ -33,7 +33,7 @@ log = logging.getLogger(__name__)
 
 
 def create_nevergrad_param_from_config(
-    config: Union[MutableSequence[Any], MutableMapping[str, Any]]
+    config: Union[MutableSequence[Any], MutableMapping[str, Any]],
 ) -> Any:
     if isinstance(config, MutableSequence):
         if isinstance(config, ListConfig):
@@ -60,7 +60,7 @@ def create_nevergrad_param_from_config(
 def create_nevergrad_parameter_from_override(override: Override) -> Any:
     val = override.value()
     if not override.is_sweep_override():
-        return val
+        return override.get_value_element_as_str()
     if override.is_choice_sweep():
         assert isinstance(val, ChoiceSweep)
         vals = [x for x in override.sweep_iterator(transformer=Transformer.encode)]
@@ -76,7 +76,7 @@ def create_nevergrad_parameter_from_override(override: Override) -> Any:
         if "log" in val.tags:
             scalar = ng.p.Log(lower=val.start, upper=val.end)
         else:
-            scalar = ng.p.Scalar(lower=val.start, upper=val.end)  # type: ignore
+            scalar = ng.p.Scalar(lower=val.start, upper=val.end)
         if isinstance(val.start, int):
             scalar.set_integer_casting()
         return scalar
@@ -117,7 +117,6 @@ class NevergradSweeperImpl(Sweeper):
         )
 
     def sweep(self, arguments: List[str]) -> None:
-
         assert self.config is not None
         assert self.launcher is not None
         assert self.job_idx is not None
@@ -130,9 +129,9 @@ class NevergradSweeperImpl(Sweeper):
         parsed = parser.parse_overrides(arguments)
 
         for override in parsed:
-            params[
-                override.get_key_element()
-            ] = create_nevergrad_parameter_from_override(override)
+            params[override.get_key_element()] = (
+                create_nevergrad_parameter_from_override(override)
+            )
 
         parametrization = ng.p.Dict(**params)
         parametrization.function.deterministic = not self.opt_config.noisy
@@ -147,7 +146,11 @@ class NevergradSweeperImpl(Sweeper):
         )
         log.info(f"with parametrization {parametrization}")
         log.info(f"Sweep output dir: {self.config.hydra.sweep.dir}")
-        optimizer = ng.optimizers.registry[opt](parametrization, remaining_budget, nw)
+        optimizer = ng.optimizers.registry[opt](
+            parametrization=parametrization,
+            budget=remaining_budget,
+            num_workers=nw,
+        )
         # loop!
         all_returns: List[Any] = []
         best: Tuple[float, ng.p.Parameter] = (float("inf"), parametrization)

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/hydra_plugins/hydra_nevergrad_sweeper/config.py

@@ -35,7 +35,6 @@ class ScalarConfigSpec:
 
 @dataclass
 class OptimConf:
-
     # name of the Nevergrad optimizer to use. Here is a sample:
     #   - "OnePlusOne" extremely simple and robust, especially at low budget, but
     #     tends to converge early.
@@ -74,7 +73,7 @@ class NevergradSweeperConf:
     )
 
     # configuration of the optimizer
-    optim: OptimConf = OptimConf()
+    optim: OptimConf = field(default_factory=OptimConf)
 
     # default parametrization of the search space
     # can be specified:

{hydra-nevergrad-sweeper-1.3.0.dev0 → hydra_nevergrad_sweeper-1.4.0.dev4}/setup.py

@@ -11,23 +11,24 @@ setup(
     author="Jeremy Rapin, Omry Yadan, Jieru Hu",
     author_email="jrapin@fb.com, omry@fb.com, jieru@fb.com",
     description="Hydra Nevergrad Sweeper plugin",
+    license="MIT",
     long_description=(Path(__file__).parent / "README.md").read_text(),
     long_description_content_type="text/markdown",
     url="https://github.com/facebookresearch/hydra/",
     packages=find_namespace_packages(include=["hydra_plugins.*"]),
     classifiers=[
-        "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
         "Operating System :: OS Independent",
         "Development Status :: 4 - Beta",
     ],
+    python_requires=">=3.10",
     install_requires=[
         "hydra-core>=1.1.0.dev7",
-        "nevergrad>=0.4.3.post9",
+        "nevergrad>=1.0.12",
     ],
     include_package_data=True,
 )

hydra_nevergrad_sweeper-1.4.0.dev4/tests/test_nevergrad_sweeper_plugin.py

@@ -0,0 +1,200 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import sys
+import warnings
+from pathlib import Path
+from typing import Any
+
+import nevergrad as ng
+from hydra.core.override_parser.overrides_parser import OverridesParser
+from hydra.core.plugins import Plugins
+from hydra.plugins.sweeper import Sweeper
+from hydra.test_utils.test_utils import (
+    TSweepRunner,
+    chdir_plugin_root,
+    run_process,
+    run_python_script,
+)
+from omegaconf import DictConfig, OmegaConf
+from pytest import mark
+
+from hydra_plugins.hydra_nevergrad_sweeper import _impl
+from hydra_plugins.hydra_nevergrad_sweeper.nevergrad_sweeper import NevergradSweeper
+
+chdir_plugin_root()
+
+
+def test_discovery() -> None:
+    assert NevergradSweeper.__name__ in [
+        x.__name__ for x in Plugins.instance().discover(Sweeper)
+    ]
+
+
+def assert_ng_param_equals(expected: Any, actual: Any) -> None:
+    assert type(expected) is type(actual)
+    if isinstance(actual, ng.p.Choice) or isinstance(actual, ng.p.TransitionChoice):
+        assert sorted(expected.choices.value) == sorted(actual.choices.value)
+    elif isinstance(actual, ng.p.Log) or isinstance(actual, ng.p.Scalar):
+        assert expected.bounds == actual.bounds
+        assert expected.integer == actual.integer
+    elif isinstance(actual, str):
+        assert expected == actual
+    else:
+        assert False, f"Unexpected type: {type(actual)}"
+
+
+def get_scalar_with_integer_bounds(lower: int, upper: int, type: Any) -> ng.p.Scalar:
+    scalar = type(lower=lower, upper=upper)
+    scalar.set_integer_casting()
+    assert isinstance(scalar, ng.p.Scalar)
+    return scalar
+
+
+@mark.parametrize(
+    "input, expected",
+    [
+        ([1, 2, 3], ng.p.Choice([1, 2, 3])),
+        (["1", "2", "3"], ng.p.Choice(["1", "2", "3"])),
+        ({"lower": 1, "upper": 12, "log": True}, ng.p.Log(lower=1, upper=12)),
+        ({"lower": 1, "upper": 12}, ng.p.Scalar(lower=1, upper=12)),
+        (
+            {"lower": 1, "upper": 12, "integer": True},
+            get_scalar_with_integer_bounds(1, 12, ng.p.Scalar),
+        ),
+        (
+            {"lower": 1, "upper": 12, "log": True, "integer": True},
+            get_scalar_with_integer_bounds(1, 12, ng.p.Log),
+        ),
+    ],
+)
+def test_create_nevergrad_parameter_from_config(
+    input: Any,
+    expected: Any,
+) -> None:
+    actual = _impl.create_nevergrad_param_from_config(input)
+    assert_ng_param_equals(expected, actual)
+
+
+@mark.parametrize(
+    "input, expected",
+    [
+        ("key=choice(1,2)", ng.p.Choice([1, 2])),
+        ("key=choice('hello','world')", ng.p.Choice(["hello", "world"])),
+        ("key=tag(ordered, choice(1,2,3))", ng.p.TransitionChoice([1, 2, 3])),
+        (
+            "key=tag(ordered, choice('hello','world', 'nevergrad'))",
+            ng.p.TransitionChoice(["hello", "world", "nevergrad"]),
+        ),
+        ("key=range(1,3)", ng.p.Choice([1, 2])),
+        ("key=shuffle(range(1,3))", ng.p.Choice([1, 2])),
+        ("key=range(1,5)", ng.p.Choice([1, 2, 3, 4])),
+        ("key=float(range(1,5))", ng.p.Choice([1.0, 2.0, 3.0, 4.0])),
+        (
+            "key=int(interval(1,12))",
+            get_scalar_with_integer_bounds(lower=1, upper=12, type=ng.p.Scalar),
+        ),
+        ("key=tag(log, interval(1,12))", ng.p.Log(lower=1, upper=12)),
+        (
+            "key=tag(log, int(interval(1,12)))",
+            get_scalar_with_integer_bounds(lower=1, upper=12, type=ng.p.Log),
+        ),
+        ("key=abc", "abc"),
+        ("key='a&c'", "'a&c'"),
+        ("key='${f:${x}}'", "'${f:${x}}'"),
+        (r"key=$\{f\:$\{x\}\}", r"$\{f\:$\{x\}\}"),
+        ("key=0", "0"),
+        ("key=true", "True"),
+        ("key=null", "null"),
+    ],
+)
+def test_create_nevergrad_parameter_from_override(
+    input: Any,
+    expected: Any,
+) -> None:
+    parser = OverridesParser.create()
+    parsed = parser.parse_overrides([input])[0]
+    param = _impl.create_nevergrad_parameter_from_override(parsed)
+    assert_ng_param_equals(param, expected)
+
+
+def test_launched_jobs(hydra_sweep_runner: TSweepRunner) -> None:
+    budget = 8
+    sweep = hydra_sweep_runner(
+        calling_file=None,
+        calling_module="hydra.test_utils.a_module",
+        config_path="configs",
+        config_name="compose.yaml",
+        task_function=None,
+        overrides=[
+            "hydra/sweeper=nevergrad",
+            "hydra/launcher=basic",
+            f"hydra.sweeper.optim.budget={budget}",  # small budget to test fast
+            "hydra.sweeper.optim.num_workers=3",
+            "foo=1,2",
+            "bar=4:8",
+        ],
+    )
+    with warnings.catch_warnings():  # ignore specific warnings raised by sweep:
+        warnings.filterwarnings(
+            "ignore", category=ng.common.errors.InefficientSettingsWarning
+        )
+        warnings.filterwarnings(
+            "ignore",
+            category=UserWarning,
+            message=r"Could not import matplotlib\.pyplot",
+        )
+        with sweep:
+            assert sweep.returns is None
+
+
+@mark.parametrize("with_commandline", (True, False))
+def test_nevergrad_example(with_commandline: bool, tmpdir: Path) -> None:
+    budget = 32 if with_commandline else 1  # make a full test only once (faster)
+    cmd = [
+        "example/my_app.py",
+        "-m",
+        "hydra.sweep.dir=" + str(tmpdir),
+        "hydra.job.chdir=True",
+        f"hydra.sweeper.optim.budget={budget}",  # small budget to test fast
+        f"hydra.sweeper.optim.num_workers={min(8, budget)}",
+        "hydra.sweeper.optim.seed=12",  # avoid random failures
+    ]
+    if with_commandline:
+        cmd += [
+            "db=mnist,cifar",
+            "batch_size=4,8,12,16",
+            "lr=tag(log, interval(0.001, 1.0))",
+            "dropout=interval(0,1)",
+        ]
+    run_python_script(cmd)
+    returns = OmegaConf.load(f"{tmpdir}/optimization_results.yaml")
+    assert isinstance(returns, DictConfig)
+    assert returns.name == "nevergrad"
+    assert len(returns) == 3
+    best_parameters = returns.best_evaluated_params
+    assert not best_parameters.dropout.is_integer()
+    if budget > 1:
+        assert best_parameters.batch_size == 4  # this argument should be easy to find
+    # check that all job folders are created
+    last_job = max(int(fp.name) for fp in Path(tmpdir).iterdir() if fp.name.isdigit())
+    assert last_job == budget - 1
+
+
+@mark.parametrize("max_failure_rate", (0.5, 1.0))
+def test_failure_rate(max_failure_rate: float, tmpdir: Path) -> None:
+    cmd = [
+        sys.executable,
+        "example/my_app.py",
+        "-m",
+        f"hydra.sweep.dir={tmpdir}",
+        "hydra.sweeper.optim.budget=2",  # small budget to test fast
+        "hydra.sweeper.optim.num_workers=2",
+        f"hydra.sweeper.optim.max_failure_rate={max_failure_rate}",
+        "error=true",
+    ]
+    out, err = run_process(cmd, print_error=False, raise_exception=False)
+    assert "Returning infinity for failed experiment" in out
+    error_string = "RuntimeError: cfg.error is True"
+    if max_failure_rate < 1.0:
+        assert error_string in err
+    else:
+        assert error_string not in err

hydra-nevergrad-sweeper-1.3.0.dev0/MANIFEST.in

@@ -1,3 +0,0 @@
-global-exclude *.pyc
-global-exclude __pycache__
-recursive-include hydra_plugins/* *.yaml