hydra-core 1.4.0.dev3__tar.gz → 1.4.0.dev5__tar.gz

{hydra_core-1.4.0.dev3/hydra_core.egg-info → hydra_core-1.4.0.dev5}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: hydra-core
-Version: 1.4.0.dev3
+Version: 1.4.0.dev5
 Summary: A framework for elegantly configuring complex applications
 Home-page: https://github.com/facebookresearch/hydra
 Author: Omry Yadan
 Author-email: omry@fb.com
 License: MIT
 Keywords: command-line configuration yaml tab-completion
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -20,7 +19,7 @@ Classifier: Operating System :: Microsoft :: Windows
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omegaconf>=2.4.0.dev11
+Requires-Dist: omegaconf>=2.4.0.dev12
 Requires-Dist: packaging
 Dynamic: author
 Dynamic: author-email
@@ -54,11 +53,7 @@ Dynamic: summary
   <i>A framework for elegantly configuring complex applications.</i>
 </p>
 <p align="center">
-  <i>Check the <a href="https://hydra.cc/">website</a> for more information,<br>
-  or click the thumbnail below for a one-minute video introduction to Hydra.</i>
-</p>
-<p align="center">
-  <a href="https://www.youtube.com/watch?v=Slc3gRQpnBI" target="_blank"><img src="https://raw.githubusercontent.com/facebookresearch/hydra/main/website/static/img/Hydra-Readme-video-thumbnail.jpg" alt="1 minute overview" width="240" height="180" border="10" /></a>
+  <i>Check the <a href="https://hydra.cc/">website</a> for more information.</i>
 </p>
 
 ----------------------
@@ -68,8 +63,10 @@ Dynamic: summary
 #### Development
 
 **Hydra 1.4** is the current development version of Hydra.
+- Hydra 1.4 is coming soon. Until the stable release is available, you can
+  install Hydra from development releases on PyPI.
 - [Documentation](https://hydra.cc/docs/intro/)
-- Installation from source : `pip install git+https://github.com/facebookresearch/hydra.git`
+- Installation: `pip install --pre --upgrade hydra-core`
 - Supported Python versions: 3.10 through 3.14.
 
 #### Stable

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/README.md

@@ -17,11 +17,7 @@
   <i>A framework for elegantly configuring complex applications.</i>
 </p>
 <p align="center">
-  <i>Check the <a href="https://hydra.cc/">website</a> for more information,<br>
-  or click the thumbnail below for a one-minute video introduction to Hydra.</i>
-</p>
-<p align="center">
-  <a href="https://www.youtube.com/watch?v=Slc3gRQpnBI" target="_blank"><img src="https://raw.githubusercontent.com/facebookresearch/hydra/main/website/static/img/Hydra-Readme-video-thumbnail.jpg" alt="1 minute overview" width="240" height="180" border="10" /></a>
+  <i>Check the <a href="https://hydra.cc/">website</a> for more information.</i>
 </p>
 
 ----------------------
@@ -31,8 +27,10 @@
 #### Development
 
 **Hydra 1.4** is the current development version of Hydra.
+- Hydra 1.4 is coming soon. Until the stable release is available, you can
+  install Hydra from development releases on PyPI.
 - [Documentation](https://hydra.cc/docs/intro/)
-- Installation from source : `pip install git+https://github.com/facebookresearch/hydra.git`
+- Installation: `pip install --pre --upgrade hydra-core`
 - Supported Python versions: 3.10 through 3.14.
 
 #### Stable

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/hydra/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 
 # Source of truth for Hydra's version
-__version__ = "1.4.0.dev3"
+__version__ = "1.4.0.dev5"
 from hydra import utils
 from hydra.errors import MissingConfigException
 from hydra.main import main

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/hydra/_internal/config_loader_impl.py

@@ -1,7 +1,6 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import copy
 import os
-import re
 import sys
 import warnings
 from textwrap import dedent
@@ -18,6 +17,7 @@ from omegaconf import (
 from omegaconf.errors import (
     ConfigAttributeError,
     ConfigKeyError,
+    MissingMandatoryValue,
     OmegaConfBaseException,
 )
 
@@ -279,13 +279,11 @@ class ConfigLoaderImpl(ConfigLoader):
 
         # Apply command line overrides after enabling strict flag
         ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
-        app_overrides = []
         for override in parsed_overrides:
             if override.is_hydra_override():
                 cfg.hydra.overrides.hydra.append(override.input_line)
             else:
                 cfg.hydra.overrides.task.append(override.input_line)
-                app_overrides.append(override)
 
         with open_dict(cfg.hydra):
             cfg.hydra.runtime.choices.update(defaults_list.overrides.known_choices)
@@ -304,12 +302,6 @@ class ConfigLoaderImpl(ConfigLoader):
         if "name" not in cfg.hydra.job:
             cfg.hydra.job.name = JobRuntime().get("name")
 
-        cfg.hydra.job.override_dirname = get_overrides_dirname(
-            overrides=app_overrides,
-            kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
-            item_sep=cfg.hydra.job.config.override_dirname.item_sep,
-            exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys,
-        )
         cfg.hydra.job.config_name = config_name
 
         return cfg
@@ -326,6 +318,10 @@ class ConfigLoaderImpl(ConfigLoader):
             overrides=overrides,
             run_mode=RunMode.RUN,
         )
+        ConfigLoaderImpl._ensure_sweep_config_controller_unchanged(
+            master_config=master_config,
+            sweep_config=sweep_config,
+        )
 
         # Copy old config cache to ensure we get the same resolved values (for things
         # like timestamps etc). Since `oc.env` does not cache environment variables
@@ -334,6 +330,36 @@ class ConfigLoaderImpl(ConfigLoader):
 
         return sweep_config
 
+    @staticmethod
+    def _ensure_sweep_config_controller_unchanged(
+        master_config: DictConfig, sweep_config: DictConfig
+    ) -> None:
+        controller_groups = (
+            ("hydra/launcher", "launcher"),
+            ("hydra/sweeper", "sweeper"),
+        )
+        master_choices = master_config.hydra.runtime.choices
+        sweep_choices = sweep_config.hydra.runtime.choices
+        for group, node in controller_groups:
+            master_choice = master_choices.get(group)
+            sweep_choice = sweep_choices.get(group)
+            if master_choice != sweep_choice:
+                # Bandit mistakes this user-facing message for a SQL snippet.
+                raise ConfigCompositionException(
+                    f"'{group}' must be selected before the sweep starts, "  # nosec B608
+                    f"but a swept job config changed it from "
+                    f"'{master_choice}' to '{sweep_choice}'. Select it in "
+                    "the primary config or from the command line."
+                )
+            if not OmegaConf.structural_equality(
+                master_config.hydra[node], sweep_config.hydra[node]
+            ):
+                raise ConfigCompositionException(
+                    f"'hydra.{node}' must be configured before the sweep starts, "
+                    "but a swept job config changed it. Configure it in the "
+                    "primary config or from the command line."
+                )
+
     def get_search_path(self) -> ConfigSearchPath:
         return self.config_search_path
 
@@ -351,29 +377,62 @@ class ConfigLoaderImpl(ConfigLoader):
             value = override.value()
             try:
                 if override.is_delete():
-                    config_val = OmegaConf.select(cfg, key, throw_on_missing=False)
-                    if config_val is None:
+                    config_val_not_found = object()
+                    config_val_missing = object()
+                    last_dot = key.rfind(".")
+                    parent: Container = cfg
+                    parent_missing = False
+                    if last_dot != -1:
+                        parent_key = key[0:last_dot]
+                        selected_parent = OmegaConf.select(
+                            cfg,
+                            parent_key,
+                            default=config_val_not_found,
+                            throw_on_missing=False,
+                        )
+                        if isinstance(selected_parent, Container):
+                            parent = selected_parent
+                        else:
+                            parent_missing = True
+
+                    try:
+                        config_val = OmegaConf.select(
+                            cfg,
+                            key,
+                            default=config_val_not_found,
+                            throw_on_missing=True,
+                        )
+                    except MissingMandatoryValue:
+                        config_val = config_val_missing
+
+                    if parent_missing or config_val is config_val_not_found:
+                        # Bandit mistakes this user-facing message for a SQL snippet.
                         raise ConfigCompositionException(
-                            f"Could not delete from config. '{override.key_or_group}'"
+                            f"Could not delete from config. '{override.key_or_group}'"  # nosec B608
                             " does not exist."
                         )
-                    elif value is not None and value != config_val:
+                    config_val_for_match = (
+                        "???" if config_val is config_val_missing else config_val
+                    )
+                    if (
+                        override.value_type is not None
+                        and value != config_val_for_match
+                    ):
+                        # Bandit mistakes this user-facing message for a SQL snippet.
                         raise ConfigCompositionException(
-                            "Could not delete from config. The value of"
-                            f" '{override.key_or_group}' is {config_val} and not"
+                            "Could not delete from config. The value of"  # nosec B608
+                            f" '{override.key_or_group}' is {config_val_for_match} and not"
                             f" {value}."
                         )
 
-                    last_dot = key.rfind(".")
                     with open_dict(cfg):
                         if last_dot == -1:
                             del cfg[key]
                         else:
-                            node = OmegaConf.select(cfg, key[0:last_dot])
                             node_key: Union[str, int] = key[last_dot + 1 :]
-                            if isinstance(node, ListConfig):
+                            if isinstance(parent, ListConfig):
                                 node_key = int(node_key)
-                            del node[node_key]
+                            del parent[node_key]
 
                 elif override.is_add():
                     if OmegaConf.select(
@@ -590,18 +649,3 @@ class ConfigLoaderImpl(ConfigLoader):
             skip_missing=run_mode == RunMode.MULTIRUN,
         )
         return defaults_list
-
-
-def get_overrides_dirname(
-    overrides: List[Override], exclude_keys: List[str], item_sep: str, kv_sep: str
-) -> str:
-    lines = []
-    for override in overrides:
-        if override.key_or_group not in exclude_keys:
-            line = override.input_line
-            assert line is not None
-            lines.append(line)
-
-    lines.sort()
-    ret = re.sub(pattern="[=]", repl=kv_sep, string=item_sep.join(lines))
-    return ret

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/hydra/_internal/defaults_list.py

@@ -72,25 +72,30 @@ class Overrides:
             if override.is_sweep_override():
                 continue
             is_group = repo.group_exists(override.key_or_group)
+            is_config = repo.config_exists(override.key_or_group)
             value = override.value()
             is_dict = isinstance(override.value(), dict)
-            if is_dict or not is_group:
+            if override.is_delete() and (is_group or is_config):
+                key = override.get_key_element()[1:]
+                if is_group:
+                    if value is not None and not isinstance(value, str):
+                        raise ValueError(
+                            f"Config group override deletion value must be a string : {override}"
+                        )
+                    self.deletions[key] = Deletion(name=value)
+                else:
+                    if value is not None:
+                        raise ValueError(
+                            f"Config path deletion does not support a value : {override}"
+                        )
+                    self.deletions[key] = Deletion(name=None)
+            elif is_dict or not is_group:
                 self.config_overrides.append(override)
             elif override.is_force_add():
                 # This could probably be made to work if there is a compelling use case.
                 raise ConfigCompositionException(
                     f"force-add of config groups is not supported: '{override.input_line}'"
                 )
-            elif override.is_delete():
-                key = override.get_key_element()[1:]
-                value = override.value()
-                if value is not None and not isinstance(value, str):
-                    raise ValueError(
-                        f"Config group override deletion value must be a string : {override}"
-                    )
-
-                self.deletions[key] = Deletion(name=value)
-
             elif not isinstance(value, (str, list)):
                 raise ValueError(
                     f"Config group override must be a string or a list. Got {type(value).__name__}"
@@ -193,22 +198,28 @@ class Overrides:
             self.known_choices_per_group[group].add(key)
 
     def is_deleted(self, default: InputDefault) -> bool:
-        if not isinstance(default, GroupDefault):
-            return False
-        key = default.get_override_key()
-        if key in self.deletions:
-            deletion = self.deletions[key]
-            if deletion.name is None:
-                return True
-            else:
-                return deletion.name == default.get_name()
+        if isinstance(default, GroupDefault):
+            key = default.get_override_key()
+            if key in self.deletions:
+                deletion = self.deletions[key]
+                if deletion.name is None:
+                    return True
+                else:
+                    return deletion.name == default.get_name()
+        elif isinstance(default, ConfigDefault):
+            key = default.get_config_path()
+            if key in self.deletions:
+                return self.deletions[key].name is None
         return False
 
     def delete(self, default: InputDefault) -> None:
-        assert isinstance(default, GroupDefault)
-        default.deleted = True
-
-        key = default.get_override_key()
+        if isinstance(default, GroupDefault):
+            default.deleted = True
+            key = default.get_override_key()
+        else:
+            assert isinstance(default, ConfigDefault)
+            default.deleted = True
+            key = default.get_config_path()
         self.deletions[key].used = True
 
 
@@ -541,14 +552,23 @@ def _create_defaults_tree_impl(
                         )
 
                     assert d.group is not None
-                    node = ConfigDefault(
-                        path=d.group + "/" + item,
-                        package=d.package,
-                        optional=d.is_optional(),
-                    )
-                    node.update_parent(
-                        parent.get_group_path(), parent.get_final_package()
-                    )
+                    if d.is_external_append():
+                        node = ConfigDefault(
+                            path=f"{d.get_group_path()}/{item}",
+                            package=d.package,
+                            optional=d.is_optional(),
+                        )
+                        # External appends are already absolute in Hydra's config namespace.
+                        node.update_parent("", "")
+                    else:
+                        node = ConfigDefault(
+                            path=f"{d.group}/{item}",
+                            package=d.package,
+                            optional=d.is_optional(),
+                        )
+                        node.update_parent(
+                            parent.get_group_path(), parent.get_final_package()
+                        )
                     new_root = DefaultsTreeNode(node=node, parent=root)
                     add_child(children, new_root)
 

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/hydra/_internal/grammar/grammar_functions.py

@@ -1,5 +1,6 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import builtins
+import decimal
 import json
 import random
 from copy import copy
@@ -361,27 +362,57 @@ def _sort_sweep(
     elif isinstance(sweep, RangeSweep):
         assert sweep.start is not None
         assert sweep.stop is not None
-        if not reverse:
-            # ascending
-            if sweep.start > sweep.stop:
-                start = sweep.stop + abs(sweep.step)
-                stop = sweep.start + abs(sweep.step)
-                sweep.start = start
-                sweep.stop = stop
-                sweep.step = -sweep.step
-        else:
-            # descending
-            if sweep.start < sweep.stop:
-                start = sweep.stop - abs(sweep.step)
-                stop = sweep.start - abs(sweep.step)
-                sweep.start = start
-                sweep.stop = stop
-                sweep.step = -sweep.step
+        if (sweep.step > 0) == reverse:
+            _reverse_range_sweep(sweep)
         return sweep
     else:
         assert False
 
 
+def _reverse_range_sweep(sweep: RangeSweep) -> None:
+    r = sweep.range()
+    if isinstance(r, builtins.range):
+        count = len(r)
+        if count > 1:
+            first = r[0]
+            last = r[-1]
+            sweep.start = last
+            sweep.step = -sweep.step
+            sweep.stop = first + sweep.step
+    else:
+        start = decimal.Decimal(str(sweep.start))
+        stop = decimal.Decimal(str(sweep.stop))
+        step = decimal.Decimal(str(sweep.step))
+        count = _float_range_count(start=start, stop=stop, step=step)
+        if count > 1:
+            first = start
+            last = start + (count - 1) * step
+            sweep.start = last
+            sweep.step = -step
+            sweep.stop = first + sweep.step / 2
+
+
+def _float_range_count(
+    start: decimal.Decimal, stop: decimal.Decimal, step: decimal.Decimal
+) -> int:
+    if step == 0:
+        return 0
+    if step > 0:
+        if start >= stop:
+            return 0
+        distance = stop - start
+    else:
+        if start <= stop:
+            return 0
+        distance = start - stop
+    step = abs(step)
+    quotient, remainder = divmod(distance, step)
+    count = int(quotient)
+    if remainder != 0:
+        count += 1
+    return count
+
+
 def glob(
     include: Union[List[str], str], exclude: Optional[Union[List[str], str]] = None
 ) -> Glob:

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/hydra/conf/__init__.py

@@ -51,11 +51,8 @@ class JobConf:
     # Will be non-optional and default to False in Hydra 1.3
     chdir: Optional[bool] = None
 
-    # Populated automatically by Hydra.
-    # Concatenation of job overrides that can be used as a part
-    # of the directory name.
-    # This can be configured via hydra.job.config.override_dirname
-    override_dirname: str = MISSING
+    # Deprecated. Use the hydra_override_dirname resolver instead.
+    override_dirname: str = "${hydra_override_dirname:}"
 
     # Job ID in underlying scheduling system
     id: str = MISSING
@@ -75,7 +72,7 @@ class JobConf:
     @dataclass
     class JobConfig:
         @dataclass
-        # configuration for the ${hydra.job.override_dirname} runtime variable
+        # Default configuration for the ${hydra_override_dirname:} resolver.
         class OverrideDirname:
             kv_sep: str = "="
             item_sep: str = ","

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/hydra/core/default_element.py

@@ -478,7 +478,7 @@ class GroupDefault(InputDefault):
             absolute = True
         else:
             group = self.group
-            absolute = False
+            absolute = self.external_append
 
         if self.parent_base_dir == "" or absolute:
             return group
@@ -508,7 +508,7 @@ class GroupDefault(InputDefault):
     def get_final_package(self, default_to_package_header: bool = True) -> str:
         name = self.get_name() if self.is_name() else None
         return self._get_final_package(
-            self._get_parent_package(),
+            "" if self.external_append else self._get_parent_package(),
             self.get_package(default_to_package_header=default_to_package_header),
             name,
         )

{hydra_core-1.4.0.dev3 → hydra_core-1.4.0.dev5}/hydra/core/override_parser/types.py

@@ -89,14 +89,15 @@ class ChoiceSweep(Sweep):
 
 @dataclass
 class FloatRange:
-    start: Union[decimal.Decimal, float]
-    stop: Union[decimal.Decimal, float]
-    step: Union[decimal.Decimal, float]
+    start: Union[decimal.Decimal, float, int]
+    stop: Union[decimal.Decimal, float, int]
+    step: Union[decimal.Decimal, float, int]
+    _idx: int = field(default=0, init=False, repr=False, compare=False)
 
     def __post_init__(self) -> None:
-        self.start = decimal.Decimal(self.start)
-        self.stop = decimal.Decimal(self.stop)
-        self.step = decimal.Decimal(self.step)
+        self.start = decimal.Decimal(str(self.start))
+        self.stop = decimal.Decimal(str(self.stop))
+        self.step = decimal.Decimal(str(self.step))
 
     def __iter__(self) -> Any:
         return self
@@ -105,18 +106,17 @@ class FloatRange:
         assert isinstance(self.start, decimal.Decimal)
         assert isinstance(self.stop, decimal.Decimal)
         assert isinstance(self.step, decimal.Decimal)
+        current = self.start + self.step * self._idx
         if self.step > 0:
-            if self.start < self.stop:
-                ret = float(self.start)
-                self.start += self.step
-                return ret
+            if current < self.stop:
+                self._idx += 1
+                return float(current)
             else:
                 raise StopIteration
         elif self.step < 0:
-            if self.start > self.stop:
-                ret = float(self.start)
-                self.start += self.step
-                return ret
+            if current > self.stop:
+                self._idx += 1
+                return float(current)
             else:
                 raise StopIteration
         else:
@@ -131,9 +131,9 @@ class RangeSweep(Sweep):
     Discrete range of numbers
     """
 
-    start: Optional[Union[int, float]] = None
-    stop: Optional[Union[int, float]] = None
-    step: Union[int, float] = 1
+    start: Optional[Union[int, float, decimal.Decimal]] = None
+    stop: Optional[Union[int, float, decimal.Decimal]] = None
+    step: Union[int, float, decimal.Decimal] = 1
 
     shuffle: bool = False