megaconf 0.1.0__tar.gz

megaconf-0.1.0/LICENSE

@@ -0,0 +1,9 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Richard Wessels
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

megaconf-0.1.0/PKG-INFO

@@ -0,0 +1,230 @@
+Metadata-Version: 2.4
+Name: megaconf
+Version: 0.1.0
+Summary: Easily create many variations of a config.
+Author: Richard Wessels
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Dist: numpy>=2.5.2
+Requires-Dist: pydantic>=2.13.5
+Requires-Dist: pyyaml>=6.0.3
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/RichardWessels/megaconf
+Project-URL: Issues, https://github.com/RichardWessels/megaconf/issues
+Description-Content-Type: text/markdown
+
+# megaconf
+
+megaconf is a tool to help with producing configs. This tool allows you to describe configuration combinations declaratively, instead of writing nested loops in code.
+
+## How it works
+
+megaconf generates configurations from two inputs:
+
+1. Base config – a starting configuration.
+2. Override config – rules describing how to modify the base config to produce multiple variants.
+
+The result is a set of generated configs.
+
+## Example
+
+Suppose we start with a base config:
+
+```yaml
+algo: NN
+lr: 0.001
+layer_count: 8
+activation: ReLU
+initializer: He
+```
+
+We want to run experiments across different algorithms and hyperparameters.
+
+This might look like:
+
+```python
+for algo in ["XGBoost", "NN"]:
+    for lr in [0.1, 0.01, 0.001]:
+        if algo == "NN":
+            for layer_count in [4, 8, 12, 16]:
+                for activation in ["ReLU", "Tanh"]:
+                    ...
+```
+
+As the number of parameters and dependencies grows, this approach becomes hard to maintain.
+
+megaconf allows expressing the same logic using override rules.
+
+Override Config:
+```yaml
+- fixed:
+    algo: XGBoost
+  product:
+    lr: [0.1, 0.01, 0.001]
+
+- fixed:
+    algo: NN
+  joint:
+    activation: [ReLU, Tanh]
+    initializer: [He, Glorot]
+  product:
+    lr: [0.1, 0.01, 0.001]
+    layer_count: [4, 8, 12, 16]
+```
+
+## Override Methods
+
+megaconf supports three override methods.
+
+---
+
+### `fixed`
+
+Sets specific values in the config.
+
+```yaml
+fixed:
+  algo: NN
+```
+
+Equivalent to:
+
+```python
+config["algo"] = "NN"
+```
+
+If the key does not exist in the base config, it is added.
+
+---
+
+### `joint`
+
+Iterates over multiple parameters together.
+
+All lists must be the same length.
+
+```yaml
+joint:
+  activation: [ReLU, Tanh]
+  initializer: [He, Glorot]
+```
+
+Result:
+```
+activation=ReLU  initializer=He
+activation=Tanh  initializer=Glorot
+```
+
+This is useful for paired parameters.
+
+---
+
+### `product`
+
+Produces the cartesian product of parameter values.
+
+```yaml
+product:
+  lr: [0.1, 0.01, 0.001]
+  layer_count: [4, 8, 12]
+```
+
+This generates:
+
+```
+lr=0.1   layer_count=4
+lr=0.1   layer_count=8
+lr=0.1   layer_count=12
+lr=0.01  layer_count=4
+...
+```
+
+This is equivalent to a grid search.
+
+## How Overrides Are Combined
+
+Within an override block:
+
+1. fixed values are applied first.
+2. joint combinations are generated.
+3. product combinations are generated.
+
+The final configs are the combination of these rules applied to the base config.
+
+## Interface
+
+Create configurations with `generate_configs`. It accepts a base config and an
+overrides config.
+
+Both inputs can be provided in two ways:
+- as a Python object (e.g., a dictionary or list).
+- as a path to a configuration file (.yaml, .yml, or .json).
+
+`generate_configs` returns an iterator that yields each generated config. The
+number of generated configs depends on the combinations produced by the override
+rules. Use `generate_configs_list` if you need all generated configs in a list.
+
+Example usage:
+
+```python
+from megaconf import generate_configs
+
+base_config = {
+    "algo": "NN",
+    "lr": 0.001,
+    "layer_count": 8,
+    "activation": "ReLU",
+    "initializer": "He"
+}
+
+overrides = [
+    {
+        "fixed": {
+            "algo": "XGBoost"
+        },
+        "product": {
+            "lr": [0.1, 0.01, 0.001]
+        }
+    },
+    {
+        "fixed": {
+            "algo": "NN"
+        },
+        "joint": {
+            "activation": ["ReLU", "Tanh"],
+            "initializer": ["He", "Glorot"]
+        },
+        "product": {
+            "lr": [0.1, 0.01],
+            "layer_count": [4, 8]
+        }
+    }
+]
+
+configs = generate_configs(base_config, overrides)
+```
+
+Or if the base and overrides separate config files:
+```python
+configs = generate_configs(
+    "base_config.yaml",
+    "overrides.yaml"
+)
+```
+
+## Nesting
+If a key is deeply nested in the config, a dot-key separator is used by default. This means that specifying a nested key is done as follows:
+```yaml
+fixed:
+  key1.key2.key3: value
+```
+However, if a dot conflicts with your key strings, you can specify a custom key separator as follows:
+```python
+configs = generate_configs(
+    "base_config.yaml",
+    "overrides.yaml",
+    key_separator="::"
+)
+```

megaconf-0.1.0/README.md

@@ -0,0 +1,213 @@
+# megaconf
+
+megaconf is a tool to help with producing configs. This tool allows you to describe configuration combinations declaratively, instead of writing nested loops in code.
+
+## How it works
+
+megaconf generates configurations from two inputs:
+
+1. Base config – a starting configuration.
+2. Override config – rules describing how to modify the base config to produce multiple variants.
+
+The result is a set of generated configs.
+
+## Example
+
+Suppose we start with a base config:
+
+```yaml
+algo: NN
+lr: 0.001
+layer_count: 8
+activation: ReLU
+initializer: He
+```
+
+We want to run experiments across different algorithms and hyperparameters.
+
+This might look like:
+
+```python
+for algo in ["XGBoost", "NN"]:
+    for lr in [0.1, 0.01, 0.001]:
+        if algo == "NN":
+            for layer_count in [4, 8, 12, 16]:
+                for activation in ["ReLU", "Tanh"]:
+                    ...
+```
+
+As the number of parameters and dependencies grows, this approach becomes hard to maintain.
+
+megaconf allows expressing the same logic using override rules.
+
+Override Config:
+```yaml
+- fixed:
+    algo: XGBoost
+  product:
+    lr: [0.1, 0.01, 0.001]
+
+- fixed:
+    algo: NN
+  joint:
+    activation: [ReLU, Tanh]
+    initializer: [He, Glorot]
+  product:
+    lr: [0.1, 0.01, 0.001]
+    layer_count: [4, 8, 12, 16]
+```
+
+## Override Methods
+
+megaconf supports three override methods.
+
+---
+
+### `fixed`
+
+Sets specific values in the config.
+
+```yaml
+fixed:
+  algo: NN
+```
+
+Equivalent to:
+
+```python
+config["algo"] = "NN"
+```
+
+If the key does not exist in the base config, it is added.
+
+---
+
+### `joint`
+
+Iterates over multiple parameters together.
+
+All lists must be the same length.
+
+```yaml
+joint:
+  activation: [ReLU, Tanh]
+  initializer: [He, Glorot]
+```
+
+Result:
+```
+activation=ReLU  initializer=He
+activation=Tanh  initializer=Glorot
+```
+
+This is useful for paired parameters.
+
+---
+
+### `product`
+
+Produces the cartesian product of parameter values.
+
+```yaml
+product:
+  lr: [0.1, 0.01, 0.001]
+  layer_count: [4, 8, 12]
+```
+
+This generates:
+
+```
+lr=0.1   layer_count=4
+lr=0.1   layer_count=8
+lr=0.1   layer_count=12
+lr=0.01  layer_count=4
+...
+```
+
+This is equivalent to a grid search.
+
+## How Overrides Are Combined
+
+Within an override block:
+
+1. fixed values are applied first.
+2. joint combinations are generated.
+3. product combinations are generated.
+
+The final configs are the combination of these rules applied to the base config.
+
+## Interface
+
+Create configurations with `generate_configs`. It accepts a base config and an
+overrides config.
+
+Both inputs can be provided in two ways:
+- as a Python object (e.g., a dictionary or list).
+- as a path to a configuration file (.yaml, .yml, or .json).
+
+`generate_configs` returns an iterator that yields each generated config. The
+number of generated configs depends on the combinations produced by the override
+rules. Use `generate_configs_list` if you need all generated configs in a list.
+
+Example usage:
+
+```python
+from megaconf import generate_configs
+
+base_config = {
+    "algo": "NN",
+    "lr": 0.001,
+    "layer_count": 8,
+    "activation": "ReLU",
+    "initializer": "He"
+}
+
+overrides = [
+    {
+        "fixed": {
+            "algo": "XGBoost"
+        },
+        "product": {
+            "lr": [0.1, 0.01, 0.001]
+        }
+    },
+    {
+        "fixed": {
+            "algo": "NN"
+        },
+        "joint": {
+            "activation": ["ReLU", "Tanh"],
+            "initializer": ["He", "Glorot"]
+        },
+        "product": {
+            "lr": [0.1, 0.01],
+            "layer_count": [4, 8]
+        }
+    }
+]
+
+configs = generate_configs(base_config, overrides)
+```
+
+Or if the base and overrides separate config files:
+```python
+configs = generate_configs(
+    "base_config.yaml",
+    "overrides.yaml"
+)
+```
+
+## Nesting
+If a key is deeply nested in the config, a dot-key separator is used by default. This means that specifying a nested key is done as follows:
+```yaml
+fixed:
+  key1.key2.key3: value
+```
+However, if a dot conflicts with your key strings, you can specify a custom key separator as follows:
+```python
+configs = generate_configs(
+    "base_config.yaml",
+    "overrides.yaml",
+    key_separator="::"
+)
+```

megaconf-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "megaconf"
+version = "0.1.0"
+description = "Easily create many variations of a config."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "numpy>=2.5.2",
+    "pydantic>=2.13.5",
+    "pyyaml>=6.0.3",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "MIT"
+license-files = ["LICENSE"]
+
+[[project.authors]]
+name = "Richard Wessels"
+
+[project.urls]
+Homepage = "https://github.com/RichardWessels/megaconf"
+Issues = "https://github.com/RichardWessels/megaconf/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "ruff>=0.16.5",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.13.0"]
+build-backend = "uv_build"

megaconf-0.1.0/pyproject.toml.orig

@@ -0,0 +1,34 @@
+[project]
+name = "megaconf"
+version = "0.1.0"
+description = "Easily create many variations of a config."
+readme = "README.md"
+authors = [
+    { name = "Richard Wessels" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "numpy>=2.5.2",
+    "pydantic>=2.13.5",
+    "pyyaml>=6.0.3",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "MIT"
+license-files = ["LICENSE"]
+
+[project.urls]
+Homepage = "https://github.com/RichardWessels/megaconf"
+Issues = "https://github.com/RichardWessels/megaconf/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "ruff>=0.16.5",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.13.0"]
+build-backend = "uv_build"

megaconf-0.1.0/src/megaconf/__init__.py

@@ -0,0 +1,3 @@
+from .generate import generate_configs, generate_configs_list
+
+__all__ = ["generate_configs", "generate_configs_list"]

megaconf-0.1.0/src/megaconf/expansion.py

@@ -0,0 +1,78 @@
+import random
+from collections.abc import Iterator
+from typing import Literal
+
+import numpy as np
+
+
+def get_joint_generator(joint_config: dict[str, list]) -> Iterator[dict]:
+    """Yield joint overrides by zipping aligned list values.
+
+    Args:
+        joint_config: Mapping of keys to equally sized lists.
+
+    Yields:
+        Dictionaries containing one value per key at each shared index.
+    """
+    if not joint_config:
+        yield {}
+        return
+
+    list_length = len(next(iter(joint_config.values())))
+
+    for i in range(list_length):
+        yield {k: v[i] for k, v in joint_config.items()}
+
+
+def get_product_generator(
+    product_config: dict[str, list],
+    sampling: Literal["without_replacement", "with_replacement"] | None = None,
+    n_samples: int | None = None,
+) -> Iterator[dict]:
+    """Yield product overrides from the Cartesian product of list values.
+
+    Args:
+        product_config: Mapping of keys to candidate values.
+        sampling: Optional sampling mode over product indices.
+        n_samples: Number of sampled combinations when sampling is enabled.
+
+    Yields:
+        Dictionaries containing one chosen value per product key.
+
+    Raises:
+        ValueError: If sampling is enabled without ``n_samples`` or mode is invalid.
+    """
+
+    if not product_config:
+        yield {}
+        return
+
+    prod_items = product_config.items()
+    prod_keys = [item[0] for item in prod_items]
+    prod_values = [item[1] for item in prod_items]
+
+    n = int(np.prod([len(it) for it in prod_values]))
+
+    if sampling is None:
+        loop_values = range(n)
+    else:
+        if n_samples is None:
+            raise ValueError("Argument `n_samples` required when using sampling.")
+        if sampling == "with_replacement":
+            loop_values = random.choices(range(n), k=n_samples)
+        elif sampling == "without_replacement":
+            loop_values = random.sample(range(n), k=n_samples)
+        else:
+            # illegal state
+            raise ValueError("Invalid value for `sampling` given.")
+
+    for num in loop_values:
+        i = num
+
+        res = []
+        for it in reversed(prod_values):
+            i, r = divmod(i, len(it))
+            res.append(it[r])
+        res = list(reversed(res))
+
+        yield {prod_keys[i]: res[i] for i in range(len(prod_keys))}

megaconf-0.1.0/src/megaconf/generate.py

@@ -0,0 +1,122 @@
+from collections.abc import Iterator
+from copy import deepcopy
+from pathlib import Path
+from typing import Literal
+
+from pydantic import TypeAdapter
+
+from .expansion import get_joint_generator, get_product_generator
+from .inputs import load_data_from_file
+from .models import BaseConfigInput, Override, OverridesInput
+from .utils import convert_flat_dict_to_nested_dict, deep_update_dict
+
+
+def _generate_configs(
+    base_config: dict,
+    overrides: list[Override],
+    key_separator: str,
+    sampling: Literal["without_replacement", "with_replacement"] | None = None,
+    n_samples: int | None = None,
+) -> Iterator[dict]:
+    """Generate merged configs for all fixed, joint, and product combinations.
+
+    Args:
+        base_config: Base configuration to copy and update.
+        overrides: Validated override groups.
+        key_separator: Separator for flat keys that target nested fields.
+        sampling: Optional sampling mode for product combinations.
+        n_samples: Number of product combinations to sample.
+
+    Yields:
+        Fully merged configuration dictionaries.
+    """
+
+    for override in overrides:
+        fixed = override.fixed
+
+        for joint_conf in get_joint_generator(override.joint):
+            # NOTE: only product space is sampled
+            for prod_conf in get_product_generator(
+                override.product, sampling=sampling, n_samples=n_samples
+            ):
+                new_config = deepcopy(base_config)
+                override_dict = fixed | joint_conf | prod_conf
+
+                override_dict = convert_flat_dict_to_nested_dict(
+                    override_dict, key_separator
+                )
+                new_config = deep_update_dict(new_config, override_dict)
+
+                yield new_config
+
+
+def generate_configs(
+    base_config: BaseConfigInput,
+    overrides: OverridesInput | None,
+    key_separator=".",
+    sampling: Literal["without_replacement", "with_replacement"] | None = None,
+    n_samples: int | None = None,
+) -> Iterator[dict]:
+    """Generate configurations from a base config and override definitions.
+
+    Args:
+        base_config: Base config dictionary or path to YAML/JSON.
+        overrides: Override list or path to YAML/JSON override definitions.
+        key_separator: Separator used in flat override keys for nesting.
+        sampling: Optional sampling mode for product combinations.
+        n_samples: Number of product combinations to sample.
+
+    Returns:
+        An iterator of generated configuration dictionaries.
+
+    Raises:
+        ValueError: If base config does not resolve to a dictionary.
+    """
+
+    if not isinstance(base_config, dict):
+        path = Path(base_config)
+        base_config = load_data_from_file(path)
+
+    if isinstance(overrides, (str, Path)):
+        path = Path(overrides)
+        overrides = load_data_from_file(path)
+
+    # validation
+    if not isinstance(base_config, dict):
+        raise ValueError("Base config must be a dictionary.")
+    overrides_validated = TypeAdapter(list[Override]).validate_python(overrides)
+
+    if (
+        not overrides
+    ):  # NOTE: need to confirm that this general falsy check is not a problem
+        yield base_config
+        return
+
+    yield from _generate_configs(
+        base_config, overrides_validated, key_separator, sampling, n_samples
+    )
+
+
+def generate_configs_list(
+    base_config: BaseConfigInput,
+    overrides: OverridesInput | None,
+    key_separator=".",
+    sampling: Literal["without_replacement", "with_replacement"] | None = None,
+    n_samples: int | None = None,
+) -> list[dict]:
+    """Return generated configurations as a list.
+
+    Args:
+        base_config: Base config dictionary or path to YAML/JSON.
+        overrides: Override list or path to YAML/JSON override definitions.
+        key_separator: Separator used in flat override keys for nesting.
+        sampling: Optional sampling mode for product combinations.
+        n_samples: Number of product combinations to sample.
+
+    Returns:
+        All generated configurations materialized in a list.
+    """
+
+    return list(
+        generate_configs(base_config, overrides, key_separator, sampling, n_samples)
+    )

megaconf-0.1.0/src/megaconf/inputs.py

@@ -0,0 +1,27 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from yaml import safe_load
+
+
+def load_data_from_file(file_path: Path | str) -> Any:
+    """Load YAML or JSON data from disk.
+
+    Args:
+        file_path: Path to a ``.yaml``, ``.yml``, or ``.json`` file.
+
+    Returns:
+        Parsed file contents.
+
+    Raises:
+        ValueError: If the file extension is unsupported.
+    """
+    file_path = Path(file_path)
+
+    with open(file_path, encoding="utf-8") as f:
+        if file_path.suffix in [".yaml", ".yml"]:
+            return safe_load(f)
+        if file_path.suffix == ".json":
+            return json.load(f)
+        raise ValueError("File must end in `.yaml`, `.yml` or `.json`.")

megaconf-0.1.0/src/megaconf/models.py

@@ -0,0 +1,38 @@
+from os import PathLike
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Override(BaseModel):
+    fixed: dict[str, Any] = Field(default_factory=dict)
+    joint: dict[str, list] = Field(default_factory=dict)
+    product: dict[str, list] = Field(default_factory=dict)
+
+    @field_validator("joint")
+    @classmethod
+    def validate_joint_lengths(cls, joint: dict[str, list]):
+        """Validate that all lists in ``joint`` have equal length.
+
+        Args:
+            joint: Mapping of override keys to lists.
+
+        Returns:
+            The validated ``joint`` mapping.
+
+        Raises:
+            ValueError: If list lengths differ across ``joint`` keys.
+        """
+        lengths = {key: len(value) for key, value in joint.items()}
+
+        if len(set(lengths.values())) > 1:
+            raise ValueError(
+                f"All lists in 'joint' must have equal length. Got lengths: {lengths}"
+            )
+
+        return joint
+
+
+type ConfigDict = dict[str, Any]
+type BaseConfigInput = ConfigDict | str | PathLike[str]
+type OverridesInput = list[dict[str, Any]] | str | PathLike[str]

megaconf-0.1.0/src/megaconf/utils.py

@@ -0,0 +1,54 @@
+from copy import deepcopy
+from typing import Any
+
+
+def convert_flat_dict_to_nested_dict(
+    dictionary: dict[str, Any], key_separator=".", override_duplicates=False
+) -> dict:
+    """
+    Convert from format: {"k1.k2": value} -> {"k1": {"k2": value}}
+    """
+    output_dict = {}
+
+    for composite_key, value in dictionary.items():
+        current_dict = output_dict
+        key_list = composite_key.split(key_separator)
+
+        # step through dictionary
+        for key in key_list[:-1]:
+            if key in current_dict and not isinstance(current_dict[key], dict):
+                if override_duplicates:
+                    current_dict[key] = {}
+                else:
+                    raise RuntimeError(
+                        f"Duplicate key found: '{composite_key}'. "
+                        "Set `override_duplicates`=True to skip this error."
+                    )
+            current_dict[key] = current_dict.get(key, {})
+            current_dict = current_dict[key]
+
+        # write value
+        key = key_list[-1]
+        if key in current_dict and not override_duplicates:
+            raise RuntimeError(
+                f"Duplicate key found: '{composite_key}'. "
+                "Set `override_duplicates`=True to skip this error."
+            )
+        current_dict[key] = value
+    return output_dict
+
+
+def deep_update_dict(base: dict, override: dict) -> dict:
+    """
+    Recursively replaces each key/value in `base` that is also present in `override`
+    When key not in `base`, adds the key.
+    """
+    output = deepcopy(base)
+
+    for key, value in override.items():
+        if isinstance(value, dict) and isinstance(output.get(key), dict):
+            output[key] = deep_update_dict(output[key], value)
+        else:
+            output[key] = deepcopy(value)
+
+    return output