toml-combine 0.1.4__py3-none-any.whl

toml_combine/__init__.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import pathlib
+from typing import Any, cast, overload
+
+from . import combiner, toml
+
+
+# Provide already parsed config
+@overload
+def combine(
+    *, config: dict[str, Any], **filters: str | list[str]
+) -> dict[str, Any]: ...
+# Provide toml config content
+@overload
+def combine(*, config: str, **filters: str | list[str]) -> dict[str, Any]: ...
+# Provide toml config file path
+@overload
+def combine(
+    *, config_file: str | pathlib.Path, **filters: str | list[str]
+) -> dict[str, Any]: ...
+
+
+def combine(*, config=None, config_file=None, **filters):
+    """
+    Generate outputs of configurations based on the provided TOML
+    configuration.
+
+    Args:
+        config: The TOML configuration as a string or an already parsed dictionary.
+        OR:
+        config_file: The path to the TOML configuration file.
+        **filters: Filters to apply to the combined configuration
+            (dimension="value" or dimension=["list", "of", "values"]).
+
+    Returns:
+        dict[str, Any]: The combined configuration ({"output_id": {...}}).
+    """
+    if (config is None) is (config_file is None):
+        raise ValueError("Either 'config' or 'config_file' must be provided.")
+
+    if isinstance(config, dict):
+        dict_config = config
+    else:
+        if config_file:
+            config_string = pathlib.Path(config_file).read_text()
+        else:
+            config = cast(str, config)
+            config_string = config
+
+        dict_config = toml.loads(config_string)
+
+    config_obj = combiner.build_config(dict_config)
+
+    return combiner.generate_outputs(config_obj, **filters)

toml_combine/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from . import cli
+
+if __name__ == "__main__":
+    cli.run_cli()

toml_combine/cli.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import sys
+from collections.abc import Mapping
+
+from . import combiner, exceptions, toml
+
+
+def get_argument_parser(
+    dimensions: Mapping[str, list[str]] | None,
+) -> argparse.ArgumentParser:
+    """Get the command-line argument parser."""
+    arg_parser = argparse.ArgumentParser(
+        description="Create combined configurations from a TOML file",
+        add_help=(dimensions is not None),
+    )
+    arg_parser.add_argument(
+        "config",
+        type=pathlib.Path,
+        help="Path to the TOML configuration file",
+    )
+    if dimensions:
+        group = arg_parser.add_argument_group(
+            "dimensions",
+            "Filter the generated outputs by dimensions",
+        )
+
+        for name, values in dimensions.items():
+            group.add_argument(
+                f"--{name}", choices=values, help=f"Limit to given {name}"
+            )
+
+    return arg_parser
+
+
+def cli(argv) -> int:
+    """Main entry point."""
+
+    # Parse the config file argument to get the dimensions
+    arg_parser = get_argument_parser(dimensions=None)
+    args, _ = arg_parser.parse_known_args(argv)
+
+    try:
+        dict_config = toml.loads(args.config.read_text())
+    except FileNotFoundError:
+        print(f"Configuration file not found: {args.config}", file=sys.stderr)
+        return 1
+    except exceptions.TomlDecodeError as exc:
+        print(exc, file=sys.stderr)
+        return 1
+
+    try:
+        config = combiner.build_config(dict_config)
+    except exceptions.TomlCombineError as exc:
+        print(exc, file=sys.stderr)
+        return 1
+
+    # Parse all arguments
+    arg_parser = get_argument_parser(dimensions=config.dimensions)
+    args = arg_parser.parse_args(argv)
+
+    dimensions_filter = {
+        key: value
+        for key, value in vars(args).items()
+        if key in config.dimensions and value
+    }
+    # Generate final configurations for each output
+    try:
+        result = combiner.generate_outputs(config=config, **dimensions_filter)
+    except exceptions.TomlCombineError as exc:
+        print(exc, file=sys.stderr)
+        return 1
+
+    if not result:
+        print("No outputs found", file=sys.stderr)
+
+    print(json.dumps(result, indent=2))
+
+    return 0
+
+
+def run_cli():
+    sys.exit(cli(sys.argv[1:]))

toml_combine/combiner.py

@@ -0,0 +1,249 @@
+from __future__ import annotations
+
+import copy
+import dataclasses
+import itertools
+from collections.abc import Mapping, Sequence
+from functools import partial
+from typing import Any
+
+from . import exceptions
+
+
+@dataclasses.dataclass()
+class Output:
+    dimensions: Mapping[str, str]
+
+    @property
+    def id(self) -> str:
+        return f"{'-'.join(self.dimensions.values())}"
+
+    def __str__(self) -> str:
+        return f"Output(id={self.id})"
+
+
+@dataclasses.dataclass()
+class Override:
+    when: Mapping[str, str | list[str]]
+    config: Mapping[str, Any]
+
+    def __str__(self) -> str:
+        return f"Override({self.when})"
+
+
+@dataclasses.dataclass()
+class Config:
+    dimensions: Mapping[str, list[str]]
+    outputs: list[Output]
+    default: Mapping[str, Any]
+    overrides: Sequence[Override]
+
+
+def wrap_in_list(value: str | list[str]) -> list[str]:
+    """
+    Wrap a string in a list if it's not already a list.
+    """
+    if isinstance(value, str):
+        return [value]
+    return value
+
+
+def clean_dimensions_dict(
+    to_sort: Mapping[str, str | list[str]], clean: dict[str, list[str]], type: str
+) -> dict[str, str]:
+    """
+    Recreate a dictionary of dimension values with the same order as the
+    dimensions list.
+    """
+    result = {}
+    remaining = dict(to_sort)
+
+    for dimension, valid_values in clean.items():
+        valid_values = set(valid_values)
+        if dimension not in to_sort:
+            continue
+
+        original_value = remaining.pop(dimension)
+        values = set(wrap_in_list(original_value))
+        if invalid_values := values - valid_values:
+            raise exceptions.DimensionValueNotFound(
+                type=type,
+                id=to_sort,
+                dimension=dimension,
+                value=", ".join(invalid_values),
+            )
+        result[dimension] = original_value
+
+    if remaining:
+        raise exceptions.DimensionNotFound(
+            type=type,
+            id=to_sort,
+            dimension=", ".join(to_sort),
+        )
+
+    return result
+
+
+def override_sort_key(
+    override: Override, dimensions: dict[str, list[str]]
+) -> tuple[int, ...]:
+    """
+    We sort overrides before applying them, and they are applied in the order of the
+    sorted list, each override replacing the common values of the previous overrides.
+
+    override_sort_key defines the sort key for overrides that ensures less specific
+    overrides come first:
+    - Overrides with fewer dimensions come first (will be overridden
+      by more specific ones)
+    - If two overrides have the same number of dimensions but define different
+      dimensions, we sort by the definition order of the dimensions.
+
+    Example:
+    dimensions = {"env": ["dev", "prod"], "region": ["us", "eu"]}
+
+    - Override with {"env": "dev"} comes before override with
+      {"env": "dev", "region": "us"} (less specific)
+    - Override with {"env": "dev"} comes before override with {"region": "us"} ("env"
+      is defined before "region" in the dimensions list)
+    """
+    result = [len(override.when)]
+    for i, dimension in enumerate(dimensions):
+        if dimension in override.when:
+            result.append(i)
+
+    return tuple(result)
+
+
+def merge_configs(a: Any, b: Any, /) -> Any:
+    """
+    Recursively merge two configuration dictionaries, with b taking precedence.
+    """
+    if isinstance(a, dict) != isinstance(b, dict):
+        raise ValueError(f"Cannot merge {type(a)} with {type(b)}")
+
+    if not isinstance(a, dict):
+        return b
+
+    result = a.copy()
+    for key, b_value in b.items():
+        if a_value := a.get(key):
+            result[key] = merge_configs(a_value, b_value)
+        else:
+            result[key] = b_value
+    return result
+
+
+def build_config(config: dict[str, Any]) -> Config:
+    # Parse dimensions
+    dimensions = config.pop("dimensions")
+
+    # Parse template
+    default = config.pop("default", {})
+
+    seen_conditions = set()
+    overrides = []
+    for override in config.pop("override", []):
+        try:
+            when = override.pop("when")
+        except KeyError:
+            raise exceptions.MissingOverrideCondition(id=override)
+
+        conditions = tuple((k, tuple(wrap_in_list(v))) for k, v in when.items())
+        if conditions in seen_conditions:
+            raise exceptions.DuplicateError(type="override", id=when)
+
+        seen_conditions.add(conditions)
+
+        overrides.append(
+            Override(
+                when=clean_dimensions_dict(
+                    to_sort=when, clean=dimensions, type="override"
+                ),
+                config=override,
+            )
+        )
+    # Sort overrides by increasing specificity
+    overrides = sorted(
+        overrides,
+        key=partial(override_sort_key, dimensions=dimensions),
+    )
+
+    outputs = []
+    seen_conditions = set()
+
+    for output in config.pop("output", []):
+        for key in output:
+            output[key] = wrap_in_list(output[key])
+
+        for cartesian_product in itertools.product(*output.values()):
+            # Create a dictionary with the same keys as when
+            single_output = dict(zip(output.keys(), cartesian_product))
+
+            conditions = tuple(single_output.items())
+            if conditions in seen_conditions:
+                raise exceptions.DuplicateError(type="output", id=output.id)
+            seen_conditions.add(conditions)
+
+            outputs.append(
+                Output(
+                    dimensions=clean_dimensions_dict(
+                        single_output, dimensions, type="output"
+                    ),
+                )
+            )
+
+    return Config(
+        dimensions=dimensions,
+        outputs=outputs,
+        default=default,
+        overrides=overrides,
+    )
+
+
+def generate_output(
+    default: Mapping[str, Any], overrides: Sequence[Override], output: Output
+) -> dict[str, Any]:
+    result = copy.deepcopy(default)
+    # Apply each matching override
+    for override in overrides:
+        # Check if all dimension values in the override match
+        if all(
+            override.when.get(dim) == output.dimensions.get(dim)
+            for dim in override.when.keys()
+        ):
+            result = merge_configs(result, override.config)
+
+    return {"dimensions": output.dimensions, **result}
+
+
+def generate_outputs(config: Config, **filter: str | list[str]) -> dict[str, Any]:
+    result = {}
+    filter_with_lists: dict[str, list[str]] = {}
+
+    for key, value in list(filter.items()):
+        if key not in config.dimensions:
+            raise exceptions.DimensionNotFound(type="arguments", id="", dimension=key)
+
+        value = wrap_in_list(value)
+
+        if set(value) - set(config.dimensions[key]):
+            raise exceptions.DimensionValueNotFound(
+                type="arguments",
+                id="",
+                dimension=key,
+                value=", ".join(set(value) - set(config.dimensions[key])),
+            )
+        filter_with_lists[key] = value
+
+    for output in config.outputs:
+        if all(
+            output.dimensions.get(key) in value
+            for key, value in filter_with_lists.items()
+        ):
+            result[output.id] = generate_output(
+                default=config.default,
+                overrides=config.overrides,
+                output=output,
+            )
+
+    return result

toml_combine/exceptions.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+
+class TomlCombineError(Exception):
+    """There was an error in the toml-combine library."""
+
+    def __init__(self, message: str = "", **kwargs) -> None:
+        message = message or type(self).__doc__ or ""
+        message = message.format(**kwargs)
+
+        super().__init__(message)
+
+
+class TomlDecodeError(TomlCombineError):
+    """Error while decoding configuration file."""
+
+
+class DuplicateError(TomlCombineError):
+    """In {type} {id}: Cannot have multiple {type}s with the same dimensions."""
+
+
+class DimensionNotFound(TomlCombineError):
+    """In {type} {id}: Dimension {dimension} not found."""
+
+
+class DimensionValueNotFound(TomlCombineError):
+    """In {type} {id}: Value {value} for dimension {dimension} not found."""
+
+
+class MissingOverrideCondition(TomlCombineError):
+    """In override {id}: Missing 'when' key in override configuration"""

toml_combine/toml.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import Any
+
+import tomli
+
+from . import exceptions
+
+
+def loads(raw_config: str) -> dict[str, Any]:
+    try:
+        return tomli.loads(raw_config)
+    except tomli.TOMLDecodeError as e:
+        raise exceptions.TomlDecodeError(
+            message="Syntax error in TOML configuration file",
+            context=str(e),
+        ) from e

toml_combine-0.1.4.dist-info/METADATA

@@ -0,0 +1,210 @@
+Metadata-Version: 2.4
+Name: toml-combine
+Version: 0.1.4
+Summary: A tool for combining complex configurations in TOML format.
+Author-email: Joachim Jablon <ewjoachim@gmail.com>
+License-Expression: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.9
+Requires-Dist: tomli>=2.2.1
+Description-Content-Type: text/markdown
+
+# Toml-combine
+
+`toml-combine` is a Python lib and CLI-tool that reads a toml configuration defining
+a default configuration and overrides with, and applies those overrides to get
+final configurations. Say: you have multiple services, and environments, and you
+want to describe them all without repeating the parts that are common to everyone.
+
+## Concepts
+
+### The config file
+
+The configuration file is usually a TOML file. Here's a small example:
+
+```toml
+[dimensions]
+environment = ["production", "staging"]
+
+[[output]]
+environment = "production"
+
+[[output]]
+environment = "staging"
+
+[default]
+name = "my-service"
+registry = "gcr.io/my-project/"
+container.image_name = "my-image"
+container.port = 8080
+service_account = "my-service-account"
+
+[[override]]
+when.environment = "staging"
+service_account = "my-staging-service-account"
+```
+
+### Dimensions
+
+Consider all the configurations you want to generate. Each one differs from the others.
+Dimensions lets you describe the main "thing" that makes manifests differents, e.g.:
+`environment` might be `staging` or `production`, region might be `eu` or `us`, and
+service might be `frontend` or `backend`. Some combinations of dimensions might not
+exists, for example, maybe there's no `staging` in `eu`.
+
+### Outputs
+
+Create a `output` for each configuration you want to generate, and specify the
+dimensions relevant for this output. It's ok to omit some dimensions when they're not
+used for a given output.
+
+> [!Note]
+> Defining a list as the value of one or more dimensions in a output
+> is a shorthand for defining all combinations of dimensions
+
+### Default
+
+The common configuration to start from, before we start overlaying overrides on top.
+
+### Overrides
+
+Overrides define a set of condition where they apply (`when.<dimension> =
+"<value>"`) and the values that are overriden. Overrides are applied in order from less
+specific to more specific, each one overriding the values of the previous ones:
+
+- If an override contains conditions on more dimensions than another one, it's applied
+  later
+- In case 2 overrides contain the same number of dimensions and they're a disjoint set,
+  then it depends on how the dimensions are defined at the top of the file: dimensions
+  defined last have a greater priority
+
+> [!Note]
+> Defining a list as the value of one or more conditions in an override
+> means that the override will apply to any of the dimension values of the list
+
+### The configuration itself
+
+Under the layer of `dimensions/output/default/override` system, what you actually define
+in the configuration is completely up to you. That said, only nested
+"dictionnaries"/"objects"/"tables"/"mapping" (those are all the same things in
+Python/JS/Toml lingo) will be merged between the default and the overrides, while
+arrays will just replace one another. See `Arrays` below.
+
+In the generated configuration, the dimensions of the output will appear in the generated
+object as an object under the `dimensions` key.
+
+### Arrays
+
+Let's look at an example:
+
+```toml
+[dimensions]
+environment = ["production", "staging"]
+
+[[output]]
+environment = ["production", "staging"]
+
+[default]
+fruits = [{name="apple", color="red"}]
+
+[[override]]
+when.environment = "staging"
+fruits = [{name="orange", color="orange"}]
+```
+
+In this example, on staging, `fruits` is `[{name="orange", color="orange"}]` and not `[{name="apple", color="red"}, {name="orange", color="orange"}]`.
+The only way to get multiple values to be merged is if they are tables: you'll need
+to chose an element to become the key:
+
+```toml
+[dimensions]
+environment = ["production", "staging"]
+
+[[output]]
+environment = ["production", "staging"]
+
+[default]
+fruits.apple.color = "red"
+
+[[override]]
+when.environment = "staging"
+fruits.orange.color = "orange"
+```
+
+In this example, on staging, `fruits` is `{apple={color="red"}, orange={color="orange"}}`.
+
+This example is simple because `name` is a natural choice for the key. In some cases,
+the choice is less natural, but you can always decide to name the elements of your
+list and use that name as a key. Also, yes, you'll loose ordering.
+
+### A bigger example
+
+```toml
+[dimensions]
+environment = ["production", "staging", "dev"]
+service = ["frontend", "backend"]
+
+# All 4 combinations of those values will exist
+[[output]]
+environment = ["production", "staging"]
+service = ["frontend", "backend"]
+
+# On dev, the "service" is not defined. That's ok.
+[[output]]
+environment = "dev"
+
+[default]
+registry = "gcr.io/my-project/"
+service_account = "my-service-account"
+
+[[override]]
+when.service = "frontend"
+name = "service-frontend"
+container.image_name = "my-image-frontend"
+
+[[override]]
+when.service = "backend"
+name = "service-backend"
+container.image_name = "my-image-backend"
+container.port = 8080
+
+[[override]]
+name = "service-dev"
+when.environment = "dev"
+container.env.DEBUG = true
+```
+
+### CLI
+
+```console
+$ toml-combine {path/to/config.toml}
+```
+
+Generates all the outputs described by the given TOML config.
+
+Note that you can restrict generation to some dimension values by passing
+`--{dimension}={value}`
+
+## Lib
+
+```python
+import toml_combine
+
+
+result = toml_combine.combine(
+        config_file=config_file,
+        environment=["production", "staging"],
+        type="job",
+        job=["manage", "special-command"],
+    )
+
+print(result)
+```

toml_combine-0.1.4.dist-info/RECORD

@@ -0,0 +1,10 @@
+toml_combine/__init__.py,sha256=l7i0GkM9k7cc__wj1yqK5XjpB3IJ0jqFU63tqKMuYlY,1625
+toml_combine/__main__.py,sha256=hmF8N8xX6UEApzbKTVZ-4E1HU5-rjgUkdXNLO-mF6vo,100
+toml_combine/cli.py,sha256=MZrAEP4wt6f9Qn0TEXIjeLoQMlvQulFpkMciwU8GRO4,2328
+toml_combine/combiner.py,sha256=98iWUnkbzDVGEkw-dXFZEt5G7io5SQzayceuTU2hRvo,7315
+toml_combine/exceptions.py,sha256=SepRFDxeWQEbD88jhF5g7laZSSULthho83BpW8u9RWs,897
+toml_combine/toml.py,sha256=_vCINvfJeS3gWid35Pmm3Yz4xyJ8LpKJRHL0axSU8nk,384
+toml_combine-0.1.4.dist-info/METADATA,sha256=_IzI_kjsNl52wFynzgII2zXeMGnD8XqJ8Mw5FHwOLIs,6051
+toml_combine-0.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+toml_combine-0.1.4.dist-info/entry_points.txt,sha256=dXUQNom54uZt_7ylEG81iNYMamYpaFo9-ItcZJU6Uzc,58
+toml_combine-0.1.4.dist-info/RECORD,,

toml_combine-0.1.4.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

toml_combine-0.1.4.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+toml-combine = toml_combine.cli:run_cli