powerconf 0.1.0__py3-none-any.whl

powerconf/cli.py

@@ -0,0 +1,51 @@
+from pathlib import Path
+from typing import Annotated, List
+
+import rich
+import typer
+import yaml
+from fspathtree import fspathtree
+
+from . import rendering
+
+app = typer.Typer()
+console = rich.console.Console()
+
+
+def load_config(config_file: Path):
+    """load config from a file."""
+    configs = []
+    if not config_file.exists():
+        raise typer.Exit(f"File '{config_file}' not found.")
+    config_text = config_file.read_text()
+    config_docs = config_text.split("---")
+    for doc in config_docs:
+        config = fspathtree(yaml.safe_load(doc))
+        configs.append(config)
+
+    return configs
+
+
+@app.command()
+def render(config_file: Path, output: Path = None):
+    if output is None:
+        output = config_file.parent / (config_file.name + ".rendered")
+    configs = load_config(config_file)
+    config_renderer = rendering.ConfigRenderer()
+    rendered_configs = []
+    for config in configs:
+        config = config_renderer.expand_and_render(config)
+        rendered_configs += config
+    if len(rendered_configs) == 1:
+        output.write_text(yaml.dump(rendered_configs[0].tree))
+
+
+@app.command()
+def help():
+    print(
+        """
+The `powerconf` command is a CLI for the powerconf python module. It allows you to read
+a configuration file, evaluate all expression, expand all batch nodes, etc, and write
+the "rendered" configurations to a file(s).
+"""
+    )

powerconf/exceptions.py

@@ -0,0 +1,8 @@
+class UndefinedVariableInExpression(RuntimeError):
+    def __init__(self, msg):
+        super().__init__(msg)
+
+
+class SyntaxErrorInExpression(RuntimeError):
+    def __init__(self, msg):
+        super().__init__(msg)

powerconf/expressions.py

@@ -0,0 +1,49 @@
+import pickle
+import subprocess
+import sys
+
+from fspathtree import fspathtree
+
+
+class ExpressionEvaluator:
+    """A class for evaluting expressions in a separate environement."""
+
+    def __init__(self):
+        pass
+
+    def start(self):
+        pass
+
+    def eval(self, text: str):
+        pass
+
+    def stop(self):
+        pass
+
+
+class ExecExpressionEvaluator(ExpressionEvaluator):
+    """A class for evaluating expressions using the builtin exec(). BE CAREFUL!"""
+
+    def __init__(self):
+        self.globals = {}
+        self.locals = {}
+        self.token = "_configurator_xyz"
+
+        import math
+
+        import numpy
+
+        self.add_global("numpy", numpy)
+        self.add_global("math", math)
+
+    def add_global(self, name, obj):
+        self.globals[name] = obj
+
+    def eval(self, text: str):
+        for forbidden_text in ["import", "open"]:
+            if forbidden_text in text.replace(" ", ""):
+                raise RuntimeError(
+                    f"Expressions are not allowed to contain the text '{forbidden_text}'."
+                )
+        exec(f"{self.token}_result = " + text, self.globals, self.locals)
+        return self.locals[self.token + "_result"]

powerconf/graphs.py

@@ -0,0 +1,17 @@
+from fspathtree import fspathtree
+import networkx as nx
+
+
+DependencyGraph = nx.DiGraph
+
+def get_dependency_chains(graph: DependencyGraph, node:fspathtree.PathType):
+    """Given a node, return the chain of nodes that depend on it. If multiple paths exists, each is returned. If node does not have any predecessors, an empty list is returned."""
+
+    chains = []
+    for pred in graph.predecessors(node):
+        this_chain = []
+        this_chain.append(node)
+
+
+
+    pass

powerconf/parsing.py

@@ -0,0 +1,8 @@
+from pyparsing import *
+
+unbraced_variable = Combine(Literal("$") + Word(alphanums + "/_")("variable name"))
+braced_variable = Combine(
+    Literal("$") + QuotedString(quote_char="{", end_quote_char="}")("variable name")
+)
+variable = unbraced_variable | braced_variable
+expression = Combine(Literal("$") + original_text_for(nested_expr())("expression body"))

powerconf/readers.py

@@ -0,0 +1,18 @@
+import pathlib
+
+import yaml
+from fspathtree import fspathtree
+
+
+def load_yaml_docs(text_or_file: pathlib.Path | str):
+    """Load all documents in YAML file."""
+    if type(text_or_file) == str:
+        text = text_or_file
+    else:
+        text = text_or_file.read_text()
+
+    configs = []
+    for doc in text.split("---"):
+        configs.append(fspathtree(yaml.safe_load(doc)))
+
+    return configs

powerconf/rendering.py

@@ -0,0 +1,327 @@
+import copy
+import itertools
+from os.path import normpath
+from typing import Any, Dict, List
+
+import pint
+from fspathtree import fspathtree
+
+from . import expressions, graphs, parsing
+
+
+def expand_partial_configs(configs: List[fspathtree], include_base=False):
+    """
+    Give a list of configuration trees, treat the frist tree
+    as a "base" configuration and all other trees as partial configurations
+    that should be merged with the base config to create instances.
+
+    @param include_base : if true, the base config will be included in the returned set.
+    """
+    assert len(configs) > 0
+
+    if len(configs) == 1:
+        return configs
+
+    full_configs = []
+    if include_base:
+        full_configs.append(copy.deepcopy(configs[0]))
+    for c in configs[1:]:
+        full_configs.append(copy.deepcopy(configs[0]))
+        full_configs[-1].update(c)
+
+    return full_configs
+
+
+class ConfigRenderer:
+    def __init__(self, expression_evaluator=None):
+        if expression_evaluator is None:
+            expression_evaluator = expressions.ExecExpressionEvaluator()
+
+        self.expression_evaluator = expression_evaluator
+        self.ureg = pint.UnitRegistry()
+        self.Quantity = self.ureg.Quantity
+
+    def expand_and_render(self, config):
+        """Expand batch configurations and render each instance."""
+        configs = self.expand_batch_nodes(config)
+        for i in range(len(configs)):
+            configs[i] = self.render(configs[i])
+
+        return configs
+
+    def expand_batch_nodes(self, config: fspathtree):
+        """Expand @batch nodes in a configuration tree into multiple configuration trees."""
+        configs = []
+
+        batch_leaves = self._get_batch_leaves(config)
+
+        for vals in itertools.product(
+            *[config[leaf + "/@batch"] for leaf in batch_leaves.keys()]
+        ):
+            instance = copy.deepcopy(config)
+            for i, leaf in enumerate(batch_leaves.keys()):
+                instance[leaf] = vals[i]
+            configs.append(instance)
+
+        return configs
+
+    def render(self, config: fspathtree, make_copy=True):
+        """Render a configuration tree, constructing all quantities, expanding all variables, and evaluating all expressions."""
+        # create a graph of the dependencies
+        G = graphs.DependencyGraph()
+        # add all leaf node paths as nodes of the graph
+        G.add_nodes_from(config.get_all_leaf_node_paths())
+        # draw edges between nodes representing dependencies.
+        # edges point from a node to its dependency.
+        # if a node has incomming edges, other nodes depend on it
+        # if a node has outgoing edges, it depends on others.
+        for node in list(G.nodes):
+            variables = [
+                fspathtree.PathType(r["variable name"])
+                for r in parsing.variable.search_string(config[node])
+            ]
+            for v in variables:
+                dep = fspathtree.PathType(
+                    normpath(v if v.is_absolute() else node.parent / v)
+                )
+                G.add_edge(node, dep)
+        # detect circular dependencies
+        cycles = sorted(graphs.nx.simple_cycles(G))
+        if len(cycles) > 0:
+            msg = "Circular dependencies detected."
+            for cycle in cycles:
+                msg += "(" + " -> ".join(map(str, cycle)) + ")"
+            raise RuntimeError(msg)
+
+        # make a copy to work with
+
+        if make_copy:
+            rendered_config = copy.deepcopy(config)
+        else:
+            rendered_config = config
+
+        rendered_config = self._construct_all_quantities(rendered_config)
+        rendered_config = self._expand_all_variables(rendered_config)
+        rendered_config = self._evaluate_all_expressions(rendered_config, G)
+
+        return rendered_config
+
+    def _evaluate_all_expressions(
+        self, config: fspathtree, graph: graphs.DependencyGraph
+    ):
+        """Evaluate the expressions in a configuration tree, using a graph of the tree dependencies to determine the render order."""
+        # We need to determine which the order to evaluate evaluate the nodes of the config gree.
+        # We have a graph that describes the dependencies. each node is the graph is a leaf node
+        # in the tree, and the edges represent dependencies between nodes.
+        # Consider an example,
+        #
+        #          A        X
+        #        /   \    /   \
+        #       B    C   Y     Z
+        #           / \ /
+        #          D   E
+        #
+        # Assume all edgest are directed DOWN.
+        # B, D, E and Z then have no dependencies. All of their edges are "in" edges,
+        # they have no "out" edges.
+        #
+        # With networkx we can easily get a list of all nodes with no dependencies, first
+        # get the number of in and out edges for each node.
+        out_degrees = dict(graph.out_degree)
+        in_degrees = dict(graph.in_degree)
+        # any nodes that have zero out edges (out_degrees = 0) have no dependencies, and we can go ahead and render these.
+        config = self._evaluate_expressions(
+            config,
+            paths=map(
+                lambda item: item[0],
+                filter(lambda item: item[1] == 0, out_degrees.items()),
+            ),
+        )
+        # Now we need to render nodes that have dependencies, but we need to determine the order that this can be
+        # done first. In the above example, we can evaluate A unit B and C are evaluated. We can evaluate C unilt D and E
+        # are evaluated.
+        #
+        # With networkx, we can easily get a list of all ancestors of a node. For node E, this would be [C,Y,A,X]
+        # Note: with the direction of our edges, an ancestor _depends_ on the node. So C is an ancestor of E, even
+        # though it seems more natural to consider E the ancestor of C. If we wanted E to be the ancestor of C,
+        # we would need to direct our edgest to point from dependencies to dependences.
+
+        # find all root dependencies. those nodes that don't depend on any others, but are depended on by others
+        # thse are all nodes with out_degrees == 0 and in_degrees > 0
+        root_dependencies = list(
+            filter(
+                lambda k: in_degrees[k] > 0,
+                filter(lambda k: out_degrees[k] == 0, out_degrees.keys()),
+            )
+        )
+
+        # For each root node, get a list of all ancestors, and then find every path that connects the ancestor to the root.
+        # in this example above, node E would have 4 ancestors, [C,Y,A,X]. And the list of paths connecting E to its ancestors
+        # would be E -> C, E -> Y, E -> C -> A, and E -> Y -> X.
+        # However, we can't simply take one of these paths and start evaluating nodes. If we tried to evaluate E -> C -> A for example,
+        # we would get an error if B had not already been determined.
+        #
+        # What we really need to be able to do is find A and X, then work backward from there...
+        #
+        # Find all leaf node dependencies. These are nodes that only have dependencies, no dependents (out_degree > 0, in_nodes = 0)
+        leaf_dependencies = list(
+            filter(
+                lambda k: in_degrees[k] == 0,
+                filter(lambda k: out_degrees[k] > 0, out_degrees.keys()),
+            )
+        )
+        # now get a set of paths that connect each leaf node to its children
+        for leaf in leaf_dependencies:
+            all_paths = []
+            for d in graphs.nx.descendants(graph, leaf):
+                all_paths += graphs.nx.all_simple_paths(graph, leaf, d)
+
+            # node batches are sets of nodes that can all be rendered
+            # at the same time.
+            node_batches = [
+                set(filter(lambda n: n is not None, nodes))
+                for nodes in itertools.zip_longest(*all_paths)
+            ]
+            node_batches.reverse()
+            for batch in node_batches[1:]:
+                config = self._evaluate_expressions(config, paths=batch)
+
+        return config
+
+        for root in root_dependencies:
+            all_chains = []
+            for a in graphs.nx.ancestors(graph, root):
+                all_chains += graphs.nx.all_simple_paths(graph, a, root)
+            nodes_to_evaluate = []
+
+        # get all paths in the graph that end on roots
+        all_chains = []
+        for root in root_dependencies:
+            for a in graphs.nx.ancestors(graph, root):
+                all_chains += graphs.nx.all_simple_paths(graph, a, root)
+        # prune paths that are included in ohters,
+        # i.e. we only want to keep paths from each root to it's
+        # oldest ancestors.
+        longest_chains = []
+        for p1 in all_chains:
+            save = True
+            for p2 in all_chains:
+                if p1 == p2:
+                    continue
+                if p2[-len(p1) :] == p1:
+                    save = False
+                    break
+            if save:
+                longest_chains.append(p1)
+
+        # determine the order to evaluate dependencies.
+
+        for chain in longest_chains:
+            chain.reverse()  # paths start with oldest ancestor and end with root node.
+            config = self._evaluate_expressions(config, paths=chain[1:])
+
+        return config
+
+    def _evaluate_expressions(self, config: fspathtree, paths: List[Any]):
+        """Evaluate expression in the config at the paths listed. Uses as a utility function on _evaluate_all_expressions(...)"""
+        for path in paths:
+            if not self._contains_expression(config[path]):
+                continue
+
+            self.expression_evaluator.globals["ctx"] = config[path.parent]
+            expressions = parsing.expression.search_string(config[path])
+            if (
+                len(expressions) == 1
+                and "$" + expressions[0]["expression body"] == config[path]
+            ):
+                # the value of the element is a single expression with no surrounding text
+                # we want to replace the expression with the evaluation
+                e = expressions[0]
+                value = self.expression_evaluator.eval(e["expression body"][1:-1])
+                config[path] = value
+            else:
+                # we have more than one expression or the expression is surrounded by text
+                # we want to evaluate each expression and replace it with a str of its value
+                old_text = config[path]
+                new_text = ""
+                i = 0
+                for tokens, start, end in parsing.expression.scan_string(old_text):
+                    new_text += old_text[i:start]
+                    i = end
+                    new_text += str(
+                        self.expression_evaluator.eval(tokens["expression body"][1:-1])
+                    )
+                new_text += old_text[i:]
+                config[path] = new_text
+        return config
+
+    def _construct_all_quantities(self, config: fspathtree):
+        """Replace strings in the tree representing quantities with pint.Quantity objects."""
+        for path in config.get_all_leaf_node_paths():
+            if type(config[path]) == str:
+                if self._contains_expression(config[path]) or self._contains_variable(
+                    config[path]
+                ):
+                    continue
+                try:
+                    q = self.Quantity(config[path])
+                    config[path] = q
+                except:
+                    pass
+
+        return config
+
+    def _expand_all_variables(
+        self, config: fspathtree, template: str = "ctx['{name}']"
+    ):
+        """
+        Expand all shell-style variables into python variables in the entire tree.
+        """
+        for path in config.get_all_leaf_node_paths():
+            if type(config[path]) == str:
+                config[path] = self._expand_variables(config[path])
+
+        return config
+
+    def _expand_variables(self, text: Any, template: str = "ctx['{name}']"):
+        """
+        Expand shell-style variables into python variables
+
+        ${x} -> c['x']
+        ${/grid/x} -> c['/grid/x']
+        """
+        i = 0
+        new_text = ""
+        for tokens, start, end in parsing.variable.scan_string(text):
+            new_text += text[i:start]
+            i = end
+            new_text += template.format(name=tokens["variable name"])
+        new_text += text[i:]
+        return new_text
+
+    def _contains_expression(self, text: Any):
+        if type(text) is not str:
+            return False
+
+        results = parsing.expression.search_string(text)
+        return len(results) > 0
+
+    def _contains_variable(self, text: Any):
+        if type(text) is not str:
+            return False
+
+        results = parsing.variable.search_string(text)
+        return len(results) > 0
+
+    def _get_batch_leaves(self, config: fspathtree):
+        """
+        Return a list of keys in a fpathtree (nested dict/list) that are marked
+        as batch.
+        """
+        batch_leaves = dict()
+        for leaf in config.get_all_leaf_node_paths():
+            if leaf.parent.parts[-1] == "@batch":
+                batch_leaves[str(leaf.parent.parent)] = (
+                    batch_leaves.get(str(leaf.parent.parent), 0) + 1
+                )
+        return batch_leaves

powerconf/utils.py

@@ -0,0 +1,24 @@
+import hashlib
+import json
+
+from fspathtree import fspathtree
+
+
+def get_id(config: fspathtree, strip_keys=None):
+    """Return a unique id for the given configuration object."""
+    if strip_keys is None:
+        strip_keys = []
+    # make a copy of the config with only keys not in the strip list
+    c = fspathtree()
+
+    def filt(path):
+        if str(path) in strip_keys:
+            return False
+        return True
+
+    # we use a filter on the get_all_leaf_node_paths(...) here for more flexability
+    for p in config.get_all_leaf_node_paths(predicate=filt):
+        c[p] = str(config[p])
+
+    text = json.dumps(c.tree, sort_keys=True).replace(" ", "")
+    return hashlib.md5(text.encode("utf-8")).hexdigest()

powerconf-0.1.0.dist-info/METADATA

@@ -0,0 +1,59 @@
+Metadata-Version: 2.1
+Name: powerconf
+Version: 0.1.0
+Summary: 
+Author: CD Clark III
+Author-email: clifton.clark@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: fspathtree (>=0.9,<0.10)
+Requires-Dist: networkx (>=3.3,<4.0)
+Requires-Dist: numpy (>=2.0.0,<3.0.0)
+Requires-Dist: pint (>=0.24,<0.25)
+Requires-Dist: pyparsing (>=3.1.2,<4.0.0)
+Description-Content-Type: text/markdown
+
+# powerconf
+
+Powerful configuration tools for numerical simulation.
+
+`powerconf` allows you to write configuration files for things like physics simulations
+with support for variable interpolation and expression evaluation. Consider a simulation
+will solve some partial differential equation on a 2-dimensional Cartesian grid. Perhaps
+the simulation itself requires us to set the min and max range and the number of points
+to use along each axis. A simple YAML configuration for the simulation might look something
+like this
+
+```yaml
+grid:
+    x:
+        min: 0 cm
+        max: 1.5 cm
+        N: 151
+    y:
+        min: 0 cm
+        max: 1.0 cm
+        N: 101
+```
+This is fine, but it might be useful to specify the resolution to use instead of the number of points.
+With `powerconf`, we can write a configuration file that looks like this
+
+```yaml
+grid:
+    resolution: 1 um
+    x:
+        min: 0 cm
+        max: 1.5 cm
+        N: $( (${max} - ${min})/${../resolution} + 1)
+    y:
+        min: 0 cm
+        max: 1.0 cm
+        N: $( (${max} - ${min})/${../resolution} + 1)
+```
+In this example, we give a resolution to use for both x and y directions and then calculate the number
+of points to use with an expression.
+
+

powerconf-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+powerconf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+powerconf/cli.py,sha256=nGGxNrM9qmraMzHtx0E-trGOQzKhZDS-FS5kLN1QySc,1384
+powerconf/exceptions.py,sha256=yw43FCBdvnNZJA_SLb2v1IVu4WggLluIOrikpjXjLgw,216
+powerconf/expressions.py,sha256=gL6ShbFsz1qUXFTtsZeuS-Rzfkw7FLLDCeIvzrYo_f4,1186
+powerconf/graphs.py,sha256=xIog0XKMpCskxZ7CNg8M9XFgKNyhWtIh02xRvPQEt6w,472
+powerconf/parsing.py,sha256=zd7_6UNjRBesf9vwEhznGaPNygFXETQTHK1gBGXt41A,359
+powerconf/readers.py,sha256=-rs3-eDUKhvFthVXNwTjnCgUrrdOgPBSgiiogAN3nL8,401
+powerconf/rendering.py,sha256=7UK2_UrKQrl_SF6HLTfiGObQCY7mo-pQylpY9FbYdDM,13291
+powerconf/utils.py,sha256=0aox4OgvKlI74lCXDXfE7nWcW7eFR_7_tJP5Bm76QvU,718
+powerconf-0.1.0.dist-info/METADATA,sha256=6HCM2NNoITXlWvuh3nGz-NKufbTIa88m3yRfsuiQUvg,1805
+powerconf-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+powerconf-0.1.0.dist-info/entry_points.txt,sha256=_W1WDvYgt6lCJXcTO6kjkEP2E7udwLYnS1C_5q7ncho,47
+powerconf-0.1.0.dist-info/RECORD,,

powerconf-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any

powerconf-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+powerconf=powerconf.cli:app
+