dyngle 0.4.0__py3-none-any.whl → 1.5.1__py3-none-any.whl

dyngle/__init__.py

@@ -1,11 +1,16 @@
 from functools import cached_property
+from pathlib import Path
 from wizlib.app import WizApp
 from wizlib.stream_handler import StreamHandler
 from wizlib.config_handler import ConfigHandler
 from wizlib.ui_handler import UIHandler
 
 from dyngle.command import DyngleCommand
-from dyngle.expression import expression
+from dyngle.error import DyngleError
+from dyngle.model.dyngleverse import Dyngleverse
+from dyngle.model.expression import expression
+from dyngle.model.operation import Operation
+from dyngle.model.template import Template
 
 
 class DyngleApp(WizApp):
@@ -14,10 +19,39 @@ class DyngleApp(WizApp):
     name = 'dyngle'
     handlers = [StreamHandler, ConfigHandler, UIHandler]
 
-    @cached_property
-    def expressions(self):
-        expr_texts = self.config.get('dyngle-expressions')
-        if expr_texts:
-            return {k: expression(t) for k, t in expr_texts.items()}
-        else:
-            return {}
+    @property
+    def dyngleverse(self):
+        """Offload the indexing of operation and expression definitions to
+        another class. But we keep import handling here in the app because we
+        might want to upstream import/include to WizLib at some point."""
+
+        if not hasattr(self, '_dyngleverse'):
+            self._dyngleverse = Dyngleverse()
+            imports = self._get_imports(self.config, [])
+            for imported_config in imports:
+                definitions = imported_config.get('dyngle')
+                self._dyngleverse.load_config(definitions)
+            self._dyngleverse.load_config(self.config.get('dyngle'))
+        return self._dyngleverse
+
+    def _get_imports(self,
+                     config_handler: ConfigHandler,
+                     no_loops: list) -> dict:
+        imports = config_handler.get('dyngle-imports')
+        confs = []
+        if imports:
+            for filename in imports:
+                import_path = Path(filename).expanduser()
+                # If the path is relative, resolve it relative to the importing
+                # config file
+                if not import_path.is_absolute() and config_handler.file:
+                    config_dir = Path(config_handler.file).parent
+                    full_filename = (config_dir / import_path).resolve()
+                else:
+                    full_filename = import_path
+                if full_filename not in no_loops:
+                    no_loops.append(full_filename)
+                    child_handler = ConfigHandler(full_filename)
+                    confs += self._get_imports(child_handler, no_loops)
+                    confs.append(ConfigHandler(full_filename))
+        return confs

dyngle/command/__init__.py

@@ -3,4 +3,4 @@ from wizlib.command import WizCommand
 
 class DyngleCommand(WizCommand):
 
-    default = 'run'
+    default = 'null'

dyngle/command/null_command.py

@@ -0,0 +1,6 @@
+from dyngle.command import DyngleCommand
+
+
+class NullCommand(DyngleCommand):
+
+    name = 'null'

dyngle/command/run_command.py

@@ -1,11 +1,12 @@
+from functools import cached_property
 import shlex
 import subprocess
 from wizlib.parser import WizParser
 from yaml import safe_load
 
 from dyngle.command import DyngleCommand
-from dyngle.expression import expression
-from dyngle.template import Template
+from dyngle.model.expression import expression
+from dyngle.model.template import Template
 from dyngle.error import DyngleError
 
 
@@ -17,33 +18,25 @@ class RunCommand(DyngleCommand):
     @classmethod
     def add_args(cls, parser: WizParser):
         super().add_args(parser)
-        parser.add_argument('operation', help='Operation name to run')
+        parser.add_argument(
+            'operation', help='Operation name to run', nargs='?')
+        parser.add_argument(
+            'args', nargs='*', help='Optional operation arguments')
 
     def handle_vals(self):
         super().handle_vals()
-
-    def _validate_operation_exists(self, operations):
-        """Validate that the requested operation exists in configuration"""
-        if self.operation not in operations:
-            available_operations = ', '.join(operations.keys())
-            raise DyngleError(
-                f'Operation "{self.operation}" not found. " + \
-                    f"Available operations: {available_operations}')
+        keys = self.app.dyngleverse.operations.keys()
+        if not self.provided('operation'):
+            self.operation = self.app.ui.get_text('Operation: ', sorted(keys))
+            if not self.operation:
+                raise DyngleError(f"Operation required.")
+        if self.operation not in keys:
+            raise DyngleError(f"Invalid operation {self.operation}.")
 
     @DyngleCommand.wrap
     def execute(self):
-        expressions = self.app.expressions
-        operations = self.app.config.get('dyngle-operations')
-        self._validate_operation_exists(operations)
-        steps = operations[self.operation]
         data_string = self.app.stream.text
-        data = safe_load(data_string)
-        for step_template in steps:
-            step = Template(step_template).render(data, expressions)
-            parts = shlex.split(step)
-            result = subprocess.run(parts)
-            if result.returncode != 0:
-                raise DyngleError(
-                    f'Task failed with code {result.returncode}: {step}')
-
+        data = safe_load(data_string) or {}
+        operation = self.app.dyngleverse.operations[self.operation]
+        operation.run(data, self.args)
         return f'Operation "{self.operation}" completed successfully'

dyngle/model/dyngleverse.py

@@ -0,0 +1,37 @@
+from functools import cached_property
+
+from dyngle.model.expression import expression
+from dyngle.model.live_data import LiveData
+from dyngle.model.operation import Operation
+
+
+class Dyngleverse:
+    """Represents the entire immutable set of definitions for operations,
+    expresssions, and values. Operates as a sort of index/database."""
+
+    def __init__(self):
+        self.operations = {}
+        self.all_globals = {}
+
+    def load_config(self, config: dict):
+        """
+        Load additional configuration, which will always take higher precedence
+        than previously loaded configuration.
+        """
+        ops_defs = config.get('operations') or {}
+        for key, op_def in ops_defs.items():
+            operation = Operation(self, op_def, key)
+            self.operations[key] = operation
+        self.all_globals |= Dyngleverse.parse_constants(config)
+
+    @staticmethod
+    def parse_constants(definition: dict):
+        """
+        At either the global (dyngleverse) or local (within an operation)
+        level, we might find values and expressions.
+        """
+
+        expr_texts = definition.get('expressions') or {}
+        expressions = {k: expression(t) for k, t in expr_texts.items()}
+        values = definition.get('values') or {}
+        return expressions | values

dyngle/model/expression.py

@@ -0,0 +1,161 @@
+from typing import Callable
+
+from dyngle.error import DyngleError
+from dyngle.model.live_data import LiveData
+from dyngle.model.safe_path import SafePath
+
+from datetime import datetime as datetime, date, timedelta
+import math
+import json
+import re
+import yaml
+
+from dyngle.model.template import Template
+
+
+def formatted_datetime(dt: datetime, format_string=None) -> str:
+    """Safe datetime formatting using string operations"""
+    if format_string is None:
+        format_string = "{year:04d}{month:02d}{day:02d}"
+    components = {
+        'year': dt.year,
+        'month': dt.month,
+        'day': dt.day,
+        'hour': dt.hour,
+        'minute': dt.minute,
+        'second': dt.second,
+        'microsecond': dt.microsecond,
+        'weekday': dt.weekday(),  # Monday is 0
+    }
+    return format_string.format(**components)
+
+
+GLOBALS = {
+    "__builtins__": {
+        # Basic data types and conversions
+        "int": int,
+        "float": float,
+        "str": str,
+        "bool": bool,
+        "list": list,
+        "dict": dict,
+        "tuple": tuple,
+        "set": set,
+
+        # Essential functions
+        "len": len,
+        "min": min,
+        "max": max,
+        "sum": sum,
+        "abs": abs,
+        "round": round,
+        "sorted": sorted,
+        "reversed": reversed,
+        "enumerate": enumerate,
+        "zip": zip,
+        "range": range,
+        "type": type
+    },
+
+    # Mathematical operations
+    "math": math,
+
+    # Date and time handling
+    "datetime": datetime,
+    "date": date,
+    "timedelta": timedelta,
+    "formatted": formatted_datetime,
+
+    # Data parsing and manipulation
+    "json": json,
+    "yaml": yaml,
+    "re": re,
+
+    # Safe Path-like operations (within cwd)
+    "Path": SafePath
+}
+
+
+def _evaluate(expression: str, locals: dict) -> str:
+    """Evaluate a Python expression with safe globals and user data context.
+
+    Safely evaluates a Python expression string using a restricted set of
+    global functions and modules, combined with user-provided data. The
+    expression is evaluated in a sandboxed environment that includes basic
+    Python built-ins, mathematical operations, date/time handling, and data
+    manipulation utilities.
+
+    Parameters
+    ----------
+    expression : str
+        A valid Python expression string to be evaluated.
+    data : dict
+        Dictionary containing variables and values to be made available during
+        expression evaluation. Note that hyphens in keys will be replaced by
+        underscores to create valid Python names.
+
+    Returns
+    -------
+    str
+        String representation of the evaluated expression result. If the result
+        is a tuple, returns the string representation of the last element.
+
+    Raises
+    ------
+    DyngleError
+        If the expression contains invalid variable names that are not found in
+        the provided data dictionary or global context.
+    """
+    try:
+        result = eval(expression, GLOBALS, locals)
+    except KeyError as error:
+        raise DyngleError(f"The following expression contains " +
+                          f"invalid name '{error}:\n{expression}")
+
+    # Allow the use of a comma to separate sub-expressions, which can then use
+    # warus to set values, and only the last exxpression in the list returns a
+    # value.
+    result = result[-1] if isinstance(result, tuple) else result
+
+    return result
+
+
+# The 'expression' function returns the expression object itself, which is
+# really just a function.
+
+def expression(text: str) -> Callable[[dict], str]:
+    """Generate an expression, which is a function based on a string
+    expression"""
+
+    def definition(live_data: LiveData | dict | None = None) -> str:
+        """The expression function itself"""
+
+        # We only work if passed some data to use - also we don't know our name
+        # so can't report it.
+
+        if live_data is None:
+            raise DyngleError('Expression called with no argument')
+
+        # Translate names to underscore-separated instead of hyphen-separated
+        # so they work within the Python namespace.
+
+        items = live_data.items() if live_data else ()
+        locals = LiveData({k.replace('-', '_'): v for k, v in items})
+
+        # Create a resolve function which allows references using the hyphen
+        # syntax too - note it relies on the original live_data object (not the
+        # locals with the key replacement). We're converting it to LiveData in
+        # case for some reason we were passed a raw dict.
+
+        live_data = LiveData(live_data)
+
+        def resolve(key):
+            return live_data.resolve(key, str_only=False)
+
+        # Passing the live_data in again allows function(data) in expressions
+        locals = locals | {'resolve': resolve, 'data': live_data}
+
+        # Perform the Python eval, expanded above
+        return _evaluate(text, locals)
+
+    return definition

dyngle/model/live_data.py

@@ -0,0 +1,38 @@
+from collections import UserDict
+from datetime import date, timedelta
+from numbers import Number
+
+from yaml import safe_dump
+
+from dyngle.error import DyngleError
+from dyngle.model.safe_path import SafePath
+
+
+class LiveData(UserDict):
+
+    def resolve(self, key: str, str_only: bool = True):
+        """Given a key (which might be dot-separated), return
+        the value (which might include evaluating expressions)."""
+
+        parts = key.split('.')
+        current = self.data
+        for part in parts:
+            if part not in current:
+                raise DyngleError(
+                    f"Invalid expression or data reference '{key}'")
+            current = current[part]
+        result = current(self) if callable(current) else current
+        return _stringify(result) if str_only else result
+
+
+def _stringify(value) -> str:
+    if isinstance(value, bool):
+        return '.' if value is True else ''
+    elif isinstance(value, (Number, str, date, timedelta, SafePath)):
+        return str(value)
+    elif isinstance(value, (list, dict, tuple)):
+        return safe_dump(value)
+    elif isinstance(value, set):
+        return safe_dump(list(value))
+    else:
+        raise DyngleError(f'Unable to serialize value of type {type(value)}')

dyngle/model/operation.py

@@ -0,0 +1,127 @@
+from dataclasses import dataclass
+from functools import cached_property
+import re
+import shlex
+import subprocess
+
+from dyngle.error import DyngleError
+from dyngle.model.live_data import LiveData
+from dyngle.model.template import Template
+
+
+class Operation:
+    """A named operation defined in configuration. Can be called from a Dyngle
+    command (i.e. `dyngle run`) or as a sub-operation."""
+
+    all_locals = {}
+
+    def __init__(self, dyngleverse, definition: dict | list, key: str):
+        """
+        definition: Either a dict containing steps and local
+        expressions/values, or a list containing only steps
+        """
+        self.dyngleverse = dyngleverse
+        if isinstance(definition, list):
+            steps_def = definition
+        elif isinstance(definition, dict):
+            steps_def = definition.get('steps') or []
+            self.all_locals = dyngleverse.parse_constants(definition)
+        self.sequence = Sequence(dyngleverse, self, steps_def)
+
+    def run(self, data: dict | LiveData, args: list):
+        """
+        data - The main set of data going into the operation
+
+        args - Arguments to the operation
+        """
+
+        # The tye of data tells us the run condition - if already a LiveData
+        # object then we don't recreate it (i.e. sub-operation)
+
+        if not isinstance(data, LiveData):
+            live_data = LiveData(data) | self.dyngleverse.all_globals
+        else:
+            live_data = data
+        live_data |= self.all_locals | {'args': args}
+        self.sequence.run(live_data)
+
+
+class Sequence:
+    """We allow for the possibility that a sequence of steps might run at other
+    levels than the operation itself, for example in a conditional block."""
+
+    def __init__(self, dyngleverse, operation: Operation, steps_def: list):
+        self.steps = [Step.parse_def(dyngleverse, d) for d in steps_def]
+
+    def run(self, live_data: LiveData):
+        for step in self.steps:
+            step.run(live_data)
+
+
+class Step:
+
+    @staticmethod
+    def parse_def(dyngleverse, definition: dict | str):
+        for step_type in [CommandStep, SubOperationStep]:
+            if step_type.fits(definition):
+                return step_type(dyngleverse, definition)
+        raise DyngleError(f"Unknown step definition\n{definition}")
+
+
+# Ideally these would be subclasses in a ClassFamily (or use an ABC)
+
+class CommandStep:
+
+    PATTERN = re.compile(
+        r'^\s*(?:([\w.-]+)\s+->\s+)?(.+?)(?:\s+=>\s+([\w.-]+))?\s*$')
+
+    @classmethod
+    def fits(cls, definition: dict | str):
+        return isinstance(definition, str)
+
+    def __init__(self, dyngleverse, markup: str):
+        self.markup = markup
+        if match := self.PATTERN.match(markup):
+            self.input, command_text, self.output = match.groups()
+            command_template = shlex.split(command_text.strip())
+            self.command_template = command_template
+        else:
+            raise DyngleError(f"Invalid step markup {{markup}}")
+
+    def run(self, live_data: LiveData):
+        command = [Template(word).render(live_data).strip()
+                   for word in self.command_template]
+        pipes = {}
+        if self.input:
+            pipes["input"] = live_data.resolve(self.input)
+        if self.output:
+            pipes['stdout'] = subprocess.PIPE
+        result = subprocess.run(command, text=True, **pipes)
+        if result.returncode != 0:
+            raise DyngleError(
+                f'Step failed with code {result.returncode}: {self.markup}')
+        if self.output:
+            live_data[self.output] = result.stdout.rstrip()
+
+
+class SubOperationStep:
+    """Instead of calling a system command, call another operation in the same
+    Dyngleverse"""
+
+    @classmethod
+    def fits(cls, definition: dict | str):
+        return isinstance(definition, dict) and 'sub' in definition
+
+    def __init__(self, dyngleverse, definition: dict):
+        self.dyngleverse = dyngleverse
+        self.operation_key = definition['sub']
+        self.args_template = definition.get('args') or ''
+
+    def run(self, live_data: LiveData):
+        # Resolve the operation at runtime, not at init time
+        operation = self.dyngleverse.operations.get(self.operation_key)
+        if not operation:
+            raise DyngleError(f"Unknown operation {self.operation_key}")
+        args = [Template(word).render(live_data).strip()
+                for word in self.args_template]
+        operation.run(live_data, args)

dyngle/model/template.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from functools import partial
+import re
+
+from dyngle.error import DyngleError
+from dyngle.model.live_data import LiveData
+
+
+PATTERN = re.compile(r'\{\{\s*([^}]+)\s*\}\}')
+
+
+@dataclass
+class Template:
+
+    template: str
+
+    def render(self, live_data: LiveData | dict | None = None) -> str:
+        """Render the template with the provided LiveData (raw data and
+        expressions)."""
+
+        live_data = LiveData(live_data)
+        resolver = partial(self._resolve, live_data=live_data)
+        return PATTERN.sub(resolver, self.template)
+
+    def _resolve(self, match, *, live_data: LiveData):
+        """Resolve a single name/path from the template. The argument is a
+        merge of the raw data and the expressions, either of which are valid
+        substitutions."""
+        key = match.group(1).strip()
+        return live_data.resolve(key)

dyngle-1.5.1.dist-info/METADATA

@@ -0,0 +1,316 @@
+Metadata-Version: 2.4
+Name: dyngle
+Version: 1.5.1
+Summary: Run lightweight local workflows
+License: MIT
+Author: Steampunk Wizard
+Author-email: dyngle@steamwiz.io
+Requires-Python: >=3.13,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: wizlib (>=3.3.11,<3.4.0)
+Description-Content-Type: text/markdown
+
+# Dyngle
+
+An experimantal, lightweight, easily configurable workflow engine for
+automating development, operations, data processing, and content management
+tasks.
+
+Technical foundations
+
+- Configuration, task definition, and flow control in YAML
+- Operations as system commands using a familiar shell-like syntax
+- Expressions and logic in pure Python
+
+## Quick installation (MacOS)
+
+```bash
+brew install python@3.11
+python3.11 -m pip install pipx
+pipx install dyngle
+```
+
+## Getting started
+
+Create a file `.dyngle.yml`:
+
+```yaml
+dyngle:
+  operations:
+    hello:
+      - echo "Hello world"
+```
+
+Run an operation:
+
+```bash
+dyngle run hello
+```
+
+## Configuration
+
+Dyngle reads configuration from YAML files. Specify the config file location using any of the following (in order of precedence):
+
+1. A `--config` command line option, OR
+2. A `DYNGLE_CONFIG` environment variable, OR
+3. `.dyngle.yml` in current directory, OR
+4. `~/.dyngle.yml` in home directory
+
+## Operations
+
+Operations are defined under `dyngle:` in the configuration. In its simplest form, an Operation is a YAML array defining the Steps, as system commands with space-separated arguments. In that sense, a Dyngle operation looks something akin to a phony Make target, a short Bash script, or a CI/CD job.
+
+As a serious example, consider the `init` operation from the Dyngle configuration delivered with the project's source code.
+
+```yaml
+dyngle:
+  operations:
+    init:
+      - rm -rf .venv
+      - python3.11 -m venv .venv
+      - .venv/bin/pip install --upgrade pip poetry
+```
+
+The elements of the YAML array _look_ like lines of Bash, but Dyngle processes them directly as system commands, allowing for template substitution and Python expression evaluation (described below). So shell-specific syntax such as `|`, `>`, and `$VARIABLE` won't work.
+
+## Data and Templates
+
+Dyngle maintains a block of "Live Data" throughout an operation, which is a set of named values (Python `dict`, YAML "mapping"). The values are usually strings but can also be other data types that are valid in both YAML and Python.
+
+The `dyngle run` command feeds the contents of stdin to the Operation as Data, by converting a YAML mapping to named Python values. The values may be substituted into commands or arguments in Steps using double-curly-bracket syntax (`{{` and `}}`) similar to Jinja2.
+
+For example, consider the following configuration:
+
+``` yaml
+dyngle:
+  operations:
+    hello:
+      - echo "Hello {{name}}!"
+```
+
+Cram some YAML into stdin to try it in your shell:
+
+```bash
+echo "name: Francis" | dyngle run hello
+```
+
+The output will say:
+
+```text
+Hello Francis!
+```
+
+## Expressions
+
+Operations may contain Expressions, written in Python, that can be referenced in Operation Step Templates using the same syntax as for Data. In the case of a naming conflict, an Expression takes precedence over Data with the same name. Expressions can reference names in the Data directly.
+
+Expressions may be defined in either of two ways in the configuration:
+
+1. Global Expressions, under the `dyngle:` mapping, using the `expressions:` key.
+2. Local Expressions, within a single Operation, in which case the Steps of the operation require a `steps:` key.
+
+Here's an example of a global Expression
+
+```yaml
+dyngle:
+  expressions:
+    count: len(name)    
+  operations:
+    say-hello:
+      - echo "Hello {{name}}! Your name has {{count}} characters."
+```
+
+For completeness, consider the following example using a local Expression for the same purpose.
+
+```yaml
+dyngle:
+  operations:
+    say-hello:
+      expressions:
+        count: len(name)
+      steps:
+        - echo "Hello {{name}}! Your name has {{count}} characters."
+```
+
+Expressions can use a controlled subset of the Python standard library, including:
+
+- Built-in data types such as `str()`
+- Essential built-in functions such as `len()`
+- The core modules from the `datetime` package (but some methods such as `strftime()` will fail)
+- A specialized function called `formatted()` to perform string formatting operations on a `datetime` object
+- A restricted version of `Path()` that only operates within the current working directory
+- Various other useful utilities, mostly read-only, such as the `math` module
+- A special function called `resolve` which resolves data expressions using the same logic as in templates
+- An array `args` containing arguments passed to the `dyngle run` command after the Operation name
+
+**NOTE** Some capabilities of the Expression namespace might be limited in the future. The goal is support purely read-only operations within Expressions.
+
+Expressions behave like functions that take no arguments, using the Data as a namespace. So Expressions reference Data directly as local names in Python.
+
+YAML keys can contain hyphens, which are fully supported in Dyngle. To reference a hyphenated key in an Expression, choose:
+
+- Reference the name using underscores instead of hyphens (they are automatically replaced), OR
+- Use the built-in special-purpose `resolve()` function (which can also be used to reference other expressions)
+
+```yaml
+dyngle:
+  expressions:
+    say-hello: >-
+        'Hello ' + full_name + '!'
+```
+
+... or using the `resolve()` function, which also allows expressions to essentially call other expressions, using the same underlying data set.
+
+```yaml
+dyngle:
+  expressions:
+    hello: >-
+        'Hello ' + resolve('formal-name') + '!'
+    formal-name: >-
+        'Ms. ' + full_name
+```
+
+Note it's also _possible_ to call other expressions by name as functions, if they only return hard-coded values (i.e. constants).
+
+```yaml
+dyngle:
+  expressions:
+    author-name: Francis Potter
+    author-hello: >-
+        'Hello ' + author_name()
+``` 
+
+Here are some slightly more sophisticated exercises using Expression reference syntax:
+
+```yaml
+dyngle:
+  operations:
+    reference-hyphenated-data-key:
+      expressions:
+        spaced-name: "' '.join([x for x in first_name])"
+        count-name: len(resolve('first-name'))
+        x-name: "'X' * int(resolve('count-name'))"
+      steps:
+        - echo "Your name is {{first-name}} with {{count-name}} characters, but I will call you '{{spaced-name}}' or maybe '{{x-name}}'"
+    reference-expression-using-function-syntax:
+      expressions:
+        name: "'George'"
+        works: "name()"
+        double: "name * 2"
+        fails: double()
+      steps:
+        - echo "It works to call you {{works}}"
+        # - echo "I have trouble calling you {{fails}}"
+```
+
+Finally, here's an example using args:
+
+```yaml
+dyngle:
+  operations:
+    name-from-arg:
+      expressions:
+        name: "args[0]"
+      steps:
+        - echo "Hello {{name}}"
+```
+
+## Passing values between Steps in an Operation
+
+The Steps parser supports two special operators designed to move data between Steps in an explicit way.
+
+- The data assignment operator (`=>`) assigns the contents of stdout from the command to an element in the data
+- The data input operator (`->`) assigns the value of an element in the data (or an evaluated expression) to stdin for the command
+
+The operators must appear in order in the step and must be isolated with whitespace, i.e.
+
+```
+<input-variable-name> -> <command and arguments> => <output-variable-name>
+```
+
+Here we get into more useful functionality, where commands can be strung together in meaningful ways without the need for Bash.
+
+```yaml
+dyngle:
+  operations:
+    weather:
+      - curl -s "https://api.open-meteo.com/v1/forecast?latitude=52.52&longitude=13.41&current_weather=true" => weather-data
+      - weather-data -> jq -j '.current_weather.temperature' => temperature
+      - echo "It's {{temperature}} degrees out there!"
+```
+
+If names overlap, data items populated using the data assignment operator take precedence over expressions and data in the original input from the beginning of the Operation.
+
+## Sub-operations
+
+Operations can call other operations as steps using the `sub:` key. This allows for composability and reuse of operation logic.
+
+Basic example:
+
+```yaml
+dyngle:
+  operations:
+    greet:
+      - echo "Hello!"
+    
+    greet-twice:
+      steps:
+        - sub: greet
+        - sub: greet
+```
+
+Sub-operations can accept arguments using the `args:` key. The called operation can access these via the `args` array in expressions:
+
+```yaml
+dyngle:
+  operations:
+    greet-person:
+      expressions:
+        person: "args[0]"
+      steps:
+        - echo "Hello, {{person}}!"
+    
+    greet-team:
+      steps:
+        - sub: greet-person
+          args: ['Alice']
+        - sub: greet-person
+          args: ['Bob']
+```
+
+Sub-operations share the same Live Data context, so data assignments and expressions from the parent operation are available to the sub-operation, and any data populated by the sub-operation is available to subsequent steps in the parent.
+
+## Lifecycle
+
+The lifecycle of an operation is:
+
+1. Load Data if it exists from YAML on stdin (if no tty)
+2. Find the named Operation in the configuration
+2. Perform template rendering on the first Step, using Data and Expressions
+3. Execute the Step in a subprocess, passing in an input value and populating an output value in the Data
+4. Continue with the next Step
+
+Note that operations in the config are _not_ full shell lines. They are passed directly to the system.
+
+## Imports
+
+Configuration files can import other configuration files, by providing an entry `imports:` with an array of filepaths. The most obvious example is a Dyngle config in a local directory which imports the user-level configuration.
+
+```yaml
+dyngle:
+  imports:
+    - ~/.dyngle.yml
+  expressions:
+  operations:
+```
+
+In the event of item name conflicts, expressions and operations are loaded from imports in the order specified, so imports lower in the array will override those higher up. The expressions and operations defined in the main file override the imports. Imports are not recursive.
+
+## Security
+
+Commands are executed using Python's `subprocess.run()` with arguments split in a shell-like fashion. The shell is not used, which reduces the likelihood of shell injection attacks. However, note that Dyngle is not robust to malicious configuration. Use with caution.
+

dyngle-1.5.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+dyngle/__init__.py,sha256=BxgXnQ_cyEuZiMpkv-zkDCTRs9UP8vZsKX2kWeZj_Ck,2336
+dyngle/__main__.py,sha256=pYRIwzix_AL8CdJaDDis_8yMBBWO2N72NNwkroo1dQo,95
+dyngle/command/__init__.py,sha256=ngNOb_k9COcXOs7It3HoFJRW0hzBDpAzxXcGUy6hhko,95
+dyngle/command/null_command.py,sha256=OX1u0z4zjlquxuV0_yu7uE1_K2Lk523WydJu-0Z82QE,96
+dyngle/command/run_command.py,sha256=YOYuTzR1l2t24G1Z2uLjl1Ya1unvE4lHWpaOopMO1Tg,1441
+dyngle/error.py,sha256=CGcTa8L4O1qsHEYnzp_JBbkvntJTv2Qz46wj_TI8NLk,39
+dyngle/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dyngle/model/dyngleverse.py,sha256=tFzx4AhPalODknPqbuFcR4G7QF1VOPlAAy2VDPfRmTA,1301
+dyngle/model/expression.py,sha256=XvHlWqnN-4gkN4ck5iTuwP3OqXXbRzTwIRTX0lzJAdY,4889
+dyngle/model/live_data.py,sha256=FxbMjfaiBIUorEbhRx5I0o-WAFFWdjYaqzw_zhFq86w,1251
+dyngle/model/operation.py,sha256=IrLx6iDZ2_D7384qKsjpS7VBrHqTsSmsOWw_lJFWimI,4465
+dyngle/model/safe_path.py,sha256=Hk2AhP6e3yKGh3kKrLLwhvAlMNx-j2jObBYJL-_doAU,3339
+dyngle/model/template.py,sha256=MeXu--ZNtj_ujABU1GjjcQ1Ea_o_M-50LocuXFeOLRE,887
+dyngle-1.5.1.dist-info/METADATA,sha256=kdyZXIVC7aEXcVvN9gMLweEfpqvh3nlkB2LVSGX-QmQ,10681
+dyngle-1.5.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+dyngle-1.5.1.dist-info/entry_points.txt,sha256=rekiGhtweiHKm9g1jdGb3FhzqDrk1kigJDeSNollZSA,48
+dyngle-1.5.1.dist-info/RECORD,,

{dyngle-0.4.0.dist-info → dyngle-1.5.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any

dyngle/expression.py

@@ -1,88 +0,0 @@
-from typing import Callable
-
-from dyngle.error import DyngleError
-from dyngle.safe_path import SafePath
-
-from datetime import datetime as datetime, date, timedelta
-import math
-import json
-import re
-import yaml
-
-
-def formatted_datetime(dt: datetime, format_string=None) -> str:
-    """Safe datetime formatting using string operations"""
-    if format_string is None:
-        format_string = "{year:04d}{month:02d}{day:02d}"
-    components = {
-        'year': dt.year,
-        'month': dt.month,
-        'day': dt.day,
-        'hour': dt.hour,
-        'minute': dt.minute,
-        'second': dt.second,
-        'microsecond': dt.microsecond,
-        'weekday': dt.weekday(),  # Monday is 0
-    }
-    return format_string.format(**components)
-
-
-GLOBALS = {
-    "__builtins__": {
-        # Basic data types and conversions
-        "int": int,
-        "float": float,
-        "str": str,
-        "bool": bool,
-        "list": list,
-        "dict": dict,
-        "tuple": tuple,
-        "set": set,
-
-        # Essential functions
-        "len": len,
-        "min": min,
-        "max": max,
-        "sum": sum,
-        "abs": abs,
-        "round": round,
-        "sorted": sorted,
-        "reversed": reversed,
-        "enumerate": enumerate,
-        "zip": zip,
-        "range": range,
-    },
-
-    # Mathematical operations
-    "math": math,
-
-    # Date and time handling
-    "datetime": datetime,
-    "date": date,
-    "timedelta": timedelta,
-    "formatted": formatted_datetime,
-
-    # Data parsing and manipulation
-    "json": json,
-    "yaml": yaml,
-    "re": re,
-
-    # Safe Path-like operations (within cwd)
-    "Path": SafePath
-}
-
-
-def _evaluate(expression: str, data: dict) -> str:
-    try:
-        result = eval(expression, GLOBALS, data)
-    except KeyError:
-        raise DyngleError(f"The following expression contains " +
-                          f"at least one invalid name: {expression}")
-    result = result[-1] if isinstance(result, tuple) else result
-    return str(result)
-
-
-def expression(text: str) -> Callable[[dict], str]:
-    def evaluate(data: dict) -> str:
-        return _evaluate(text, data)
-    return evaluate

dyngle/template.py

@@ -1,29 +0,0 @@
-from dataclasses import dataclass
-from functools import partial
-import re
-
-
-PATTERN = re.compile(r'\{\{\s*([^}]+)\s*\}\}')
-
-
-@dataclass
-class Template:
-
-    template: str
-
-    def render(self, data: dict, expressions: dict = None) -> str:
-        """Render the template with the provided data."""
-        resolver = partial(self._resolve, data=data, expressions=expressions)
-        return PATTERN.sub(resolver, self.template)
-
-    def _resolve(self, match, *, data: dict, expressions: dict):
-        """Resolve a single name/path from the template."""
-        key = match.group(1).strip()
-        if expressions and (key in expressions):
-            return expressions[key](data)
-        else:
-            parts = key.split('.')
-            current = data
-            for part in parts:
-                current = current[part]
-            return current

dyngle-0.4.0.dist-info/METADATA

@@ -1,107 +0,0 @@
-Metadata-Version: 2.3
-Name: dyngle
-Version: 0.4.0
-Summary: Run lightweight local workflows
-License: MIT
-Author: Steampunk Wizard
-Author-email: dyngle@steamwiz.io
-Requires-Python: >=3.11,<3.12
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: requests (>=2.32.3,<3.0.0)
-Requires-Dist: wizlib (>=3.1.4,<4.0.0)
-Description-Content-Type: text/markdown
-
-# Dyngle
-
-## Run lightweight local workflows
-
-Dyngle is a simple workflow runner that executes sequences of commands defined in configuration files. It's like a lightweight combination of Make and a task runner, designed for automating common development and operational tasks.
-
-## Basic usage
-
-Create a configuration file (e.g., `.dyngle.yml`) with your workflows:
-
-```yaml
-dyngle:
-  operations:
-    build:
-      - python -m pip install -e .
-      - python -m pytest
-    deploy:
-      - docker build -t myapp .
-      - docker push myapp
-    clean:
-      - rm -rf __pycache__
-      - rm -rf .pytest_cache
-```
-
-Run an operation:
-
-```bash
-dyngle run build
-```
-
-## Configuration
-
-Dyngle reads configuration from YAML files. You can specify the config file location using:
-
-- `--config` command line option
-- `DYNGLE_CONFIG` environment variable  
-- `.dyngle.yml` in current directory
-- `~/.dyngle.yml` in home directory
-
-## Workflow structure
-
-Each operation is defined as a list of tasks under `dyngle.operations`. Tasks are executed sequentially using Python's subprocess module for security.
-
-Example with multiple operations:
-
-```yaml
-dyngle:
-  operations:
-    test:
-      - python -m unittest discover
-      - python -m coverage report
-    docs:
-      - sphinx-build docs docs/_build
-      - open docs/_build/index.html
-    setup:
-      - python -m venv venv
-      - source venv/bin/activate
-      - pip install -r requirements.txt
-```
-
-## Expressions
-
-Configs can also contain expressions.
-
-```yaml
-dyngle:
-  expressions:
-    say-hello: >-
-        'Hello ' + name + '!'
-  operations:
-    say-hello: echo {{say-hello}}
-```
-Expressions can use a controlled subset of the Python standard library, including:
-
-- Built-in data types such as `str()`
-- Essential built-in functions such as `len()`
-- The core modules from the `datetime` package (but some methods such as `strftime()` will fail)
-- A specialized function called `formatted()` to perform string formatting operations on a `datetime` object
-- A restricted version of `Path()` that only operates within the current working directory
-
-## Security
-
-Commands are executed using Python's `subprocess.run()` with arguments split in a shell-like fashion. The shell is not used, which reduces the likelihood of shell injection attacks. However, note that Dyngle is not robust to malicious configuration. 
-
-## Quick installation (MacOS)
-
-```bash
-brew install python@3.11
-python3.11 -m pip install pipx
-pipx install dyngle
-```
-

dyngle-0.4.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-dyngle/__init__.py,sha256=drqd18F3T7LMTNqAU38WYjanczl1kANAA1vVBjbPEyU,653
-dyngle/__main__.py,sha256=pYRIwzix_AL8CdJaDDis_8yMBBWO2N72NNwkroo1dQo,95
-dyngle/command/__init__.py,sha256=1S86gbef8MYvG-TWD5JRIWzFg7qV5xKhp9QXx9zEx5c,94
-dyngle/command/run_command.py,sha256=0E8Bg7TRa8qXdfZNyC4Wygb1cEG4-ejRJ5ZfYz_L1_Y,1718
-dyngle/error.py,sha256=CGcTa8L4O1qsHEYnzp_JBbkvntJTv2Qz46wj_TI8NLk,39
-dyngle/expression.py,sha256=x-2Ald34lmXfG8070H96Wszln5Rd1AWGyYuYtf7iZFA,2134
-dyngle/safe_path.py,sha256=Hk2AhP6e3yKGh3kKrLLwhvAlMNx-j2jObBYJL-_doAU,3339
-dyngle/template.py,sha256=5R8gUrv6BhppchlMqr_SZtkSVsUo8FYseCw4ETAJxeE,855
-dyngle-0.4.0.dist-info/METADATA,sha256=Vx4RonlVHvFKAW2y1a_OD8_XUvyI2LSI0CMf2tZ8KsM,2941
-dyngle-0.4.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-dyngle-0.4.0.dist-info/entry_points.txt,sha256=rekiGhtweiHKm9g1jdGb3FhzqDrk1kigJDeSNollZSA,48
-dyngle-0.4.0.dist-info/RECORD,,