dyngle 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

dyngle/command/run_command.py

@@ -34,21 +34,14 @@ class RunCommand(DyngleCommand):
 
     @DyngleCommand.wrap
     def execute(self):
-        operations = self.app.operations
-        self._validate_operation_exists(operations)
-        operation = operations[self.operation]
-        steps = operation.steps
-        expressions = self.app.global_expressions | \
-            operation.local_expressions
         data_string = self.app.stream.text
         data = safe_load(data_string) or {}
         data['args'] = self.args
-        for step_template in steps:
-            step = Template(step_template).render(data, expressions)
-            parts = shlex.split(step)
-            result = subprocess.run(parts)
-            if result.returncode != 0:
-                raise DyngleError(
-                    f'Task failed with code {result.returncode}: {step}')
+
+        operations = self.app.operations
+        self._validate_operation_exists(operations)
+        operation = operations[self.operation]
+
+        operation.run(data, self.app.global_expressions)
 
         return f'Operation "{self.operation}" completed successfully'

dyngle/operation.py

@@ -1,4 +1,11 @@
 from dataclasses import dataclass
+from functools import cached_property
+import re
+import shlex
+import subprocess
+
+from dyngle.error import DyngleError
+from dyngle.template import Template
 
 
 @dataclass
@@ -7,3 +14,47 @@ class Operation:
     local_expressions: dict
 
     steps: list
+
+    def run(self, data: dict, global_expressions: dict):
+        # The data dict is mutable
+        steps = self.steps
+        expressions = global_expressions | self.local_expressions
+        for markup in steps:
+            step = Step(markup)
+            step.run(data, expressions)
+
+
+STEP_PATTERN = re.compile(
+    r'^\s*(?:([\w.-]+)\s+->\s+)?(.+?)(?:\s+=>\s+([\w.-]+))?\s*$')
+
+
+def parse_step(markup):
+    if match := STEP_PATTERN.match(markup):
+        input, command_text, output = match.groups()
+        command_template = shlex.split(command_text.strip())
+        return input, command_template, output
+    else:
+        raise DyngleError(f"Invalid step markup {{markup}}")
+
+
+@dataclass
+class Step:
+
+    markup: str
+
+    def __post_init__(self):
+        self.input, self.command_template, self.output = \
+            parse_step(self.markup)
+
+    def run(self, data, expressions):
+        command = [Template(word).render(data, expressions)
+                   for word in self.command_template]
+        pipes = {}
+        if self.output:
+            pipes['stdout'] = subprocess.PIPE
+        result = subprocess.run(command, text=True, **pipes)
+        if result.returncode != 0:
+            raise DyngleError(
+                f'Step failed with code {result.returncode}: {self.markup}')
+        if self.output:
+            data[self.output] = result.stdout

dyngle/template.py

@@ -48,7 +48,7 @@ class Template:
         for part in parts:
             if part not in current:
                 raise DyngleError(
-                    f"Invalid expression or data reference '{key}")
+                    f"Invalid expression or data reference '{key}'")
             current = current[part]
         if callable(current):
             return current(live_data)

dyngle-0.7.0.dist-info/METADATA

@@ -0,0 +1,276 @@
+Metadata-Version: 2.3
+Name: dyngle
+Version: 0.7.0
+Summary: Run lightweight local workflows
+License: MIT
+Author: Steampunk Wizard
+Author-email: dyngle@steamwiz.io
+Requires-Python: >=3.11,<3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: wizlib (>=3.1.4,<4.0.0)
+Description-Content-Type: text/markdown
+
+# Dyngle
+
+Use cases
+
+- A task runner 
+- A lightweight workflow engine
+- A replacement for Make in Python projects
+- A replacement for short functions in RC files
+- Freedom from quirky Bash syntax
+
+Technical foundations
+
+- Configuration, task definition, and flow control in YAML
+- Operations as system commands using a familiar shell-like syntax
+- Expressions and logic in pure Python
+
+## Quick installation (MacOS)
+
+```bash
+brew install python@3.11
+python3.11 -m pip install pipx
+pipx install dyngle
+```
+
+## Getting started
+
+Create a file `.dyngle.yml`:
+
+```yaml
+dyngle:
+  operations:
+    hello:
+      - echo "Hello world"
+```
+
+Run an operation:
+
+```bash
+dyngle run hello
+```
+
+## Configuration
+
+Dyngle reads configuration from YAML files. You can specify the config file location using:
+
+- `--config` command line option
+- `DYNGLE_CONFIG` environment variable  
+- `.dyngle.yml` in current directory
+- `~/.dyngle.yml` in home directory
+
+## Operations
+
+Operations are defined under `dyngle:` in the configuration. In its simplest form, an Operation takes the form of a YAML array defining the steps, as a system command with space-separated arguments. In that sense, a Dyngle operation looks something akin to a "PHONY" Make target, a short Bash script, or a CI/CD job. As a serious example, consider the `init` operation from the Dyngle configuration delivered with the project's source code.
+
+```yaml
+dyngle:
+  operations:
+    init:
+      - rm -rf .venv
+      - python3.11 -m venv .venv
+      - .venv/bin/pip install --upgrade pip poetry
+```
+
+The elements of the YAML array _look_ like lines of Bash, but Dyngle replaces the shell with YAML-based flow control and Python-based expressions (described below). So shell-specific operators such as `|`, `>`, and `$VARIABLES` won't work.
+
+## Data and Templates
+
+Dyngle maintains a block of Data throughout an operation, which is a set of named values (Python `dict`, YAML "mapping"). The values are usually strings but can also be other data types that are valid in both YAML and Python.
+
+The `dyngle run` command feeds the contents of stdin to the Operation as Data, by converting a YAML mapping to named Python values. The values may be substituted into commands or arguments in Steps using double-curly-bracket syntax (`{{` and `}}`) similar to Jinja2.
+
+For example, consider the following configuration:
+
+``` yaml
+dyngle:
+  operations:
+    hello:
+      - echo "Hello {{name}}!"
+```
+
+Cram some YAML into stdin to try it in your shell:
+
+```bash
+echo "name: Francis" | dyngle run hello
+```
+
+The output will say:
+
+```text
+Hello Francis!
+```
+
+## Expressions
+
+Operations may contain Expressions, written in Python, that can be referenced in operation steps using the same syntax as for Data. In the case of a naming conflict, an Expression takes precedence over Data with the same name. Expressions can reference names in the Data directly.
+
+Expressions may be defined in either of two ways in the configuration:
+
+1. Global Expressions, under the `dyngle:` mapping, using the `expressions:` key.
+2. Local Expressions, within a single Operation, in which case the Steps of the operation require a `steps:` key.
+
+Here's an example of a global Expression
+
+```yaml
+dyngle:
+  expressions:
+    count: len(name)    
+  operations:
+    say-hello:
+      - echo "Hello {{name}}! Your name has {{count}} characters."
+```
+
+For completeness, consider the following example using a local Expression for the same purpose.
+
+```yaml
+dyngle:
+  operations:
+    say-hello:
+      expressions:
+        count: len(name)
+      steps:
+        - echo "Hello {{name}}! Your name has {{count}} characters."
+```
+
+Expressions can use a controlled subset of the Python standard library, including:
+
+- Built-in data types such as `str()`
+- Essential built-in functions such as `len()`
+- The core modules from the `datetime` package (but some methods such as `strftime()` will fail)
+- A specialized function called `formatted()` to perform string formatting operations on a `datetime` object
+- A restricted version of `Path()` that only operates within the current working directory
+- Various other useful utilities, mostly read-only, such as the `math` module
+- A special function called `resolve` which resolves data expressions using the same logic as in templates
+- An array `args` containing arguments passed to the `dyngle run` command after the Operation name
+
+**NOTE** Some capabilities of the Expression namespace might be limited in the future. The goal is support purely read-only operations within Expressions.
+
+Expressions behave like functions that take no arguments, using the Data as a namespace. So Expressions reference Data directly as local names in Python.
+
+YAML keys can contain hyphens, which are fully supported in Dyngle. To reference a hyphenated key in an Expression, choose:
+
+- Reference the name using underscores instead of hyphens (they are automatically replaced), OR
+- Use the built-in special-purpose `resolve()` function (which can also be used to reference other expressions)
+
+
+
+```yaml
+dyngle:
+  expressions:
+    say-hello: >-
+        'Hello ' + full_name + '!'
+```
+
+... or using the `resolve()` function, which also allows expressions to essentially call other expressions, using the same underlying data set.
+
+```yaml
+dyngle:
+  expressions:
+    hello: >-
+        'Hello ' + resolve('formal-name') + '!'
+    formal-name: >-
+        'Ms. ' + full_name
+```
+
+Note it's also _possible_ to call other expressions by name as functions, if they only return hard-coded values (i.e. constants).
+
+```yaml
+dyngle:
+  expressions:
+    author-name: Francis Potter
+    author-hello: >-
+        'Hello ' + author_name()
+``` 
+
+Here are some slightly more sophisticated exercises using Expression reference syntax:
+
+```yaml
+dyngle:
+  operations:
+    reference-hyphenated-data-key:
+      expressions:
+        spaced-name: "' '.join([x for x in first_name])"
+        count-name: len(resolve('first-name'))
+        x-name: "'X' * int(resolve('count-name'))"
+      steps:
+        - echo "Your name is {{first-name}} with {{count-name}} characters, but I will call you '{{spaced-name}}' or maybe '{{x-name}}'"
+    reference-expression-using-function-syntax:
+      expressions:
+        name: "'George'"
+        works: "name()"
+        double: "name * 2"
+        fails: double()
+      steps:
+        - echo "It works to call you {{works}}"
+        # - echo "I have trouble calling you {{fails}}"
+```
+
+Finally, here's an example using args:
+
+```yaml
+dyngle:
+  operations:
+    name-from-arg:
+      expressions:
+        name: "args[0]"
+      steps:
+        - echo "Hello {{name}}"
+```
+
+## Data assignment operator
+
+The Steps parser supports one special operator which assigns the output (stdout) of its command to a Data field for use in subsequent steps.
+
+The operator is `=>` and must go after the command and its arguments. Follow the operator with the name of the Data key to assign.
+
+Example:
+
+```yaml
+dyngle:
+  operations:
+    today:
+      expressions:
+        just-the-date: resolve('full-date')[0:10]
+      steps:
+        - date => full-date
+        - echo "Today is {{just-the-date}}"
+```
+
+(Note Dyngle does provide Python date operations in the Expression namespace, which might provide a better way to perform the same operation as this example, but it suffices to demonstrate assignment)
+
+## Lifecycle
+
+The lifecycle of an operation is:
+
+1. Load Data if it exists from YAML on stdin (if no tty)
+2. Find the named Operation in the configuration
+2. Perform template rendering on the first Step, using Data and Expressions
+3. Execute the Step in a subprocess
+4. Continue with the next Step
+
+Note that operations in the config are _not_ full shell lines. They are passed directly to the system.
+
+## Imports
+
+Configuration files can import other configuration files, by providing an entry `imports:` with an array of filepaths. The most obvious example is a Dyngle config in a local directory which imports the user-level configuration.
+
+```yaml
+dyngle:
+  imports:
+    - ~/.dyngle.yml
+  expressions:
+  operations:
+```
+
+In the event of item name conflicts, expressions and operations are loaded from imports in the order specified, so imports lower in the array will override those higher up. The expressions and operations defined in the main file override the imports. Imports are not recursive.
+
+## Security
+
+Commands are executed using Python's `subprocess.run()` with arguments split in a shell-like fashion. The shell is not used, which reduces the likelihood of shell injection attacks. However, note that Dyngle is not robust to malicious configuration. Use with caution.
+
+

{dyngle-0.6.0.dist-info → dyngle-0.7.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 dyngle/__init__.py,sha256=kCFjx6sIuf0hiuH2C4QyHIaC9DY6YKudvkiiziFybgk,2716
 dyngle/__main__.py,sha256=pYRIwzix_AL8CdJaDDis_8yMBBWO2N72NNwkroo1dQo,95
 dyngle/command/__init__.py,sha256=1S86gbef8MYvG-TWD5JRIWzFg7qV5xKhp9QXx9zEx5c,94
-dyngle/command/run_command.py,sha256=s6hR4PHBXbZ9pfQ-thVmpcvSW5DP5rwSpz3aginaPJ8,1914
+dyngle/command/run_command.py,sha256=0cRjCQIfbO0KZupp4f1h_TSZh7A0o_7slG-BXEJqjZs,1513
 dyngle/error.py,sha256=CGcTa8L4O1qsHEYnzp_JBbkvntJTv2Qz46wj_TI8NLk,39
 dyngle/expression.py,sha256=-uLVbrO8ovNZGGLNqMZWIy_StCK-0laZqcQ1gOPhU6w,3476
-dyngle/operation.py,sha256=QCAUH2YAYq9dtCO49F_irWgIWxBDaxyDv4X7Xen04Jo,110
+dyngle/operation.py,sha256=SCiaFNAxigXep2W95mXaT94-n1EZgQeouAHM0_Wh6GY,1631
 dyngle/safe_path.py,sha256=Hk2AhP6e3yKGh3kKrLLwhvAlMNx-j2jObBYJL-_doAU,3339
-dyngle/template.py,sha256=AbgGCVi9B0K_SY_0BO0Q8vbFNvFM0aYAVBJoOE4N_pE,1612
-dyngle-0.6.0.dist-info/METADATA,sha256=r511S0EBtCDb5SdfP6V8kuhcGYaAF1sxKfRSewmyt14,5524
-dyngle-0.6.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-dyngle-0.6.0.dist-info/entry_points.txt,sha256=rekiGhtweiHKm9g1jdGb3FhzqDrk1kigJDeSNollZSA,48
-dyngle-0.6.0.dist-info/RECORD,,
+dyngle/template.py,sha256=IJtHu1R3hfniSeCnI4PhmJ_8ojh2YBhH9l-7_SqsfoE,1613
+dyngle-0.7.0.dist-info/METADATA,sha256=8leyci3OsUBplVMVH4mQmZ7nkhzj4Lk6vYNX-MP9wwk,9072
+dyngle-0.7.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+dyngle-0.7.0.dist-info/entry_points.txt,sha256=rekiGhtweiHKm9g1jdGb3FhzqDrk1kigJDeSNollZSA,48
+dyngle-0.7.0.dist-info/RECORD,,

dyngle-0.6.0.dist-info/METADATA

@@ -1,185 +0,0 @@
-Metadata-Version: 2.3
-Name: dyngle
-Version: 0.6.0
-Summary: Run lightweight local workflows
-License: MIT
-Author: Steampunk Wizard
-Author-email: dyngle@steamwiz.io
-Requires-Python: >=3.11,<3.12
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: requests (>=2.32.3,<3.0.0)
-Requires-Dist: wizlib (>=3.1.4,<4.0.0)
-Description-Content-Type: text/markdown
-
-# Dyngle
-
-## Run lightweight local workflows
-
-Dyngle is a simple workflow runner that executes sequences of commands defined in configuration files. It's like a lightweight combination of Make and a task runner, designed for automating common development and operational tasks.
-
-## Basic usage
-
-Create a configuration file (e.g., `.dyngle.yml`) with your workflows:
-
-```yaml
-dyngle:
-  operations:
-    build:
-      - python -m pip install -e .
-      - python -m pytest
-    deploy:
-      - docker build -t myapp .
-      - docker push myapp
-    clean:
-      - rm -rf __pycache__
-      - rm -rf .pytest_cache
-```
-
-Run an operation:
-
-```bash
-dyngle run build
-```
-
-## Configuration
-
-Dyngle reads configuration from YAML files. You can specify the config file location using:
-
-- `--config` command line option
-- `DYNGLE_CONFIG` environment variable  
-- `.dyngle.yml` in current directory
-- `~/.dyngle.yml` in home directory
-
-The configuration has 2 parts: `operations:` and `expressions`.
-
-Configuration files can import other configuration files, by providing an entry `imports:` with an array of filepaths. The most obvious example is a Dyngle config in a local directory which imports the user-level configuration.
-
-```yaml
-dyngle:
-  imports:
-    - ~/.dyngle.yml
-  expressions:
-  operations:
-```
-
-In the event of item name conflicts, expressions and operations are loaded from imports in the order specified, so imports lower in the array will override those higher up. The expressions and operations defined in the main file override the imports. Imports are not recursive.
-
-## Data
-
-Dyngle maintains a block of data throughout operations, which is parsed from YAML in stdin.
-
-## Operations
-
-Operations contain steps as a YAML array. The lifecycle of an operation is:
-
-1. Load input data if it exists from YAML on stdin (if no tty)
-2. Perform template rendering on a step, using data and expressions (see below)
-3. Execute the step in a subprocess
-4. Continue with the next step
-
-Note that operations in the config are _not_ full shell lines. They are passed directly to the system.
-
-## Templates
-
-Prior to running commands, the line containing that command is processed as a template. Entries from the data set can be substituted into the command line using Jinja-like expressions in double-curly brackets (`{{` and `}}`).
-
-For example, if stdin contains the following data:
-
-```yaml
-name: Francis
-```
-
-And the command looks like:
-
-``` yaml
-- echo "Hello {{name}}!"
-```
-
-Then the command will output "Hello Francis!".
-
-
-## Expressions
-
-Configs can also contain expressions, written in Python, that can also be referenced in operation steps.
-
-```yaml
-dyngle:
-  expressions:
-    say-hello: >-
-        'Hello ' + name + '!'
-  operations:
-    say-hello: echo {{say-hello}}
-```
-
-Expressions can use a controlled subset of the Python standard library, including:
-
-- Built-in data types such as `str()`
-- Essential built-in functions such as `len()`
-- The core modules from the `datetime` package (but some methods such as `strftime()` will fail)
-- A specialized function called `formatted()` to perform string formatting operations on a `datetime` object
-- A restricted version of `Path()` that only operates within the current working directory
-- Various other useful utilities, mostly read-only, such as the `math` module
-- A special function called `resolve` which resolves data expressions using the same logic as in templates
-
-Data keys containing hyphens are converted to valid Python names by replacing hyphens with underscores.
-
-Expressions can reference data directly as local names in Python (using the underscore replacements)...
-
-```yaml
-dyngle:
-  expressions:
-    say-hello: >-
-        'Hello ' + full_name + '!'
-```
-
-... or using the `resolve()` function, which also allows expressions to essentially call other expressions, using the same underlying data set.
-
-```yaml
-dyngle:
-  expressions:
-    hello: >-
-        'Hello ' + resolve('formal-name') + '!'
-    formal-name: >-
-        'Ms. ' + full_name
-```
-
-Note it's also _possible_ to call other expressions by name as functions, if they only return hard-coded values (i.e. constants).
-
-```yaml
-dyngle:
-  expressions:
-    author-name: Francis Potter
-    author-hello: >-
-        'Hello ' + author_name()
-``` 
-
-## Local Expressions
-
-Expressions can also be defined in a way that applies only to one operation - especially useful for command-line arguments.
-
-In this case, the operation definition has a different structure. See the example below.
-
-```yaml
-dyngle:
-  operations:
-    name_from_arg:
-      expressions:
-        local_name: "args[0]"
-      steps:
-        - echo "Hello {{local_name}}"
-```
-
-## Security
-
-Commands are executed using Python's `subprocess.run()` with arguments split in a shell-like fashion. The shell is not used, which reduces the likelihood of shell injection attacks. However, note that Dyngle is not robust to malicious configuration. Use with caution.
-
-## Quick installation (MacOS)
-
-```bash
-brew install python@3.11
-python3.11 -m pip install pipx
-pipx install dyngle
-```
-