sane-workflows 1.0.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sane_workflows-1.0.0rc1/LICENSE +21 -0
- sane_workflows-1.0.0rc1/PKG-INFO +256 -0
- sane_workflows-1.0.0rc1/README.md +231 -0
- sane_workflows-1.0.0rc1/pyproject.toml +42 -0
- sane_workflows-1.0.0rc1/sane/__init__.py +30 -0
- sane_workflows-1.0.0rc1/sane/action.py +600 -0
- sane_workflows-1.0.0rc1/sane/action_launcher.py +54 -0
- sane_workflows-1.0.0rc1/sane/config.py +32 -0
- sane_workflows-1.0.0rc1/sane/dag.py +103 -0
- sane_workflows-1.0.0rc1/sane/dagvis.py +117 -0
- sane_workflows-1.0.0rc1/sane/env_from_script.sh +72 -0
- sane_workflows-1.0.0rc1/sane/environment.py +207 -0
- sane_workflows-1.0.0rc1/sane/host.py +133 -0
- sane_workflows-1.0.0rc1/sane/hpc_host.py +593 -0
- sane_workflows-1.0.0rc1/sane/json_config.py +78 -0
- sane_workflows-1.0.0rc1/sane/logger.py +75 -0
- sane_workflows-1.0.0rc1/sane/orchestrator.py +721 -0
- sane_workflows-1.0.0rc1/sane/resources.py +558 -0
- sane_workflows-1.0.0rc1/sane/sane_runner.py +319 -0
- sane_workflows-1.0.0rc1/sane/save_state.py +58 -0
- sane_workflows-1.0.0rc1/sane/user_space.py +3 -0
- sane_workflows-1.0.0rc1/sane/utdict.py +27 -0
- sane_workflows-1.0.0rc1/sane_workflows.egg-info/PKG-INFO +256 -0
- sane_workflows-1.0.0rc1/sane_workflows.egg-info/SOURCES.txt +35 -0
- sane_workflows-1.0.0rc1/sane_workflows.egg-info/dependency_links.txt +1 -0
- sane_workflows-1.0.0rc1/sane_workflows.egg-info/entry_points.txt +2 -0
- sane_workflows-1.0.0rc1/sane_workflows.egg-info/top_level.txt +1 -0
- sane_workflows-1.0.0rc1/setup.cfg +8 -0
- sane_workflows-1.0.0rc1/tests/test_action.py +205 -0
- sane_workflows-1.0.0rc1/tests/test_dag.py +255 -0
- sane_workflows-1.0.0rc1/tests/test_environment.py +74 -0
- sane_workflows-1.0.0rc1/tests/test_host.py +12 -0
- sane_workflows-1.0.0rc1/tests/test_hpc_host.py +102 -0
- sane_workflows-1.0.0rc1/tests/test_orchestrator.py +79 -0
- sane_workflows-1.0.0rc1/tests/test_resources.py +213 -0
- sane_workflows-1.0.0rc1/tests/test_sane_runner.py +98 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Anthony Islas
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sane-workflows
|
|
3
|
+
Version: 1.0.0rc1
|
|
4
|
+
Summary: Simple Action 'n Environment Workflows
|
|
5
|
+
Author-email: Anthony Islas <aislas@ucar.edu>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/islas/sane_workflows.git
|
|
8
|
+
Project-URL: Issues, https://github.com/islas/sane_workflows/issues
|
|
9
|
+
Keywords: sane,workflow,dag,orchestration
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Software Development :: Testing
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Requires-Python: >=3.6
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# SANE Workflows
|
|
27
|
+
[](https://www.python.org/downloads/) [](https://github.com/islas/sane_workflows/actions/workflows/unittest.yml)
|
|
28
|
+
|
|
29
|
+
Simple Action 'n Environment Workflow
|
|
30
|
+
|
|
31
|
+
SANE is a DAG-based action workflow runner augmented with environments provided
|
|
32
|
+
by hosts.
|
|
33
|
+
|
|
34
|
+
It provides:
|
|
35
|
+
* DAG-based action mapping, with statefulness to preserve runs
|
|
36
|
+
* both python and JSON config based workflow management
|
|
37
|
+
* sourcing workflows from multiple directories
|
|
38
|
+
* resource management between host and actions
|
|
39
|
+
* HPC-resource enabled hosts (PBS base fully implemented)
|
|
40
|
+
* environment variable manipulation via shell scripts, lmod, and explicit setting
|
|
41
|
+
* extensible `Host`, `Environment`, and `Action` classes that can be derived from
|
|
42
|
+
* derived classes allowed within user workflow directories
|
|
43
|
+
* derived classes accessible within user JSON configs
|
|
44
|
+
* a priority-based python registration decorator
|
|
45
|
+
* a priority-based JSON patching feature
|
|
46
|
+
|
|
47
|
+
## Overview
|
|
48
|
+
|
|
49
|
+
Below is a high level overview of how running a workflow works. Many of the complex
|
|
50
|
+
nuances, such as type finding, HPC submission, resource management, etc., are left out.
|
|
51
|
+
|
|
52
|
+
The focus should instead be:
|
|
53
|
+
* a single _orchestrator_ manages the workflow
|
|
54
|
+
* a node in the DAG-workflow is an _action_
|
|
55
|
+
* a _host_ provides _environments_ for an _action_
|
|
56
|
+
* _actions_ are run independently in an order informed by DAG dependencies
|
|
57
|
+
* an _action_ itself is `run()` in a totally separate subprocess (not python subprocess!)
|
|
58
|
+
* instance information is transferred via python pickling
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
## Install
|
|
63
|
+
|
|
64
|
+
This package is designed to work both as an installed python package or from source
|
|
65
|
+
code with no modifications necessary. To install the package use:
|
|
66
|
+
```
|
|
67
|
+
python3 -m pip install sane-workflows
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
To utilize from source, clone this repository. You may add the path to your `PYTHONPATH`
|
|
71
|
+
if you want to use it outside of the provided runner script, but this is not necessary.
|
|
72
|
+
|
|
73
|
+
Usage when installed:
|
|
74
|
+
```
|
|
75
|
+
sane_runner -h
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Usage when from source:
|
|
79
|
+
```
|
|
80
|
+
<path to source>/bin/sane_runner.py -h
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Python Usage
|
|
84
|
+
To utilize `sane` in a python setting, create a python file (module) and import the
|
|
85
|
+
`sane` package. Assuming you are running via the provided entry point `sane_runner[.py]`,
|
|
86
|
+
you do not need to ensure `sane` is within your `PYTHONPATH`. Afterwards, to add,
|
|
87
|
+
remove, or modify the _orchestrator_ use the `@sane.register(priority=0)` decorator.
|
|
88
|
+
Providing a priority is optional, and if no priority is given, no `()` call is necessary,
|
|
89
|
+
as seen below. The _orchestrator_ is provided as the single argument to the decorated
|
|
90
|
+
function.
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
import sane
|
|
94
|
+
|
|
95
|
+
@sane.register
|
|
96
|
+
def my_workflow( orch ):
|
|
97
|
+
my_action = sane.Action( "id" )
|
|
98
|
+
orch.add_action( my_action )
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
If a priority is given, functions will be evaluated in descending order (highest
|
|
102
|
+
priority first)
|
|
103
|
+
```python
|
|
104
|
+
import sane
|
|
105
|
+
|
|
106
|
+
@sane.register
|
|
107
|
+
def last( orch ):
|
|
108
|
+
# defaul priority is 0
|
|
109
|
+
pass
|
|
110
|
+
|
|
111
|
+
@sane.register( priority=5 )
|
|
112
|
+
def second( orch ):
|
|
113
|
+
pass
|
|
114
|
+
|
|
115
|
+
@sane.register( 99 )
|
|
116
|
+
def first( orch ):
|
|
117
|
+
pass
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## JSON Usage
|
|
121
|
+
To utilize `sane` in a JSON config file setting, create a JSON file (config) that
|
|
122
|
+
contains at least one of the keys : `"hosts"`, `"actions"`, or `"patches"`. Refer
|
|
123
|
+
to the [`docs/template.jsonc`](docs/template.jsonc) on what default fields are appropriate.
|
|
124
|
+
Note that if you define your own type (and thus add your own `load_extra_config()`),
|
|
125
|
+
additional fields may be provided in the config.
|
|
126
|
+
```jsonc
|
|
127
|
+
{
|
|
128
|
+
"hosts" :
|
|
129
|
+
{
|
|
130
|
+
"dummy" : { "environment" : "generic" }
|
|
131
|
+
},
|
|
132
|
+
"actions" :
|
|
133
|
+
{
|
|
134
|
+
"my_action" :
|
|
135
|
+
{
|
|
136
|
+
"config" : { "command" : "echo", "arguments" : [ 1 ] },
|
|
137
|
+
"environment" : "generic"
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
By default, you may utilize _action_ attributes inside of the generic `"config" : {}`
|
|
144
|
+
dictionary field. The attributes are automatically scoped to the current _action_
|
|
145
|
+
and are accessed via YAML-like dereferencing (`${{}}`):
|
|
146
|
+
```jsonc
|
|
147
|
+
// ... previous config
|
|
148
|
+
"actions" :
|
|
149
|
+
{
|
|
150
|
+
"my_action" :
|
|
151
|
+
{
|
|
152
|
+
"config" : { "command" : "echo", "arguments" : [ "${{ id }}" ] }
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// ... rest of config
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## Running a workflow
|
|
159
|
+
To run a workflow, place all your `.py` and `.json[c]` files into any directory
|
|
160
|
+
layout you want, but try to isolate your workflow files from other non-workflow
|
|
161
|
+
`.py` and `.json[c]` files as all matching files under listed directories are
|
|
162
|
+
loaded. Supplementary files, like shell scripts (`.sh`), will not be loaded.
|
|
163
|
+
|
|
164
|
+
Provide the paths of your workflow with `-p`/`--path`, then list or filter for
|
|
165
|
+
whichever _actions_ you want to operate with, along with the `-r` flag to run these
|
|
166
|
+
_actions_:
|
|
167
|
+
```bash
|
|
168
|
+
<path to sane_workflows>/bin/sane_runner.py -p <workflow path> [-p <other path>] -a my_action -r
|
|
169
|
+
```
|
|
170
|
+
> [!NOTE]
|
|
171
|
+
> All paths provided are added to `sys.path` for importing of modules. Thus, when
|
|
172
|
+
> using custom classes within your workflow, `import` their module as if from the
|
|
173
|
+
> workflow path, e.g. `-p .workflow` for `.workflow/custom_actions/my_action_def.py`
|
|
174
|
+
> as `import custom_actions.my_action_def`
|
|
175
|
+
|
|
176
|
+
You will get output that looks like so:
|
|
177
|
+
```
|
|
178
|
+
./bin/sane_runner.py -p demo/ -a action_000 -r
|
|
179
|
+
2025-10-08 18:22:56 INFO [sane_runner] Logging output to /home/aislas/sane_workflows/log/runner.log
|
|
180
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching for workflow files...
|
|
181
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching demo/ for *.json
|
|
182
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/custom_def_usage.json
|
|
183
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/simple_action.json
|
|
184
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/resource_action.json
|
|
185
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/hpc_host.json
|
|
186
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/patches.json
|
|
187
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching demo/ for *.jsonc
|
|
188
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/simple_host.jsonc
|
|
189
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching demo/ for *.py
|
|
190
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/my_workflow.py
|
|
191
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/simple_host.py
|
|
192
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/actual_workflow.py
|
|
193
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/custom_defs.py
|
|
194
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/my_workflow.py as 'my_workflow'
|
|
195
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/simple_host.py as 'simple_host'
|
|
196
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/actual_workflow.py as 'actual_workflow'
|
|
197
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/custom_defs.py as 'custom_defs'
|
|
198
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] Creation of universe
|
|
199
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] Creation of world
|
|
200
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] Hello world from my_workflow
|
|
201
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] <class 'custom_defs.MyAction'>
|
|
202
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/custom_def_usage.json
|
|
203
|
+
2025-10-08 18:22:56 WARNING [fib_seq_fixed] Unused keys in config : ['unused_action_param']
|
|
204
|
+
2025-10-08 18:22:56 WARNING [orchestrator] Unused keys in config : ['unused_orch_param']
|
|
205
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/simple_action.json
|
|
206
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/resource_action.json
|
|
207
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/hpc_host.json
|
|
208
|
+
2025-10-08 18:22:56 INFO [example_pbs] Adding homogeneous node resources for 'cpu'
|
|
209
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/patches.json
|
|
210
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/simple_host.jsonc
|
|
211
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch to Host 'unique_host_config'
|
|
212
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch to Action 'fib_seq_fixed'
|
|
213
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch to Action 'fib_seq_calc_mult'
|
|
214
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_090'
|
|
215
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_091'
|
|
216
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_092'
|
|
217
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_093'
|
|
218
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_094'
|
|
219
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_095'
|
|
220
|
+
2025-10-08 18:22:56 WARNING [orchestrator::patch] Unused keys in patch : ['unused_patch_param']
|
|
221
|
+
2025-10-08 18:22:56 INFO [orchestrator] No previous save file to load
|
|
222
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running actions:
|
|
223
|
+
2025-10-08 18:22:56 INFO [orchestrator] action_000
|
|
224
|
+
2025-10-08 18:22:56 INFO [orchestrator] and any necessary dependencies
|
|
225
|
+
2025-10-08 18:22:56 INFO [orchestrator] Full action set:
|
|
226
|
+
2025-10-08 18:22:56 INFO [orchestrator] action_000
|
|
227
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking host "generic"
|
|
228
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running as 'generic'
|
|
229
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking ability to run all actions on 'generic'...
|
|
230
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking environments...
|
|
231
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking resource availability...
|
|
232
|
+
2025-10-08 18:22:56 INFO [orchestrator] * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
|
233
|
+
2025-10-08 18:22:56 INFO [orchestrator] * * * * * * * * * * All prerun checks for 'generic' passed * * * * * * * * * *
|
|
234
|
+
2025-10-08 18:22:56 INFO [orchestrator] * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
|
235
|
+
2025-10-08 18:22:56 INFO [orchestrator] Saving host information...
|
|
236
|
+
2025-10-08 18:22:56 INFO [orchestrator] Setting state of all inactive actions to pending
|
|
237
|
+
2025-10-08 18:22:56 INFO [orchestrator] No previous save file to load
|
|
238
|
+
2025-10-08 18:22:56 INFO [orchestrator] Using working directory : '/home/aislas/sane_workflows'
|
|
239
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running actions...
|
|
240
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running 'action_000' on 'generic'
|
|
241
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Saving action information for launch...
|
|
242
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Using working directory : '/home/aislas/sane_workflows'
|
|
243
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Running command:
|
|
244
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] /home/aislas/sane_workflows/sane/action_launcher.py /home/aislas/sane_workflows /home/aislas/sane_workflows/tmp/action_action_000.json
|
|
245
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Command output will be captured to logfile /home/aislas/sane_workflows/log/action_000.log
|
|
246
|
+
2025-10-08 18:22:56 INFO [orchestrator] FINISHED Action 'action_000' completed with 'success'
|
|
247
|
+
2025-10-08 18:22:56 INFO [orchestrator] Finished running queued actions
|
|
248
|
+
2025-10-08 18:22:56 INFO [orchestrator] action_000: success
|
|
249
|
+
2025-10-08 18:22:56 INFO [orchestrator] All actions finished with success
|
|
250
|
+
2025-10-08 18:22:56 INFO [orchestrator] Save file at /home/aislas/sane_workflows/tmp/orchestrator.json
|
|
251
|
+
2025-10-08 18:22:56 INFO [orchestrator] JUnit file at /home/aislas/sane_workflows/log/results.xml
|
|
252
|
+
2025-10-08 18:22:56 INFO [sane_runner] Finished
|
|
253
|
+
```
|
|
254
|
+
> [!TIP]
|
|
255
|
+
> The above is generated from the `demo/` folder in the repository. It should
|
|
256
|
+
> provide a decent starting example of what a workflow may look like.
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
# SANE Workflows
|
|
2
|
+
[](https://www.python.org/downloads/) [](https://github.com/islas/sane_workflows/actions/workflows/unittest.yml)
|
|
3
|
+
|
|
4
|
+
Simple Action 'n Environment Workflow
|
|
5
|
+
|
|
6
|
+
SANE is a DAG-based action workflow runner augmented with environments provided
|
|
7
|
+
by hosts.
|
|
8
|
+
|
|
9
|
+
It provides:
|
|
10
|
+
* DAG-based action mapping, with statefulness to preserve runs
|
|
11
|
+
* both python and JSON config based workflow management
|
|
12
|
+
* sourcing workflows from multiple directories
|
|
13
|
+
* resource management between host and actions
|
|
14
|
+
* HPC-resource enabled hosts (PBS base fully implemented)
|
|
15
|
+
* environment variable manipulation via shell scripts, lmod, and explicit setting
|
|
16
|
+
* extensible `Host`, `Environment`, and `Action` classes that can be derived from
|
|
17
|
+
* derived classes allowed within user workflow directories
|
|
18
|
+
* derived classes accessible within user JSON configs
|
|
19
|
+
* a priority-based python registration decorator
|
|
20
|
+
* a priority-based JSON patching feature
|
|
21
|
+
|
|
22
|
+
## Overview
|
|
23
|
+
|
|
24
|
+
Below is a high level overview of how running a workflow works. Many of the complex
|
|
25
|
+
nuances, such as type finding, HPC submission, resource management, etc., are left out.
|
|
26
|
+
|
|
27
|
+
The focus should instead be:
|
|
28
|
+
* a single _orchestrator_ manages the workflow
|
|
29
|
+
* a node in the DAG-workflow is an _action_
|
|
30
|
+
* a _host_ provides _environments_ for an _action_
|
|
31
|
+
* _actions_ are run independently in an order informed by DAG dependencies
|
|
32
|
+
* an _action_ itself is `run()` in a totally separate subprocess (not python subprocess!)
|
|
33
|
+
* instance information is transferred via python pickling
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
This package is designed to work both as an installed python package or from source
|
|
40
|
+
code with no modifications necessary. To install the package use:
|
|
41
|
+
```
|
|
42
|
+
python3 -m pip install sane-workflows
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
To utilize from source, clone this repository. You may add the path to your `PYTHONPATH`
|
|
46
|
+
if you want to use it outside of the provided runner script, but this is not necessary.
|
|
47
|
+
|
|
48
|
+
Usage when installed:
|
|
49
|
+
```
|
|
50
|
+
sane_runner -h
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Usage when from source:
|
|
54
|
+
```
|
|
55
|
+
<path to source>/bin/sane_runner.py -h
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Python Usage
|
|
59
|
+
To utilize `sane` in a python setting, create a python file (module) and import the
|
|
60
|
+
`sane` package. Assuming you are running via the provided entry point `sane_runner[.py]`,
|
|
61
|
+
you do not need to ensure `sane` is within your `PYTHONPATH`. Afterwards, to add,
|
|
62
|
+
remove, or modify the _orchestrator_ use the `@sane.register(priority=0)` decorator.
|
|
63
|
+
Providing a priority is optional, and if no priority is given, no `()` call is necessary,
|
|
64
|
+
as seen below. The _orchestrator_ is provided as the single argument to the decorated
|
|
65
|
+
function.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import sane
|
|
69
|
+
|
|
70
|
+
@sane.register
|
|
71
|
+
def my_workflow( orch ):
|
|
72
|
+
my_action = sane.Action( "id" )
|
|
73
|
+
orch.add_action( my_action )
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
If a priority is given, functions will be evaluated in descending order (highest
|
|
77
|
+
priority first)
|
|
78
|
+
```python
|
|
79
|
+
import sane
|
|
80
|
+
|
|
81
|
+
@sane.register
|
|
82
|
+
def last( orch ):
|
|
83
|
+
# defaul priority is 0
|
|
84
|
+
pass
|
|
85
|
+
|
|
86
|
+
@sane.register( priority=5 )
|
|
87
|
+
def second( orch ):
|
|
88
|
+
pass
|
|
89
|
+
|
|
90
|
+
@sane.register( 99 )
|
|
91
|
+
def first( orch ):
|
|
92
|
+
pass
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## JSON Usage
|
|
96
|
+
To utilize `sane` in a JSON config file setting, create a JSON file (config) that
|
|
97
|
+
contains at least one of the keys : `"hosts"`, `"actions"`, or `"patches"`. Refer
|
|
98
|
+
to the [`docs/template.jsonc`](docs/template.jsonc) on what default fields are appropriate.
|
|
99
|
+
Note that if you define your own type (and thus add your own `load_extra_config()`),
|
|
100
|
+
additional fields may be provided in the config.
|
|
101
|
+
```jsonc
|
|
102
|
+
{
|
|
103
|
+
"hosts" :
|
|
104
|
+
{
|
|
105
|
+
"dummy" : { "environment" : "generic" }
|
|
106
|
+
},
|
|
107
|
+
"actions" :
|
|
108
|
+
{
|
|
109
|
+
"my_action" :
|
|
110
|
+
{
|
|
111
|
+
"config" : { "command" : "echo", "arguments" : [ 1 ] },
|
|
112
|
+
"environment" : "generic"
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
By default, you may utilize _action_ attributes inside of the generic `"config" : {}`
|
|
119
|
+
dictionary field. The attributes are automatically scoped to the current _action_
|
|
120
|
+
and are accessed via YAML-like dereferencing (`${{}}`):
|
|
121
|
+
```jsonc
|
|
122
|
+
// ... previous config
|
|
123
|
+
"actions" :
|
|
124
|
+
{
|
|
125
|
+
"my_action" :
|
|
126
|
+
{
|
|
127
|
+
"config" : { "command" : "echo", "arguments" : [ "${{ id }}" ] }
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
// ... rest of config
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Running a workflow
|
|
134
|
+
To run a workflow, place all your `.py` and `.json[c]` files into any directory
|
|
135
|
+
layout you want, but try to isolate your workflow files from other non-workflow
|
|
136
|
+
`.py` and `.json[c]` files as all matching files under listed directories are
|
|
137
|
+
loaded. Supplementary files, like shell scripts (`.sh`), will not be loaded.
|
|
138
|
+
|
|
139
|
+
Provide the paths of your workflow with `-p`/`--path`, then list or filter for
|
|
140
|
+
whichever _actions_ you want to operate with, along with the `-r` flag to run these
|
|
141
|
+
_actions_:
|
|
142
|
+
```bash
|
|
143
|
+
<path to sane_workflows>/bin/sane_runner.py -p <workflow path> [-p <other path>] -a my_action -r
|
|
144
|
+
```
|
|
145
|
+
> [!NOTE]
|
|
146
|
+
> All paths provided are added to `sys.path` for importing of modules. Thus, when
|
|
147
|
+
> using custom classes within your workflow, `import` their module as if from the
|
|
148
|
+
> workflow path, e.g. `-p .workflow` for `.workflow/custom_actions/my_action_def.py`
|
|
149
|
+
> as `import custom_actions.my_action_def`
|
|
150
|
+
|
|
151
|
+
You will get output that looks like so:
|
|
152
|
+
```
|
|
153
|
+
./bin/sane_runner.py -p demo/ -a action_000 -r
|
|
154
|
+
2025-10-08 18:22:56 INFO [sane_runner] Logging output to /home/aislas/sane_workflows/log/runner.log
|
|
155
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching for workflow files...
|
|
156
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching demo/ for *.json
|
|
157
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/custom_def_usage.json
|
|
158
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/simple_action.json
|
|
159
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/resource_action.json
|
|
160
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/hpc_host.json
|
|
161
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/patches.json
|
|
162
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching demo/ for *.jsonc
|
|
163
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/simple_host.jsonc
|
|
164
|
+
2025-10-08 18:22:56 INFO [orchestrator] Searching demo/ for *.py
|
|
165
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/my_workflow.py
|
|
166
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/simple_host.py
|
|
167
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/actual_workflow.py
|
|
168
|
+
2025-10-08 18:22:56 INFO [orchestrator] Found demo/custom_defs.py
|
|
169
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/my_workflow.py as 'my_workflow'
|
|
170
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/simple_host.py as 'simple_host'
|
|
171
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/actual_workflow.py as 'actual_workflow'
|
|
172
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading python file demo/custom_defs.py as 'custom_defs'
|
|
173
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] Creation of universe
|
|
174
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] Creation of world
|
|
175
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] Hello world from my_workflow
|
|
176
|
+
2025-10-08 18:22:56 INFO [orchestrator::register] <class 'custom_defs.MyAction'>
|
|
177
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/custom_def_usage.json
|
|
178
|
+
2025-10-08 18:22:56 WARNING [fib_seq_fixed] Unused keys in config : ['unused_action_param']
|
|
179
|
+
2025-10-08 18:22:56 WARNING [orchestrator] Unused keys in config : ['unused_orch_param']
|
|
180
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/simple_action.json
|
|
181
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/resource_action.json
|
|
182
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/hpc_host.json
|
|
183
|
+
2025-10-08 18:22:56 INFO [example_pbs] Adding homogeneous node resources for 'cpu'
|
|
184
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/patches.json
|
|
185
|
+
2025-10-08 18:22:56 INFO [orchestrator] Loading config file demo/simple_host.jsonc
|
|
186
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch to Host 'unique_host_config'
|
|
187
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch to Action 'fib_seq_fixed'
|
|
188
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch to Action 'fib_seq_calc_mult'
|
|
189
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_090'
|
|
190
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_091'
|
|
191
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_092'
|
|
192
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_093'
|
|
193
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_094'
|
|
194
|
+
2025-10-08 18:22:56 INFO [orchestrator::patch] Applying patch filter to Action 'action_095'
|
|
195
|
+
2025-10-08 18:22:56 WARNING [orchestrator::patch] Unused keys in patch : ['unused_patch_param']
|
|
196
|
+
2025-10-08 18:22:56 INFO [orchestrator] No previous save file to load
|
|
197
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running actions:
|
|
198
|
+
2025-10-08 18:22:56 INFO [orchestrator] action_000
|
|
199
|
+
2025-10-08 18:22:56 INFO [orchestrator] and any necessary dependencies
|
|
200
|
+
2025-10-08 18:22:56 INFO [orchestrator] Full action set:
|
|
201
|
+
2025-10-08 18:22:56 INFO [orchestrator] action_000
|
|
202
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking host "generic"
|
|
203
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running as 'generic'
|
|
204
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking ability to run all actions on 'generic'...
|
|
205
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking environments...
|
|
206
|
+
2025-10-08 18:22:56 INFO [orchestrator] Checking resource availability...
|
|
207
|
+
2025-10-08 18:22:56 INFO [orchestrator] * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
|
208
|
+
2025-10-08 18:22:56 INFO [orchestrator] * * * * * * * * * * All prerun checks for 'generic' passed * * * * * * * * * *
|
|
209
|
+
2025-10-08 18:22:56 INFO [orchestrator] * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
|
210
|
+
2025-10-08 18:22:56 INFO [orchestrator] Saving host information...
|
|
211
|
+
2025-10-08 18:22:56 INFO [orchestrator] Setting state of all inactive actions to pending
|
|
212
|
+
2025-10-08 18:22:56 INFO [orchestrator] No previous save file to load
|
|
213
|
+
2025-10-08 18:22:56 INFO [orchestrator] Using working directory : '/home/aislas/sane_workflows'
|
|
214
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running actions...
|
|
215
|
+
2025-10-08 18:22:56 INFO [orchestrator] Running 'action_000' on 'generic'
|
|
216
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Saving action information for launch...
|
|
217
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Using working directory : '/home/aislas/sane_workflows'
|
|
218
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Running command:
|
|
219
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] /home/aislas/sane_workflows/sane/action_launcher.py /home/aislas/sane_workflows /home/aislas/sane_workflows/tmp/action_action_000.json
|
|
220
|
+
2025-10-08 18:22:56 INFO [action_000[thread_0]] Command output will be captured to logfile /home/aislas/sane_workflows/log/action_000.log
|
|
221
|
+
2025-10-08 18:22:56 INFO [orchestrator] FINISHED Action 'action_000' completed with 'success'
|
|
222
|
+
2025-10-08 18:22:56 INFO [orchestrator] Finished running queued actions
|
|
223
|
+
2025-10-08 18:22:56 INFO [orchestrator] action_000: success
|
|
224
|
+
2025-10-08 18:22:56 INFO [orchestrator] All actions finished with success
|
|
225
|
+
2025-10-08 18:22:56 INFO [orchestrator] Save file at /home/aislas/sane_workflows/tmp/orchestrator.json
|
|
226
|
+
2025-10-08 18:22:56 INFO [orchestrator] JUnit file at /home/aislas/sane_workflows/log/results.xml
|
|
227
|
+
2025-10-08 18:22:56 INFO [sane_runner] Finished
|
|
228
|
+
```
|
|
229
|
+
> [!TIP]
|
|
230
|
+
> The above is generated from the `demo/` folder in the repository. It should
|
|
231
|
+
> provide a decent starting example of what a workflow may look like.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sane-workflows"
|
|
7
|
+
version = "1.0.0-rc.1"
|
|
8
|
+
description = "Simple Action 'n Environment Workflows"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.6"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = [ "LICENSE*" ]
|
|
13
|
+
authors = [{ name = "Anthony Islas", email = "aislas@ucar.edu" }]
|
|
14
|
+
keywords = ["sane", "workflow", "dag", "orchestration"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Topic :: Software Development :: Testing",
|
|
20
|
+
|
|
21
|
+
# Specify the Python versions you support here.
|
|
22
|
+
"Programming Language :: Python :: 3.6",
|
|
23
|
+
"Programming Language :: Python :: 3.7",
|
|
24
|
+
"Programming Language :: Python :: 3.8",
|
|
25
|
+
"Programming Language :: Python :: 3.9",
|
|
26
|
+
"Programming Language :: Python :: 3.10",
|
|
27
|
+
"Programming Language :: Python :: 3.11",
|
|
28
|
+
"Programming Language :: Python :: 3.12",
|
|
29
|
+
"Programming Language :: Python :: 3.13",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Repository = "https://github.com/islas/sane_workflows.git"
|
|
34
|
+
Issues = "https://github.com/islas/sane_workflows/issues"
|
|
35
|
+
|
|
36
|
+
[project.scripts]
|
|
37
|
+
sane_runner = "sane:sane_runner.main"
|
|
38
|
+
|
|
39
|
+
[tool.setuptools]
|
|
40
|
+
include-package-data = true
|
|
41
|
+
package-data = { "sane" = [ "*.sh" ] }
|
|
42
|
+
packages = [ "sane" ]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from .action import Action, DependencyType, ActionState
|
|
5
|
+
from .environment import Environment
|
|
6
|
+
from .host import Host
|
|
7
|
+
from .hpc_host import HPCHost, PBSHost
|
|
8
|
+
from .orchestrator import Orchestrator, register
|
|
9
|
+
from .logger import DispatchingFormatter
|
|
10
|
+
from .user_space import user_modules
|
|
11
|
+
|
|
12
|
+
log_formatter = DispatchingFormatter(
|
|
13
|
+
{
|
|
14
|
+
f"{__name__}.logger" : logging.Formatter(
|
|
15
|
+
fmt="%(asctime)s %(levelname)-8s %(message)s",
|
|
16
|
+
datefmt="%Y-%m-%d %H:%M:%S"
|
|
17
|
+
),
|
|
18
|
+
f"{__name__}.raw" : logging.Formatter()
|
|
19
|
+
},
|
|
20
|
+
logging.Formatter( "%(message)s" )
|
|
21
|
+
)
|
|
22
|
+
console_handler = logging.StreamHandler( sys.stdout )
|
|
23
|
+
console_handler.setFormatter( log_formatter )
|
|
24
|
+
internal_logger = logging.getLogger( __name__ )
|
|
25
|
+
internal_logger.setLevel( logging.INFO )
|
|
26
|
+
internal_logger.addHandler( console_handler )
|
|
27
|
+
|
|
28
|
+
logging.addLevelName( 25, "STDOUT" )
|
|
29
|
+
for i in range( logging.DEBUG, logging.INFO ):
|
|
30
|
+
logging.addLevelName( i, f"DEBUG {i}" )
|