sirocco 0.0.0.dev0__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sirocco-0.0.0.dev0 → sirocco-0.1.0}/.gitignore +16 -1
- sirocco-0.1.0/LICENSE +21 -0
- sirocco-0.1.0/PKG-INFO +199 -0
- sirocco-0.1.0/README.md +140 -0
- sirocco-0.1.0/pyproject.toml +225 -0
- sirocco-0.1.0/src/sirocco/__init__.py +5 -0
- sirocco-0.1.0/src/sirocco/cli.py +680 -0
- sirocco-0.1.0/src/sirocco/core/__init__.py +31 -0
- sirocco-0.1.0/src/sirocco/core/graph_items.py +457 -0
- sirocco-0.1.0/src/sirocco/core/namelistfile.py +140 -0
- sirocco-0.1.0/src/sirocco/core/scheduler.py +207 -0
- sirocco-0.1.0/src/sirocco/core/tasks/__init__.py +5 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/__init__.py +4 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/icon_task.py +609 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/models.py +82 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/ports.py +336 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/runtime/santis/generate_hostfile.sh +64 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/runtime/santis/main.sh +51 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/runtime/santis/santis_environments.sh +108 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/runtime/santis/santis_hiopy_wrapper.sh +6 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/runtime/santis/santis_icon_wrapper.sh +42 -0
- sirocco-0.1.0/src/sirocco/core/tasks/icon_task/task_distribution.py +133 -0
- sirocco-0.1.0/src/sirocco/core/tasks/shell_task.py +90 -0
- sirocco-0.1.0/src/sirocco/core/tasks/sirocco_task.py +47 -0
- sirocco-0.1.0/src/sirocco/core/workflow.py +536 -0
- sirocco-0.1.0/src/sirocco/engines/__init__.py +0 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/__init__.py +28 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/adapter.py +338 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/builder.py +264 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/calcjob_builders.py +248 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/code_factory.py +236 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/dependency_resolvers.py +469 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/execute.py +105 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/models.py +228 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/monitoring.py +92 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/patches/__init__.py +7 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/patches/firecrest_symlink.py +131 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/patches/slurm_dependencies.py +268 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/patches/workgraph_window.py +307 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/spec_builders.py +446 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/spec_resolvers.py +473 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/task_pairs.py +200 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/topology.py +56 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/types.py +129 -0
- sirocco-0.1.0/src/sirocco/engines/aiida/utils.py +272 -0
- sirocco-0.1.0/src/sirocco/parsing/__init__.py +7 -0
- sirocco-0.1.0/src/sirocco/parsing/_utils.py +164 -0
- sirocco-0.1.0/src/sirocco/parsing/cycling.py +86 -0
- sirocco-0.1.0/src/sirocco/parsing/target_cycle.py +23 -0
- sirocco-0.1.0/src/sirocco/parsing/when.py +39 -0
- sirocco-0.1.0/src/sirocco/parsing/yaml_data_models.py +1182 -0
- sirocco-0.1.0/src/sirocco/pretty_print.py +191 -0
- sirocco-0.1.0/src/sirocco/py.typed +0 -0
- sirocco-0.1.0/src/sirocco/svg-interactive-script.js +236 -0
- sirocco-0.1.0/src/sirocco/svg-interactive-style.css +85 -0
- sirocco-0.1.0/src/sirocco/vizgraph.py +312 -0
- sirocco-0.0.0.dev0/PKG-INFO +0 -52
- sirocco-0.0.0.dev0/README.md +0 -30
- sirocco-0.0.0.dev0/pyproject.toml +0 -87
- sirocco-0.0.0.dev0/src/sirocco/__init__.py +0 -5
- sirocco-0.0.0.dev0/src/sirocco/core.py +0 -689
- sirocco-0.0.0.dev0/src/sirocco/parsing/__init__.py +0 -5
- sirocco-0.0.0.dev0/src/sirocco/parsing/_utils.py +0 -24
- sirocco-0.0.0.dev0/src/sirocco/parsing/_yaml_data_models.py +0 -361
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
# Project specific / generated files
|
|
2
|
+
tests/cases/*/data/*.new.txt
|
|
3
|
+
tests/cases/*/svg/*
|
|
4
|
+
!tests/cases/*/svg/.gitkeep
|
|
5
|
+
|
|
6
|
+
# Docs
|
|
7
|
+
docs/assets/.comments/*
|
|
8
|
+
|
|
1
9
|
# Byte-compiled / optimized / DLL files
|
|
2
10
|
__pycache__/
|
|
3
11
|
*.py[cod]
|
|
@@ -44,7 +52,7 @@ htmlcov/
|
|
|
44
52
|
.coverage.*
|
|
45
53
|
.cache
|
|
46
54
|
nosetests.xml
|
|
47
|
-
coverage
|
|
55
|
+
coverage.*
|
|
48
56
|
*.cover
|
|
49
57
|
*.py,cover
|
|
50
58
|
.hypothesis/
|
|
@@ -161,3 +169,10 @@ cython_debug/
|
|
|
161
169
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
162
170
|
#.idea/
|
|
163
171
|
|
|
172
|
+
html/
|
|
173
|
+
**/*.svg
|
|
174
|
+
!docs/assets/**/*.svg
|
|
175
|
+
!docs/overrides/**
|
|
176
|
+
**/*.resolved.yml
|
|
177
|
+
**/*.resolved.yaml
|
|
178
|
+
pyrightconfig.json
|
sirocco-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 ETH Zürich
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
sirocco-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sirocco
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A library for creating weather and climate workflows.
|
|
5
|
+
Project-URL: Repository, https://github.com/C2SM/Sirocco.git
|
|
6
|
+
Project-URL: Documentation, https://sirocco.ethz.ch
|
|
7
|
+
Project-URL: Changelog, https://github.com/C2SM/Sirocco/blob/main/CHANGELOG.md
|
|
8
|
+
Author-email: Matthieu Leclair <matthieu.leclair@c2sm.ethz.ch>, Julian Geiger <julian.geiger@psi.ch>, Alexander Goscinski <alexander.goscinski@psi.ch>
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2024 ETH Zürich
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: icon,aiida,aiida-workgraph,workflow
|
|
32
|
+
Classifier: Development Status :: 3 - Alpha
|
|
33
|
+
Classifier: Intended Audience :: Science/Research
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
36
|
+
Classifier: Programming Language :: Python
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
40
|
+
Classifier: Topic :: Scientific/Engineering
|
|
41
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
42
|
+
Requires-Python: >=3.12
|
|
43
|
+
Requires-Dist: coverage~=7.0
|
|
44
|
+
Requires-Dist: dotenv~=0.9
|
|
45
|
+
Requires-Dist: f90nml~=1.5
|
|
46
|
+
Requires-Dist: ipdb~=0.13
|
|
47
|
+
Requires-Dist: isodate~=0.7
|
|
48
|
+
Requires-Dist: isoduration~=20.11
|
|
49
|
+
Requires-Dist: jinja2~=3.0
|
|
50
|
+
Requires-Dist: lxml~=6.0
|
|
51
|
+
Requires-Dist: pydantic~=2.0
|
|
52
|
+
Requires-Dist: pygraphviz~=2.0
|
|
53
|
+
Requires-Dist: rich~=15.0.0
|
|
54
|
+
Requires-Dist: ruamel-yaml~=0.19
|
|
55
|
+
Requires-Dist: termcolor~=3.0
|
|
56
|
+
Requires-Dist: typer~=0.27.2
|
|
57
|
+
Requires-Dist: types-pyyaml~=6.0
|
|
58
|
+
Description-Content-Type: text/markdown
|
|
59
|
+
|
|
60
|
+
<p align="center">
|
|
61
|
+
<img alt="Sirocco logo" src="./docs/assets/Sirocco_Logo2026_BrunRVB.svg#gh-light-mode-only" width="500" />
|
|
62
|
+
<img alt="Sirocco logo" src="./docs/assets/Sirocco_Logo2026_BeigeRVB.svg#gh-dark-mode-only" width="500" />
|
|
63
|
+
</p>
|
|
64
|
+
|
|
65
|
+
<p align="center">
|
|
66
|
+
<a href="https://sirocco.ethz.ch"><b>Documentation</b></a> ·
|
|
67
|
+
<a href="https://sirocco.ethz.ch/usage/standalone/">Getting started</a> ·
|
|
68
|
+
<a href="./CHANGELOG.md">Changelog</a>
|
|
69
|
+
</p>
|
|
70
|
+
|
|
71
|
+
**Sirocco** is a Python workflow tool for climate and weather applications running on HPC systems. Workflows are described in a YAML dialect inspired by [Cylc](https://cylc.github.io/) and [AiiDA](https://www.aiida.net/), and unrolled into a graph of tasks and data.
|
|
72
|
+
|
|
73
|
+
- **Data as graph nodes.** Tasks explicitly declare the data they consume and produce, through named ports. No task needs to make assumptions on where others read or write, so workflows are truly composable.
|
|
74
|
+
- **Cycles.** Recurring sets of tasks with their own periodicity, so that describing climate and weather workflows feels natural.
|
|
75
|
+
- **Parameters.** Ensemble and sensitivity experiments through parameterized tasks and data.
|
|
76
|
+
- **Deep propagating task front.** Tasks are submitted ahead of time, chained by scheduler dependencies, in a window that moves through the graph as tasks complete. Tasks no longer wait for their parents to finish before entering the queue.
|
|
77
|
+
- **Plugins.** A general purpose `shell` plugin and a dedicated plugin for the [ICON](https://www.icon-model.org) model, covering complex setups such as coupled runs on hybrid CPU/GPU architectures.
|
|
78
|
+
- **Two engines.** A standalone engine running autonomously on the HPC system, with the scheduler as the sole daemon, and an [AiiDA](https://www.aiida.net/) based engine (under development) running on a dedicated server.
|
|
79
|
+
|
|
80
|
+
## Example
|
|
81
|
+
|
|
82
|
+
An ICON atmosphere simulation running month by month, restarting from its previous state, with its output streams post-processed by a user script:
|
|
83
|
+
|
|
84
|
+
```yaml
|
|
85
|
+
scheduler: slurm
|
|
86
|
+
cycles:
|
|
87
|
+
- monthly:
|
|
88
|
+
cycling:
|
|
89
|
+
start_date: "2000-01-01T00:00"
|
|
90
|
+
stop_date: "2000-07-01T00:00"
|
|
91
|
+
period: "P1M"
|
|
92
|
+
tasks:
|
|
93
|
+
- icon:
|
|
94
|
+
components:
|
|
95
|
+
atm:
|
|
96
|
+
inputs:
|
|
97
|
+
dynamics_grid_file: [icon grid]
|
|
98
|
+
restart_in:
|
|
99
|
+
- restart:
|
|
100
|
+
when:
|
|
101
|
+
after: "2000-01-01T00:00"
|
|
102
|
+
target_cycle:
|
|
103
|
+
lag: "-P1M"
|
|
104
|
+
outputs:
|
|
105
|
+
restart_out: [restart]
|
|
106
|
+
output_streams: [atm 2d, atm 3d]
|
|
107
|
+
- post proc:
|
|
108
|
+
inputs:
|
|
109
|
+
streams: [atm 2d, atm 3d]
|
|
110
|
+
outputs:
|
|
111
|
+
diagnostics: [diagnostics]
|
|
112
|
+
tasks:
|
|
113
|
+
- ROOT:
|
|
114
|
+
computer: santis
|
|
115
|
+
account: my_account
|
|
116
|
+
walltime: "01:00:00"
|
|
117
|
+
- SIROCCO: # For the standalone orchestration engine
|
|
118
|
+
walltime: "00:05:00"
|
|
119
|
+
venv: /path/to/sirocco/.venv
|
|
120
|
+
- icon:
|
|
121
|
+
plugin: icon
|
|
122
|
+
uenv: icon/25.2:v3
|
|
123
|
+
nodes: 4
|
|
124
|
+
procs_per_node: 128
|
|
125
|
+
exe:
|
|
126
|
+
cpu:
|
|
127
|
+
path: /path/to/icon
|
|
128
|
+
procs:
|
|
129
|
+
atm:
|
|
130
|
+
compute_weight: 1
|
|
131
|
+
streams: 2
|
|
132
|
+
namelists:
|
|
133
|
+
- ICON/icon_master.namelist
|
|
134
|
+
- ICON/NAMELIST_exclaim_ape_R02B04
|
|
135
|
+
- post proc:
|
|
136
|
+
plugin: shell
|
|
137
|
+
src: scripts/post_proc.sh
|
|
138
|
+
command: ./post_proc.sh {PORT::streams}
|
|
139
|
+
data:
|
|
140
|
+
available:
|
|
141
|
+
- icon grid:
|
|
142
|
+
path: /path/to/icon_grid_0013_R02B04_R.nc
|
|
143
|
+
generated:
|
|
144
|
+
- restart: {}
|
|
145
|
+
- atm 2d: {}
|
|
146
|
+
- atm 3d: {}
|
|
147
|
+
- diagnostics:
|
|
148
|
+
path: diagnostics
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
The namelists are files of the configuration directory, next to `sirocco.yaml`. See the [configuration](https://sirocco.ethz.ch/configuration/configuration/) and [ICON plugin](https://sirocco.ethz.ch/plugins/icon/) documentation for the complete syntax.
|
|
152
|
+
|
|
153
|
+
## Getting started
|
|
154
|
+
|
|
155
|
+
Sirocco requires Python `>=3.12` and, for the standalone engine, an HPC system using the [SLURM](https://slurm.schedmd.com/) scheduler (currently the only supported one). Install it on the HPC system, preferably with [uv](https://docs.astral.sh/uv/):
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
uv pip install sirocco
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Then, from the configuration directory (containing `sirocco.yaml`):
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
sirocco verify # validate the configuration
|
|
165
|
+
sirocco visualize # render the unrolled workflow graph as an svg
|
|
166
|
+
sirocco start # start the workflow
|
|
167
|
+
sirocco stviz # render the current state of the task front
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
See [Usage](https://sirocco.ethz.ch/usage/standalone/) for all commands, including `stop` and `restart`.
|
|
171
|
+
|
|
172
|
+
The AiiDA engine requires a development install (`git clone` then `uv sync --group aiida` or `hatch shell`), see the [AiiDA engine documentation](https://sirocco.ethz.ch/usage/aiida/).
|
|
173
|
+
|
|
174
|
+
## Development
|
|
175
|
+
|
|
176
|
+
Sirocco uses [hatch](https://hatch.pypa.io) to manage environments:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
git clone https://github.com/C2SM/Sirocco.git
|
|
180
|
+
cd Sirocco
|
|
181
|
+
hatch test # run the tests
|
|
182
|
+
hatch test --all # run the tests on all supported Python versions
|
|
183
|
+
hatch check fmt # check formatting
|
|
184
|
+
hatch check code # lint
|
|
185
|
+
hatch run types:check # type check
|
|
186
|
+
hatch run docs:serve # live preview of the documentation
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
## Contact
|
|
190
|
+
|
|
191
|
+
For questions about Sirocco or to discuss a use case, contact [Matthieu Leclair](mailto:matthieu.leclair@c2sm.ethz.ch). For bugs and feature requests, please open an [issue](https://github.com/C2SM/Sirocco/issues).
|
|
192
|
+
|
|
193
|
+
## License
|
|
194
|
+
|
|
195
|
+
Sirocco is released under the [MIT license](./LICENSE).
|
|
196
|
+
|
|
197
|
+
## Acknowledgments
|
|
198
|
+
|
|
199
|
+
We would like to acknowledge the [EXCLAIM](https://exclaim.ethz.ch) and [Swiss Twins](https://www.cscs.ch/about/collaborations/swisstwins) projects for the initial funding of Sirocco.
|
sirocco-0.1.0/README.md
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img alt="Sirocco logo" src="./docs/assets/Sirocco_Logo2026_BrunRVB.svg#gh-light-mode-only" width="500" />
|
|
3
|
+
<img alt="Sirocco logo" src="./docs/assets/Sirocco_Logo2026_BeigeRVB.svg#gh-dark-mode-only" width="500" />
|
|
4
|
+
</p>
|
|
5
|
+
|
|
6
|
+
<p align="center">
|
|
7
|
+
<a href="https://sirocco.ethz.ch"><b>Documentation</b></a> ·
|
|
8
|
+
<a href="https://sirocco.ethz.ch/usage/standalone/">Getting started</a> ·
|
|
9
|
+
<a href="./CHANGELOG.md">Changelog</a>
|
|
10
|
+
</p>
|
|
11
|
+
|
|
12
|
+
**Sirocco** is a Python workflow tool for climate and weather applications running on HPC systems. Workflows are described in a YAML dialect inspired by [Cylc](https://cylc.github.io/) and [AiiDA](https://www.aiida.net/), and unrolled into a graph of tasks and data.
|
|
13
|
+
|
|
14
|
+
- **Data as graph nodes.** Tasks explicitly declare the data they consume and produce, through named ports. No task needs to make assumptions on where others read or write, so workflows are truly composable.
|
|
15
|
+
- **Cycles.** Recurring sets of tasks with their own periodicity, so that describing climate and weather workflows feels natural.
|
|
16
|
+
- **Parameters.** Ensemble and sensitivity experiments through parameterized tasks and data.
|
|
17
|
+
- **Deep propagating task front.** Tasks are submitted ahead of time, chained by scheduler dependencies, in a window that moves through the graph as tasks complete. Tasks no longer wait for their parents to finish before entering the queue.
|
|
18
|
+
- **Plugins.** A general purpose `shell` plugin and a dedicated plugin for the [ICON](https://www.icon-model.org) model, covering complex setups such as coupled runs on hybrid CPU/GPU architectures.
|
|
19
|
+
- **Two engines.** A standalone engine running autonomously on the HPC system, with the scheduler as the sole daemon, and an [AiiDA](https://www.aiida.net/) based engine (under development) running on a dedicated server.
|
|
20
|
+
|
|
21
|
+
## Example
|
|
22
|
+
|
|
23
|
+
An ICON atmosphere simulation running month by month, restarting from its previous state, with its output streams post-processed by a user script:
|
|
24
|
+
|
|
25
|
+
```yaml
|
|
26
|
+
scheduler: slurm
|
|
27
|
+
cycles:
|
|
28
|
+
- monthly:
|
|
29
|
+
cycling:
|
|
30
|
+
start_date: "2000-01-01T00:00"
|
|
31
|
+
stop_date: "2000-07-01T00:00"
|
|
32
|
+
period: "P1M"
|
|
33
|
+
tasks:
|
|
34
|
+
- icon:
|
|
35
|
+
components:
|
|
36
|
+
atm:
|
|
37
|
+
inputs:
|
|
38
|
+
dynamics_grid_file: [icon grid]
|
|
39
|
+
restart_in:
|
|
40
|
+
- restart:
|
|
41
|
+
when:
|
|
42
|
+
after: "2000-01-01T00:00"
|
|
43
|
+
target_cycle:
|
|
44
|
+
lag: "-P1M"
|
|
45
|
+
outputs:
|
|
46
|
+
restart_out: [restart]
|
|
47
|
+
output_streams: [atm 2d, atm 3d]
|
|
48
|
+
- post proc:
|
|
49
|
+
inputs:
|
|
50
|
+
streams: [atm 2d, atm 3d]
|
|
51
|
+
outputs:
|
|
52
|
+
diagnostics: [diagnostics]
|
|
53
|
+
tasks:
|
|
54
|
+
- ROOT:
|
|
55
|
+
computer: santis
|
|
56
|
+
account: my_account
|
|
57
|
+
walltime: "01:00:00"
|
|
58
|
+
- SIROCCO: # For the standalone orchestration engine
|
|
59
|
+
walltime: "00:05:00"
|
|
60
|
+
venv: /path/to/sirocco/.venv
|
|
61
|
+
- icon:
|
|
62
|
+
plugin: icon
|
|
63
|
+
uenv: icon/25.2:v3
|
|
64
|
+
nodes: 4
|
|
65
|
+
procs_per_node: 128
|
|
66
|
+
exe:
|
|
67
|
+
cpu:
|
|
68
|
+
path: /path/to/icon
|
|
69
|
+
procs:
|
|
70
|
+
atm:
|
|
71
|
+
compute_weight: 1
|
|
72
|
+
streams: 2
|
|
73
|
+
namelists:
|
|
74
|
+
- ICON/icon_master.namelist
|
|
75
|
+
- ICON/NAMELIST_exclaim_ape_R02B04
|
|
76
|
+
- post proc:
|
|
77
|
+
plugin: shell
|
|
78
|
+
src: scripts/post_proc.sh
|
|
79
|
+
command: ./post_proc.sh {PORT::streams}
|
|
80
|
+
data:
|
|
81
|
+
available:
|
|
82
|
+
- icon grid:
|
|
83
|
+
path: /path/to/icon_grid_0013_R02B04_R.nc
|
|
84
|
+
generated:
|
|
85
|
+
- restart: {}
|
|
86
|
+
- atm 2d: {}
|
|
87
|
+
- atm 3d: {}
|
|
88
|
+
- diagnostics:
|
|
89
|
+
path: diagnostics
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The namelists are files of the configuration directory, next to `sirocco.yaml`. See the [configuration](https://sirocco.ethz.ch/configuration/configuration/) and [ICON plugin](https://sirocco.ethz.ch/plugins/icon/) documentation for the complete syntax.
|
|
93
|
+
|
|
94
|
+
## Getting started
|
|
95
|
+
|
|
96
|
+
Sirocco requires Python `>=3.12` and, for the standalone engine, an HPC system using the [SLURM](https://slurm.schedmd.com/) scheduler (currently the only supported one). Install it on the HPC system, preferably with [uv](https://docs.astral.sh/uv/):
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
uv pip install sirocco
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Then, from the configuration directory (containing `sirocco.yaml`):
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
sirocco verify # validate the configuration
|
|
106
|
+
sirocco visualize # render the unrolled workflow graph as an svg
|
|
107
|
+
sirocco start # start the workflow
|
|
108
|
+
sirocco stviz # render the current state of the task front
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
See [Usage](https://sirocco.ethz.ch/usage/standalone/) for all commands, including `stop` and `restart`.
|
|
112
|
+
|
|
113
|
+
The AiiDA engine requires a development install (`git clone` then `uv sync --group aiida` or `hatch shell`), see the [AiiDA engine documentation](https://sirocco.ethz.ch/usage/aiida/).
|
|
114
|
+
|
|
115
|
+
## Development
|
|
116
|
+
|
|
117
|
+
Sirocco uses [hatch](https://hatch.pypa.io) to manage environments:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
git clone https://github.com/C2SM/Sirocco.git
|
|
121
|
+
cd Sirocco
|
|
122
|
+
hatch test # run the tests
|
|
123
|
+
hatch test --all # run the tests on all supported Python versions
|
|
124
|
+
hatch check fmt # check formatting
|
|
125
|
+
hatch check code # lint
|
|
126
|
+
hatch run types:check # type check
|
|
127
|
+
hatch run docs:serve # live preview of the documentation
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Contact
|
|
131
|
+
|
|
132
|
+
For questions about Sirocco or to discuss a use case, contact [Matthieu Leclair](mailto:matthieu.leclair@c2sm.ethz.ch). For bugs and feature requests, please open an [issue](https://github.com/C2SM/Sirocco/issues).
|
|
133
|
+
|
|
134
|
+
## License
|
|
135
|
+
|
|
136
|
+
Sirocco is released under the [MIT license](./LICENSE).
|
|
137
|
+
|
|
138
|
+
## Acknowledgments
|
|
139
|
+
|
|
140
|
+
We would like to acknowledge the [EXCLAIM](https://exclaim.ethz.ch) and [Swiss Twins](https://www.cscs.ch/about/collaborations/swisstwins) projects for the initial funding of Sirocco.
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
# See https://www.python.org/dev/peps/pep-0621/
|
|
7
|
+
name = 'sirocco'
|
|
8
|
+
dynamic = ["version"]
|
|
9
|
+
description = 'A library for creating weather and climate workflows.'
|
|
10
|
+
authors = [
|
|
11
|
+
{name = 'Matthieu Leclair', email = 'matthieu.leclair@c2sm.ethz.ch'},
|
|
12
|
+
{name = 'Julian Geiger', email = 'julian.geiger@psi.ch'},
|
|
13
|
+
{name = 'Alexander Goscinski', email = 'alexander.goscinski@psi.ch'},
|
|
14
|
+
]
|
|
15
|
+
readme = 'README.md'
|
|
16
|
+
classifiers = [
|
|
17
|
+
'Development Status :: 3 - Alpha',
|
|
18
|
+
'Intended Audience :: Science/Research',
|
|
19
|
+
'Operating System :: POSIX :: Linux',
|
|
20
|
+
'Programming Language :: Python',
|
|
21
|
+
'Programming Language :: Python :: 3.12',
|
|
22
|
+
'Programming Language :: Python :: 3.13',
|
|
23
|
+
'Programming Language :: Python :: 3.14',
|
|
24
|
+
'Topic :: Scientific/Engineering',
|
|
25
|
+
'Topic :: Scientific/Engineering :: Atmospheric Science',
|
|
26
|
+
'License :: OSI Approved :: MIT License',
|
|
27
|
+
]
|
|
28
|
+
keywords = ["workflow"," icon", "aiida", "aiida-workgraph"]
|
|
29
|
+
requires-python = '>=3.12'
|
|
30
|
+
dependencies = [
|
|
31
|
+
"isoduration~=20.11",
|
|
32
|
+
"pydantic~=2.0",
|
|
33
|
+
"ruamel.yaml~=0.19",
|
|
34
|
+
"ipdb~=0.13",
|
|
35
|
+
"termcolor~=3.0",
|
|
36
|
+
"pygraphviz~=2.0",
|
|
37
|
+
"lxml~=6.0",
|
|
38
|
+
"f90nml~=1.5",
|
|
39
|
+
"rich~=15.0.0",
|
|
40
|
+
"typer~=0.27.2",
|
|
41
|
+
"jinja2~=3.0",
|
|
42
|
+
"types-PyYAML~=6.0",
|
|
43
|
+
"coverage~=7.0",
|
|
44
|
+
"isodate~=0.7",
|
|
45
|
+
"dotenv~=0.9",
|
|
46
|
+
]
|
|
47
|
+
license = {file = "LICENSE"}
|
|
48
|
+
|
|
49
|
+
[dependency-groups]
|
|
50
|
+
# Not published on PyPI, which rejects direct references: the AiiDA engine requires a
|
|
51
|
+
# development install, see `docs/usage/aiida.md`
|
|
52
|
+
aiida = [
|
|
53
|
+
"aiida-core==2.7.3",
|
|
54
|
+
# pinned to my branch with the temporary fix
|
|
55
|
+
"aiida-firecrest @ git+https://github.com/aiidateam/aiida-firecrest.git@c4e287ffe40ae57db7032fe1c80d2c4def0dc7fa",
|
|
56
|
+
# pinned on my branch that contains all the changes for DYAMOND, so far
|
|
57
|
+
"aiida-icon @ git+https://github.com/aiida-icon/aiida-icon.git@6a15937b3656242cfd9812b1dd5046216547b9ca",
|
|
58
|
+
"aiida-workgraph~=0.9",
|
|
59
|
+
"aiida-shell>=0.8.2,<1",
|
|
60
|
+
"aiida-gui~=0.2",
|
|
61
|
+
"aiida-gui-workgraph>=0.1.3,<1",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
[project.urls]
|
|
66
|
+
Repository = "https://github.com/C2SM/Sirocco.git"
|
|
67
|
+
Documentation = "https://sirocco.ethz.ch"
|
|
68
|
+
Changelog = "https://github.com/C2SM/Sirocco/blob/main/CHANGELOG.md"
|
|
69
|
+
|
|
70
|
+
[project.scripts]
|
|
71
|
+
sirocco = "sirocco.cli:app"
|
|
72
|
+
|
|
73
|
+
[tool.pytest.ini_options]
|
|
74
|
+
# Configuration for [pytest](https://docs.pytest.org)
|
|
75
|
+
log_level = "ERROR"
|
|
76
|
+
log_cli = true
|
|
77
|
+
log_cli_level = "ERROR"
|
|
78
|
+
log_cli_format = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
|
|
79
|
+
addopts = "--pdbcls=IPython.terminal.debugger:TerminalPdb"
|
|
80
|
+
norecursedirs = [
|
|
81
|
+
"tests/cases",
|
|
82
|
+
"docs"
|
|
83
|
+
]
|
|
84
|
+
markers = [
|
|
85
|
+
"slow: slow integration tests which are not recommended to run locally for normal development",
|
|
86
|
+
"requires_icon: marks test to require icon installation"
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
filterwarnings = [
|
|
90
|
+
"ignore::UserWarning",
|
|
91
|
+
'ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated:DeprecationWarning', # from aio_pika via duration
|
|
92
|
+
"ignore:There is no current event loop:DeprecationWarning", # from plumpy via aiida testing tools
|
|
93
|
+
"ignore:Object of type <DbNode> not in session:sqlalchemy.exc.SAWarning", # sqlalchemy via aiida testing tools
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
[tool.coverage.run]
|
|
97
|
+
# Configuration of [coverage.py](https://coverage.readthedocs.io)
|
|
98
|
+
# reporting which lines of your plugin are covered by tests
|
|
99
|
+
source = ["sirocco"]
|
|
100
|
+
|
|
101
|
+
[tool.ruff]
|
|
102
|
+
include = ["src/*py", "tests/*py"]
|
|
103
|
+
target-version = "py312"
|
|
104
|
+
|
|
105
|
+
[tool.ruff.lint]
|
|
106
|
+
ignore = [
|
|
107
|
+
"TID252", # prefer relative import over absolute
|
|
108
|
+
"TRY003", # write custom error messages for formatting
|
|
109
|
+
"PLC0415", # import should be at the top-level of a file (lazy imports)
|
|
110
|
+
"N806", # variable in function should be lowercase
|
|
111
|
+
"SLF001", # private member accessed
|
|
112
|
+
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
[tool.ruff.lint.per-file-ignores]
|
|
116
|
+
"tests/**/*.py" = [
|
|
117
|
+
"T201", # No use of `print`
|
|
118
|
+
"S101", # No use of `assert`
|
|
119
|
+
"PLR2004", # No use of magic values
|
|
120
|
+
"S607", # Starting a process with a partial executable path
|
|
121
|
+
"RUF015", # Prefer `next` over single element slice
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
## Hatch configurations
|
|
125
|
+
|
|
126
|
+
[tool.hatch.build.targets.sdist]
|
|
127
|
+
include = [
|
|
128
|
+
"src/sirocco/",
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
[tool.hatch.version]
|
|
132
|
+
path = "src/sirocco/__init__.py"
|
|
133
|
+
|
|
134
|
+
[tool.hatch.metadata]
|
|
135
|
+
allow-direct-references = true # editable installs
|
|
136
|
+
|
|
137
|
+
[tool.hatch.envs.default]
|
|
138
|
+
installer = "uv"
|
|
139
|
+
python = "3.12"
|
|
140
|
+
dependency-groups = ["aiida"]
|
|
141
|
+
|
|
142
|
+
[tool.hatch.envs.hatch-test]
|
|
143
|
+
installer = "uv"
|
|
144
|
+
dependency-groups = ["aiida"]
|
|
145
|
+
extra-dependencies = [
|
|
146
|
+
"ipdb",
|
|
147
|
+
"pytest-cov",
|
|
148
|
+
"pytest-sugar", # Nicer pytest output
|
|
149
|
+
"pytest-xdist", # Parallel test execution
|
|
150
|
+
"coverage", # For showing coverage gutter in IDEs
|
|
151
|
+
]
|
|
152
|
+
default-args = []
|
|
153
|
+
extra-args = ["--doctest-modules", "-m", "not slow and not requires_icon"]
|
|
154
|
+
|
|
155
|
+
[[tool.hatch.envs.hatch-test.matrix]]
|
|
156
|
+
python = ["3.12", "3.13", "3.14"]
|
|
157
|
+
|
|
158
|
+
[tool.hatch.envs.docs]
|
|
159
|
+
installer = "uv"
|
|
160
|
+
skip-install = true # Don't install sirocco itself for docs
|
|
161
|
+
description = "Build the documentation"
|
|
162
|
+
dependencies = [
|
|
163
|
+
"zensical",
|
|
164
|
+
"sirocco_yaml_lexer @ {root:uri}/docs/tools",
|
|
165
|
+
]
|
|
166
|
+
|
|
167
|
+
[tool.hatch.envs.docs.scripts]
|
|
168
|
+
build = [
|
|
169
|
+
"zensical build",
|
|
170
|
+
"echo 'Now open site/index.html with your browser'"
|
|
171
|
+
]
|
|
172
|
+
serve = [
|
|
173
|
+
"zensical serve",
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
[tool.hatch.envs.types]
|
|
177
|
+
python = "3.12"
|
|
178
|
+
dependency-groups = ["aiida"]
|
|
179
|
+
extra-dependencies = [
|
|
180
|
+
"mypy>=1.0.0",
|
|
181
|
+
"pytest",
|
|
182
|
+
"lxml-stubs",
|
|
183
|
+
"types-setuptools",
|
|
184
|
+
"types-docutils",
|
|
185
|
+
"types-colorama",
|
|
186
|
+
"types-Pygments",
|
|
187
|
+
"types-termcolor",
|
|
188
|
+
"types-requests",
|
|
189
|
+
"types-PyYAML",
|
|
190
|
+
"ty",
|
|
191
|
+
]
|
|
192
|
+
|
|
193
|
+
[tool.hatch.envs.types.scripts]
|
|
194
|
+
check = "mypy --exclude 'tests/cases/*' --exclude 'docs/assets/example_config' --exclude 'docs/tools' --no-incremental {args:.}"
|
|
195
|
+
ty-check = "ty check --python-version 3.12"
|
|
196
|
+
|
|
197
|
+
[tool.mypy]
|
|
198
|
+
disable_error_code = ["import-untyped"]
|
|
199
|
+
exclude = ["^tests/"]
|
|
200
|
+
|
|
201
|
+
[[tool.mypy.overrides]]
|
|
202
|
+
module = ["isoduration", "isoduration.*"]
|
|
203
|
+
follow_untyped_imports = true
|
|
204
|
+
|
|
205
|
+
[[tool.mypy.overrides]]
|
|
206
|
+
module = ["pygraphviz"]
|
|
207
|
+
ignore_missing_imports = true
|
|
208
|
+
|
|
209
|
+
[[tool.mypy.overrides]]
|
|
210
|
+
module = ["f90nml"]
|
|
211
|
+
ignore_missing_imports = true
|
|
212
|
+
|
|
213
|
+
[tool.ty.environment]
|
|
214
|
+
python-version = "3.12"
|
|
215
|
+
|
|
216
|
+
[tool.ty.src]
|
|
217
|
+
# Aligns with mypy's case exclusions using glob patterns
|
|
218
|
+
exclude = ["tests/cases/**", "tests/integration/engines/**", "tests/unit/engines/**", "src/sirocco/engines/**"]
|
|
219
|
+
|
|
220
|
+
[tool.ty.rules]
|
|
221
|
+
# Aligns with mypy's disable_error_code = ["import-untyped"] globally
|
|
222
|
+
import-not-found = "ignore"
|
|
223
|
+
# `ty: ignore` and `type: ignore` comments (the latter are meant for mypy)
|
|
224
|
+
unused-ignore-comment = "ignore"
|
|
225
|
+
unused-type-ignore-comment = "ignore"
|