python-workflow-definition 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/PKG-INFO +2 -2
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/pyproject.toml +28 -2
- python_workflow_definition-0.1.3/src/python_workflow_definition/__init__.py +3 -0
- python_workflow_definition-0.1.3/src/python_workflow_definition/_version.py +34 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/models.py +19 -2
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/pyiron_base.py +19 -5
- python_workflow_definition-0.1.2/.condarc +0 -3
- python_workflow_definition-0.1.2/.github/workflows/deploy.yml +0 -35
- python_workflow_definition-0.1.2/.github/workflows/pipeline.yml +0 -161
- python_workflow_definition-0.1.2/.github/workflows/pypi.yml +0 -31
- python_workflow_definition-0.1.2/CITATION.cff +0 -40
- python_workflow_definition-0.1.2/README.md +0 -119
- python_workflow_definition-0.1.2/binder/apt.txt +0 -2
- python_workflow_definition-0.1.2/binder/environment.yml +0 -17
- python_workflow_definition-0.1.2/binder/postBuild +0 -8
- python_workflow_definition-0.1.2/binder/setup-miniconda-patched-environment.yml +0 -18
- python_workflow_definition-0.1.2/documentation/_config.yml +0 -13
- python_workflow_definition-0.1.2/documentation/_toc.yml +0 -20
- python_workflow_definition-0.1.2/documentation/arithmetic.md +0 -44
- python_workflow_definition-0.1.2/documentation/conclusion.md +0 -3
- python_workflow_definition-0.1.2/documentation/environment.yml +0 -4
- python_workflow_definition-0.1.2/documentation/evcurve.md +0 -4
- python_workflow_definition-0.1.2/documentation/intro.md +0 -21
- python_workflow_definition-0.1.2/documentation/nfdi.md +0 -4
- python_workflow_definition-0.1.2/example_workflows/arithmetic/aiida.ipynb +0 -270
- python_workflow_definition-0.1.2/example_workflows/arithmetic/cwl.ipynb +0 -97
- python_workflow_definition-0.1.2/example_workflows/arithmetic/jobflow.ipynb +0 -410
- python_workflow_definition-0.1.2/example_workflows/arithmetic/pyiron_base.ipynb +0 -322
- python_workflow_definition-0.1.2/example_workflows/arithmetic/pyiron_workflow.ipynb +0 -920
- python_workflow_definition-0.1.2/example_workflows/arithmetic/universal_workflow.ipynb +0 -357
- python_workflow_definition-0.1.2/example_workflows/arithmetic/workflow.json +0 -19
- python_workflow_definition-0.1.2/example_workflows/arithmetic/workflow.py +0 -10
- python_workflow_definition-0.1.2/example_workflows/nfdi/aiida.ipynb +0 -398
- python_workflow_definition-0.1.2/example_workflows/nfdi/cwl.ipynb +0 -159
- python_workflow_definition-0.1.2/example_workflows/nfdi/jobflow.ipynb +0 -370
- python_workflow_definition-0.1.2/example_workflows/nfdi/pyiron_base.ipynb +0 -355
- python_workflow_definition-0.1.2/example_workflows/nfdi/pyiron_workflow.ipynb +0 -1
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/envs/postprocessing.yaml +0 -5
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/envs/preprocessing.yaml +0 -5
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/envs/processing.yaml +0 -5
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/macros.tex.template +0 -3
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/paper.tex +0 -22
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/poisson.py +0 -108
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/postprocessing.py +0 -54
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/prepare_paper_macros.py +0 -43
- python_workflow_definition-0.1.2/example_workflows/nfdi/source/unit_square.geo +0 -9
- python_workflow_definition-0.1.2/example_workflows/nfdi/universal_workflow.ipynb +0 -357
- python_workflow_definition-0.1.2/example_workflows/nfdi/workflow.json +0 -33
- python_workflow_definition-0.1.2/example_workflows/nfdi/workflow.py +0 -135
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/aiida.ipynb +0 -5799
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/cwl.ipynb +0 -117
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/environment.yml +0 -9
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/espresso/pseudo/Al.pbe-n-kjpaw_psl.1.0.0.UPF +0 -18497
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/jobflow.ipynb +0 -5747
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/pyiron_base.ipynb +0 -4993
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/pyiron_workflow.ipynb +0 -6015
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/qe_xml_parser/pyproject.toml +0 -13
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/qe_xml_parser/src/qe_xml_parser/__init__.py +0 -0
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/qe_xml_parser/src/qe_xml_parser/parsers.py +0 -70
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/qe_xml_parser/src/qe_xml_parser/schemas/__init__.py +0 -0
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/qe_xml_parser/src/qe_xml_parser/schemas/qes_230310.xsd +0 -1384
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/universal_workflow.ipynb +0 -505
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/workflow.json +0 -100
- python_workflow_definition-0.1.2/example_workflows/quantum_espresso/workflow.py +0 -88
- python_workflow_definition-0.1.2/src/python_workflow_definition/__init__.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/.gitignore +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/LICENSE +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/aiida.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/cwl/__init__.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/cwl/__main__.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/executorlib.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/jobflow.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/plot.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/purepython.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/pyiron_workflow.py +0 -0
- {python_workflow_definition-0.1.2 → python_workflow_definition-0.1.3}/src/python_workflow_definition/shared.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python_workflow_definition
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Python Workflow Definition - workflow interoperability for aiida, jobflow and pyiron
|
|
5
5
|
Author-email: Jan Janssen <janssen@mpie.de>, Janine George <janine.geogre@bam.de>, Julian Geiger <julian.geiger@psi.ch>, Xing Wang <xing.wang@psi.ch>, Marnik Bercx <marnik.bercx@psi.ch>, Christina Ertural <christina.ertural@bam.de>
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -44,4 +44,4 @@ Requires-Dist: ipython<=9.8.0,>=7.33.0; extra == 'plot'
|
|
|
44
44
|
Requires-Dist: networkx<=3.5,>=2.8.8; extra == 'plot'
|
|
45
45
|
Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'plot'
|
|
46
46
|
Provides-Extra: pyiron
|
|
47
|
-
Requires-Dist: pyiron-base<=0.15.
|
|
47
|
+
Requires-Dist: pyiron-base<=0.15.12,>=0.11.10; extra == 'pyiron'
|
|
@@ -4,7 +4,6 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "python_workflow_definition"
|
|
7
|
-
version = "0.1.2"
|
|
8
7
|
description = "Python Workflow Definition - workflow interoperability for aiida, jobflow and pyiron"
|
|
9
8
|
authors = [
|
|
10
9
|
{ name = "Jan Janssen", email = "janssen@mpie.de" },
|
|
@@ -19,6 +18,7 @@ dependencies = [
|
|
|
19
18
|
"numpy>=1.21",
|
|
20
19
|
"pydantic>=2.7.0,<=2.12.4",
|
|
21
20
|
]
|
|
21
|
+
dynamic = ["version"]
|
|
22
22
|
|
|
23
23
|
[project.optional-dependencies]
|
|
24
24
|
aiida = [
|
|
@@ -28,10 +28,36 @@ jobflow = [
|
|
|
28
28
|
"jobflow>=0.1.18,<=0.2.1",
|
|
29
29
|
]
|
|
30
30
|
pyiron = [
|
|
31
|
-
"pyiron_base>=0.11.10,<=0.15.
|
|
31
|
+
"pyiron_base>=0.11.10,<=0.15.12",
|
|
32
32
|
]
|
|
33
33
|
plot = [
|
|
34
34
|
"pygraphviz>=1.10,<=1.14",
|
|
35
35
|
"networkx>=2.8.8,<=3.5",
|
|
36
36
|
"ipython>=7.33.0,<=9.8.0",
|
|
37
37
|
]
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build]
|
|
40
|
+
include = [
|
|
41
|
+
"src/python_workflow_definition"
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.hooks.vcs]
|
|
45
|
+
version-file = "src/python_workflow_definition/_version.py"
|
|
46
|
+
|
|
47
|
+
[tool.hatch.build.targets.sdist]
|
|
48
|
+
include = [
|
|
49
|
+
"src/python_workflow_definition"
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.wheel]
|
|
53
|
+
packages = [
|
|
54
|
+
"src/python_workflow_definition"
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[tool.hatch.version]
|
|
58
|
+
source = "vcs"
|
|
59
|
+
path = "src/python_workflow_definition/_version.py"
|
|
60
|
+
|
|
61
|
+
[tool.coverage.run]
|
|
62
|
+
source = ["python_workflow_definition"]
|
|
63
|
+
command_line = "-m unittest discover tests"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"__version_tuple__",
|
|
7
|
+
"version",
|
|
8
|
+
"version_tuple",
|
|
9
|
+
"__commit_id__",
|
|
10
|
+
"commit_id",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
TYPE_CHECKING = False
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from typing import Tuple
|
|
16
|
+
from typing import Union
|
|
17
|
+
|
|
18
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
+
COMMIT_ID = Union[str, None]
|
|
20
|
+
else:
|
|
21
|
+
VERSION_TUPLE = object
|
|
22
|
+
COMMIT_ID = object
|
|
23
|
+
|
|
24
|
+
version: str
|
|
25
|
+
__version__: str
|
|
26
|
+
__version_tuple__: VERSION_TUPLE
|
|
27
|
+
version_tuple: VERSION_TUPLE
|
|
28
|
+
commit_id: COMMIT_ID
|
|
29
|
+
__commit_id__: COMMIT_ID
|
|
30
|
+
|
|
31
|
+
__version__ = version = '0.1.3'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 1, 3)
|
|
33
|
+
|
|
34
|
+
__commit_id__ = commit_id = None
|
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import (
|
|
3
|
+
List,
|
|
4
|
+
Union,
|
|
5
|
+
Optional,
|
|
6
|
+
Literal,
|
|
7
|
+
Any,
|
|
8
|
+
Annotated,
|
|
9
|
+
Type,
|
|
10
|
+
TypeVar,
|
|
11
|
+
)
|
|
12
|
+
from typing_extensions import TypeAliasType
|
|
3
13
|
from pydantic import BaseModel, Field, field_validator, field_serializer
|
|
4
14
|
from pydantic import ValidationError
|
|
5
15
|
import json
|
|
@@ -19,6 +29,13 @@ __all__ = (
|
|
|
19
29
|
)
|
|
20
30
|
|
|
21
31
|
|
|
32
|
+
JsonPrimitive = Union[str, int, float, bool, None]
|
|
33
|
+
AllowableDefaults = TypeAliasType(
|
|
34
|
+
"AllowableDefaults",
|
|
35
|
+
"Union[JsonPrimitive, dict[str, AllowableDefaults], list[AllowableDefaults]]",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
22
39
|
class PythonWorkflowDefinitionBaseNode(BaseModel):
|
|
23
40
|
"""Base model for all node types, containing common fields."""
|
|
24
41
|
|
|
@@ -33,7 +50,7 @@ class PythonWorkflowDefinitionInputNode(PythonWorkflowDefinitionBaseNode):
|
|
|
33
50
|
|
|
34
51
|
type: Literal["input"]
|
|
35
52
|
name: str
|
|
36
|
-
value: Optional[
|
|
53
|
+
value: Optional[AllowableDefaults] = None
|
|
37
54
|
|
|
38
55
|
|
|
39
56
|
class PythonWorkflowDefinitionOutputNode(PythonWorkflowDefinitionBaseNode):
|
|
@@ -285,18 +285,32 @@ def write_workflow_json(
|
|
|
285
285
|
)
|
|
286
286
|
|
|
287
287
|
nodes_store_lst = []
|
|
288
|
-
|
|
288
|
+
translate_dict = {}
|
|
289
|
+
for i, k in enumerate(list(nodes_new_dict.keys())[::-1]):
|
|
290
|
+
v = nodes_new_dict[k]
|
|
291
|
+
translate_dict[k] = i
|
|
289
292
|
if isfunction(v):
|
|
290
293
|
mod = v.__module__
|
|
291
294
|
if mod == "python_workflow_definition.pyiron_base":
|
|
292
295
|
mod = "python_workflow_definition.shared"
|
|
293
296
|
nodes_store_lst.append(
|
|
294
|
-
{"id":
|
|
297
|
+
{"id": i, "type": "function", "value": mod + "." + v.__name__}
|
|
295
298
|
)
|
|
296
299
|
elif isinstance(v, np.ndarray):
|
|
297
|
-
nodes_store_lst.append({"id":
|
|
300
|
+
nodes_store_lst.append({"id": i, "type": "input", "value": v.tolist()})
|
|
298
301
|
else:
|
|
299
|
-
nodes_store_lst.append({"id":
|
|
302
|
+
nodes_store_lst.append({"id": i, "type": "input", "value": v})
|
|
303
|
+
|
|
304
|
+
print(translate_dict)
|
|
305
|
+
edges_store_lst = [
|
|
306
|
+
{
|
|
307
|
+
TARGET_LABEL: translate_dict[edge[TARGET_LABEL]],
|
|
308
|
+
TARGET_PORT_LABEL: edge[TARGET_PORT_LABEL],
|
|
309
|
+
SOURCE_LABEL: translate_dict[edge[SOURCE_LABEL]],
|
|
310
|
+
SOURCE_PORT_LABEL: edge[SOURCE_PORT_LABEL],
|
|
311
|
+
}
|
|
312
|
+
for edge in edges_new_lst
|
|
313
|
+
]
|
|
300
314
|
|
|
301
315
|
PythonWorkflowDefinitionWorkflow(
|
|
302
316
|
**set_result_node(
|
|
@@ -304,7 +318,7 @@ def write_workflow_json(
|
|
|
304
318
|
workflow_dict={
|
|
305
319
|
VERSION_LABEL: VERSION_NUMBER,
|
|
306
320
|
NODES_LABEL: nodes_store_lst,
|
|
307
|
-
EDGES_LABEL:
|
|
321
|
+
EDGES_LABEL: edges_store_lst,
|
|
308
322
|
}
|
|
309
323
|
)
|
|
310
324
|
)
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
name: Deploy
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches: [ main ]
|
|
6
|
-
|
|
7
|
-
jobs:
|
|
8
|
-
documentation:
|
|
9
|
-
runs-on: ubuntu-latest
|
|
10
|
-
steps:
|
|
11
|
-
- uses: actions/checkout@v4
|
|
12
|
-
- name: Conda config
|
|
13
|
-
run: echo -e "channels:\n - conda-forge\n" > .condarc
|
|
14
|
-
- uses: conda-incubator/setup-miniconda@v3
|
|
15
|
-
with:
|
|
16
|
-
python-version: "3.12"
|
|
17
|
-
miniforge-version: latest
|
|
18
|
-
condarc-file: .condarc
|
|
19
|
-
environment-file: documentation/environment.yml
|
|
20
|
-
- name: Install Jupyterbook
|
|
21
|
-
shell: bash -l {0}
|
|
22
|
-
run: |
|
|
23
|
-
cp documentation/_config.yml .
|
|
24
|
-
cp documentation/_toc.yml .
|
|
25
|
-
cp documentation/*.md .
|
|
26
|
-
jupyter-book build . --path-output public
|
|
27
|
-
mv public/_build/html public_html
|
|
28
|
-
touch public_html/.nojekyll
|
|
29
|
-
- name: Deploy 🚀
|
|
30
|
-
uses: JamesIves/github-pages-deploy-action@3.7.1
|
|
31
|
-
with:
|
|
32
|
-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
33
|
-
BRANCH: gh-pages # The branch the action should deploy to.
|
|
34
|
-
FOLDER: public_html # The folder the action should deploy.
|
|
35
|
-
CLEAN: true
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
name: Pipeline
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches: [ main ]
|
|
6
|
-
pull_request:
|
|
7
|
-
|
|
8
|
-
jobs:
|
|
9
|
-
black:
|
|
10
|
-
runs-on: ubuntu-latest
|
|
11
|
-
steps:
|
|
12
|
-
- uses: actions/checkout@v4
|
|
13
|
-
- uses: psf/black@stable
|
|
14
|
-
with:
|
|
15
|
-
options: "--check --diff"
|
|
16
|
-
src: ./src/python_workflow_definition
|
|
17
|
-
- uses: psf/black@stable
|
|
18
|
-
with:
|
|
19
|
-
options: "--check --diff"
|
|
20
|
-
src: ./example_workflows/quantum_espresso/qe_xml_parser/src/qe_xml_parser
|
|
21
|
-
|
|
22
|
-
pip_check:
|
|
23
|
-
runs-on: ubuntu-latest
|
|
24
|
-
steps:
|
|
25
|
-
- uses: actions/checkout@v4
|
|
26
|
-
- name: Conda config
|
|
27
|
-
run: echo -e "channels:\n - conda-forge\n" > .condarc
|
|
28
|
-
- name: Setup Mambaforge
|
|
29
|
-
uses: conda-incubator/setup-miniconda@v3
|
|
30
|
-
with:
|
|
31
|
-
python-version: "3.12"
|
|
32
|
-
miniforge-version: latest
|
|
33
|
-
condarc-file: .condarc
|
|
34
|
-
environment-file: binder/environment.yml
|
|
35
|
-
- name: Pip check
|
|
36
|
-
shell: bash -l {0}
|
|
37
|
-
run: |
|
|
38
|
-
pip install --no-deps --no-build-isolation -e .
|
|
39
|
-
pip check
|
|
40
|
-
|
|
41
|
-
nfdi4ing:
|
|
42
|
-
runs-on: ubuntu-22.04
|
|
43
|
-
steps:
|
|
44
|
-
- uses: actions/checkout@v4
|
|
45
|
-
- name: Conda config
|
|
46
|
-
run: echo -e "channels:\n - conda-forge\n" > .condarc
|
|
47
|
-
- uses: conda-incubator/setup-miniconda@v3
|
|
48
|
-
with:
|
|
49
|
-
python-version: "3.12"
|
|
50
|
-
miniforge-version: latest
|
|
51
|
-
condarc-file: .condarc
|
|
52
|
-
environment-file: binder/environment.yml
|
|
53
|
-
- name: Installation and setup
|
|
54
|
-
shell: bash -l {0}
|
|
55
|
-
run: |
|
|
56
|
-
pip install --no-deps --no-build-isolation -e .
|
|
57
|
-
conda install -c conda-forge jupyter papermill
|
|
58
|
-
verdi presto --profile-name pwd
|
|
59
|
-
sudo apt-get install -y $(cat binder/apt.txt)
|
|
60
|
-
- name: Create Additional Conda Environments
|
|
61
|
-
shell: bash -l {0}
|
|
62
|
-
run: |
|
|
63
|
-
conda env create -n preprocessing -f example_workflows/nfdi/source/envs/preprocessing.yaml -y
|
|
64
|
-
conda env create -n processing -f example_workflows/nfdi/source/envs/processing.yaml -y
|
|
65
|
-
conda env create -n postprocessing -f example_workflows/nfdi/source/envs/postprocessing.yaml -y
|
|
66
|
-
- name: Tests
|
|
67
|
-
shell: bash -l {0}
|
|
68
|
-
run: |
|
|
69
|
-
cd example_workflows/nfdi/
|
|
70
|
-
papermill aiida.ipynb aiida_out.ipynb -k "python3"
|
|
71
|
-
papermill cwl.ipynb cwl_out.ipynb -k "python3"
|
|
72
|
-
papermill jobflow.ipynb jobflow_out.ipynb -k "python3"
|
|
73
|
-
papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3"
|
|
74
|
-
papermill pyiron_workflow.ipynb pyiron_workflow_out.ipynb -k "python3"
|
|
75
|
-
papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3"
|
|
76
|
-
|
|
77
|
-
quantumespresso:
|
|
78
|
-
runs-on: ubuntu-latest
|
|
79
|
-
steps:
|
|
80
|
-
- uses: actions/checkout@v4
|
|
81
|
-
- name: Setup environment
|
|
82
|
-
shell: bash -l {0}
|
|
83
|
-
run: |
|
|
84
|
-
echo -e "channels:\n - conda-forge\n" > .condarc
|
|
85
|
-
tail --lines=+4 example_workflows/quantum_espresso/environment.yml >> binder/environment.yml
|
|
86
|
-
- uses: conda-incubator/setup-miniconda@v3
|
|
87
|
-
with:
|
|
88
|
-
python-version: "3.12"
|
|
89
|
-
miniforge-version: latest
|
|
90
|
-
condarc-file: .condarc
|
|
91
|
-
environment-file: binder/environment.yml
|
|
92
|
-
- name: Installation and setup
|
|
93
|
-
shell: bash -l {0}
|
|
94
|
-
run: |
|
|
95
|
-
pip install --no-deps --no-build-isolation -e example_workflows/quantum_espresso/qe_xml_parser
|
|
96
|
-
pip install --no-deps --no-build-isolation -e .
|
|
97
|
-
conda install -c conda-forge jupyter papermill
|
|
98
|
-
verdi presto --profile-name pwd
|
|
99
|
-
- name: Tests
|
|
100
|
-
shell: bash -l {0}
|
|
101
|
-
run: |
|
|
102
|
-
export ESPRESSO_PSEUDO=$(pwd)/espresso/pseudo
|
|
103
|
-
cp -r example_workflows/quantum_espresso/espresso .
|
|
104
|
-
cd example_workflows/quantum_espresso
|
|
105
|
-
papermill aiida.ipynb aiida_out.ipynb -k "python3"
|
|
106
|
-
papermill cwl.ipynb cwl_out.ipynb -k "python3"
|
|
107
|
-
papermill jobflow.ipynb jobflow_out.ipynb -k "python3"
|
|
108
|
-
papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3"
|
|
109
|
-
papermill pyiron_workflow.ipynb pyiron_workflow_out.ipynb -k "python3"
|
|
110
|
-
papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3"
|
|
111
|
-
|
|
112
|
-
arithmetic:
|
|
113
|
-
runs-on: ubuntu-latest
|
|
114
|
-
steps:
|
|
115
|
-
- uses: actions/checkout@v4
|
|
116
|
-
- name: Conda config
|
|
117
|
-
run: echo -e "channels:\n - conda-forge\n" > .condarc
|
|
118
|
-
- uses: conda-incubator/setup-miniconda@v3
|
|
119
|
-
with:
|
|
120
|
-
python-version: "3.12"
|
|
121
|
-
miniforge-version: latest
|
|
122
|
-
condarc-file: .condarc
|
|
123
|
-
environment-file: binder/environment.yml
|
|
124
|
-
- name: Installation and setup
|
|
125
|
-
shell: bash -l {0}
|
|
126
|
-
run: |
|
|
127
|
-
pip install --no-deps --no-build-isolation -e .
|
|
128
|
-
conda install -c conda-forge jupyter papermill
|
|
129
|
-
verdi presto --profile-name pwd
|
|
130
|
-
- name: Tests
|
|
131
|
-
shell: bash -l {0}
|
|
132
|
-
run: |
|
|
133
|
-
cd example_workflows/arithmetic
|
|
134
|
-
papermill aiida.ipynb aiida_out.ipynb -k "python3"
|
|
135
|
-
papermill cwl.ipynb cwl_out.ipynb -k "python3"
|
|
136
|
-
papermill jobflow.ipynb jobflow_out.ipynb -k "python3"
|
|
137
|
-
papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3"
|
|
138
|
-
papermill pyiron_workflow.ipynb pyiron_workflow_out.ipynb -k "python3"
|
|
139
|
-
papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3"
|
|
140
|
-
|
|
141
|
-
documentation:
|
|
142
|
-
runs-on: ubuntu-latest
|
|
143
|
-
steps:
|
|
144
|
-
- uses: actions/checkout@v4
|
|
145
|
-
- name: Conda config
|
|
146
|
-
run: echo -e "channels:\n - conda-forge\n" > .condarc
|
|
147
|
-
- uses: conda-incubator/setup-miniconda@v3
|
|
148
|
-
with:
|
|
149
|
-
python-version: "3.12"
|
|
150
|
-
miniforge-version: latest
|
|
151
|
-
condarc-file: .condarc
|
|
152
|
-
environment-file: documentation/environment.yml
|
|
153
|
-
- name: Install Jupyterbook
|
|
154
|
-
shell: bash -l {0}
|
|
155
|
-
run: |
|
|
156
|
-
cp documentation/_config.yml .
|
|
157
|
-
cp documentation/_toc.yml .
|
|
158
|
-
cp documentation/*.md .
|
|
159
|
-
jupyter-book build . --path-output public
|
|
160
|
-
mv public/_build/html public_html
|
|
161
|
-
touch public_html/.nojekyll
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
name: Pypi
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
pull_request:
|
|
6
|
-
workflow_dispatch:
|
|
7
|
-
|
|
8
|
-
jobs:
|
|
9
|
-
release:
|
|
10
|
-
if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
environment:
|
|
13
|
-
name: pypi
|
|
14
|
-
url: https://pypi.org/p/python_workflow_definition
|
|
15
|
-
permissions:
|
|
16
|
-
id-token: write
|
|
17
|
-
steps:
|
|
18
|
-
- uses: actions/checkout@v4
|
|
19
|
-
- name: Conda config
|
|
20
|
-
run: echo -e "channels:\n - conda-forge\n" > .condarc
|
|
21
|
-
- uses: conda-incubator/setup-miniconda@v3
|
|
22
|
-
with:
|
|
23
|
-
python-version: "3.12"
|
|
24
|
-
miniforge-version: latest
|
|
25
|
-
condarc-file: .condarc
|
|
26
|
-
environment-file: binder/environment.yml
|
|
27
|
-
- name: Build
|
|
28
|
-
shell: bash -l {0}
|
|
29
|
-
run: hatchling build -t sdist -t wheel
|
|
30
|
-
- name: Publish distribution 📦 to PyPI
|
|
31
|
-
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
cff-version: "1.2.0"
|
|
2
|
-
authors:
|
|
3
|
-
- family-names: Janssen
|
|
4
|
-
given-names: Jan
|
|
5
|
-
orcid: "https://orcid.org/0000-0001-9948-7119"
|
|
6
|
-
- family-names: George
|
|
7
|
-
given-names: Janine
|
|
8
|
-
orcid: "https://orcid.org/0000-0001-8907-0336"
|
|
9
|
-
- family-names: Geiger
|
|
10
|
-
given-names: Julian
|
|
11
|
-
orcid: "https://orcid.org/0000-0003-0023-1960"
|
|
12
|
-
- family-names: Bercx
|
|
13
|
-
given-names: Marnik
|
|
14
|
-
orcid: "https://orcid.org/0000-0001-8470-1012"
|
|
15
|
-
- family-names: Wang
|
|
16
|
-
given-names: Xing
|
|
17
|
-
- family-names: Ertural
|
|
18
|
-
given-names: Christina
|
|
19
|
-
orcid: "https://orcid.org/0000-0002-7696-5824"
|
|
20
|
-
- family-names: Schaarschmidt
|
|
21
|
-
given-names: Joerg
|
|
22
|
-
orcid: "https://orcid.org/0000-0002-4389-2366"
|
|
23
|
-
- family-names: Ganose
|
|
24
|
-
given-names: Alex
|
|
25
|
-
orcid: "https://orcid.org/0000-0002-4486-3321"
|
|
26
|
-
- family-names: Pizzi
|
|
27
|
-
given-names: Giovanni
|
|
28
|
-
orcid: "https://orcid.org/0000-0002-3583-4377"
|
|
29
|
-
- family-names: Hickel
|
|
30
|
-
given-names: Tilmann
|
|
31
|
-
orcid: "https://orcid.org/0000-0003-0698-4891"
|
|
32
|
-
- family-names: Neugebauer
|
|
33
|
-
given-names: Joerg
|
|
34
|
-
orcid: "https://orcid.org/0000-0002-7903-2472"
|
|
35
|
-
title: "A Python workflow definition for computational materials design"
|
|
36
|
-
version: 0.1.1
|
|
37
|
-
doi: 10.1039/D5DD00231A
|
|
38
|
-
date-released: 2025-10-25
|
|
39
|
-
license: bsd-3-clause
|
|
40
|
-
url: "https://github.com/pythonworkflow/python-workflow-definition"
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
# Python Workflow Definition
|
|
2
|
-
[](https://github.com/pythonworkflow/python-workflow-definition/actions/workflows/pipeline.yml)
|
|
3
|
-
[](https://mybinder.org/v2/gh/pythonworkflow/python-workflow-definition/HEAD)
|
|
4
|
-
[](https://doi.org/10.5281/zenodo.15516179)
|
|
5
|
-
|
|
6
|
-
## Definition
|
|
7
|
-
In the Python Workflow Definition (PWD) each node represents a Python function, with the edges defining the connection
|
|
8
|
-
between input and output of the different Python functions.
|
|
9
|
-
|
|
10
|
-
Published in: [J. Janssen, J. George, J. Geiger, M. Bercx, X. Wang, C. Ertural, J. Schaarschmidt, A.M. Ganose, G. Pizzi, T. Hickel and J. Neugebauer. A python workflow definition for computational materials design. Digital Discovery, 2025](https://doi.org/10.1039/D5DD00231A)
|
|
11
|
-
|
|
12
|
-
## Format
|
|
13
|
-
Each workflow consists of three files, a Python module which defines the individual Pythons, a JSON file which defines
|
|
14
|
-
the connections between the different Python functions and a conda environment file to define the software dependencies.
|
|
15
|
-
The files are not intended to be human readable, but rather interact as a machine readable exchange format between the
|
|
16
|
-
different workflow engines to enable interoperability.
|
|
17
|
-
|
|
18
|
-
## Installation
|
|
19
|
-
The Python Workflow Definition can either be installed via pypi or via conda. For the [pypi installation](https://pypi.org/project/python-workflow-definition/) use:
|
|
20
|
-
```
|
|
21
|
-
pip install python-workflow-definition
|
|
22
|
-
```
|
|
23
|
-
For the conda installation via the [conda-forge community channel](https://anaconda.org/conda-forge/python-workflow-definition) use:
|
|
24
|
-
```
|
|
25
|
-
conda install conda-forge::python-workflow-definition
|
|
26
|
-
```
|
|
27
|
-
|
|
28
|
-
## Examples
|
|
29
|
-
### Simple Example
|
|
30
|
-
As a first example we define two Python functions which add multiple inputs:
|
|
31
|
-
```python
|
|
32
|
-
def get_sum(x, y):
|
|
33
|
-
return x + y
|
|
34
|
-
|
|
35
|
-
def get_prod_and_div(x: float, y: float) -> dict:
|
|
36
|
-
return {"prod": x * y, "div": x / y}
|
|
37
|
-
```
|
|
38
|
-
These two Python functions are combined in the following example workflow:
|
|
39
|
-
```python
|
|
40
|
-
def combined_workflow(x=1, y=2):
|
|
41
|
-
tmp_dict = get_prod_and_div(x=x, y=y)
|
|
42
|
-
return get_sum(x=tmp_dict["prod"], y=tmp_dict["div"])
|
|
43
|
-
```
|
|
44
|
-
For the workflow representation of these Python functions the Python functions are stored in the [example_workflows/arithmetic/workflow.py](example_workflows/arithmetic/workflow.py)
|
|
45
|
-
Python module. The connection of the Python functions are stored in the [example_workflows/arithmetic/workflow.json](example_workflows/arithmetic/workflow.json)
|
|
46
|
-
JSON file:
|
|
47
|
-
```JSON
|
|
48
|
-
{
|
|
49
|
-
"nodes": [
|
|
50
|
-
{"id": 0, "type": "function", "value": "workflow.get_prod_and_div"},
|
|
51
|
-
{"id": 1, "type": "function", "value": "workflow.get_sum"},
|
|
52
|
-
{"id": 2, "type": "input", "value": 1, "name": "x"},
|
|
53
|
-
{"id": 3, "type": "input", "value": 2, "name": "y"},
|
|
54
|
-
{"id": 4, "type": "output", "name": "result"}
|
|
55
|
-
],
|
|
56
|
-
"edges": [
|
|
57
|
-
{"target": 0, "targetPort": "x", "source": 2, "sourcePort": null},
|
|
58
|
-
{"target": 0, "targetPort": "y", "source": 3, "sourcePort": null},
|
|
59
|
-
{"target": 1, "targetPort": "x", "source": 0, "sourcePort": "prod"},
|
|
60
|
-
{"target": 1, "targetPort": "y", "source": 0, "sourcePort": "div"},
|
|
61
|
-
{"target": 4, "targetPort": null, "source": 1, "sourcePort": null}
|
|
62
|
-
]
|
|
63
|
-
}
|
|
64
|
-
```
|
|
65
|
-
The abbreviations in the definition of the edges are:
|
|
66
|
-
* `target` - target node
|
|
67
|
-
* `targetPort` - target port - for a node with multiple input parameters the target port specifies which input parameter to use.
|
|
68
|
-
* `source` - source node
|
|
69
|
-
* `sourcePort` - source port - for a node with multiple output parameters the source port specifies which output parameter to use.
|
|
70
|
-
|
|
71
|
-
As the workflow does not require any additional resources, as it is only using built-in functionality of the Python standard
|
|
72
|
-
library.
|
|
73
|
-
|
|
74
|
-
The corresponding Jupyter notebooks demonstrate this functionality:
|
|
75
|
-
|
|
76
|
-
| Example | Explanation |
|
|
77
|
-
|----------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
|
|
78
|
-
| [example_workflows/arithmetic/aiida.ipynb](example_workflows/arithmetic/aiida.ipynb) | Define Workflow with aiida and execute it with jobflow and pyiron_base. |
|
|
79
|
-
| [example_workflows/arithmetic/jobflow.ipynb](example_workflows/arithmetic/jobflow.ipynb) | Define Workflow with jobflow and execute it with aiida and pyiron_base. |
|
|
80
|
-
| [example_workflows/arithmetic/pyiron_base.ipynb](example_workflows/arithmetic/pyiron_base.ipynb) | Define Workflow with pyiron_base and execute it with aiida and jobflow. |
|
|
81
|
-
| [example_workflows/arithmetic/universal_workflow.ipynb](example_workflows/arithmetic/universal_workflow.ipynb) | Execute workflow defined in the Python Workflow Definition with aiida, executorlib, jobflow, pyiron_base and pure Python. |
|
|
82
|
-
|
|
83
|
-
### Quantum Espresso Workflow
|
|
84
|
-
The second workflow example is the calculation of an energy volume curve with Quantum Espresso. In the first step the
|
|
85
|
-
initial structure is relaxed, afterward it is strained and the total energy is calculated.
|
|
86
|
-
* [example_workflows/quantum_espresso/workflow.py](example_workflows/quantum_espresso/workflow.py) Python functions
|
|
87
|
-
* [example_workflows/quantum_espresso/workflow.json](example_workflows/quantum_espresso/workflow.json) Workflow definition in the Python Workflow Definition.
|
|
88
|
-
* [example_workflows/quantum_espresso/environment.yml](example_workflows/quantum_espresso/environment.yml) Conda environment
|
|
89
|
-
|
|
90
|
-
| Example | Explanation |
|
|
91
|
-
|----------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
|
|
92
|
-
| [example_workflows/quantum_espresso/aiida.ipynb](example_workflows/quantum_espresso/aiida.ipynb) | Define Workflow with aiida and execute it with jobflow and pyiron_base. |
|
|
93
|
-
| [example_workflows/quantum_espresso/jobflow.ipynb](example_workflows/quantum_espresso/jobflow.ipynb) | Define Workflow with jobflow and execute it with aiida and pyiron_base. |
|
|
94
|
-
| [example_workflows/quantum_espresso/pyiron_base.ipynb](example_workflows/quantum_espresso/pyiron_base.ipynb) | Define Workflow with pyiron_base and execute it with aiida and jobflow. |
|
|
95
|
-
| [example_workflows/quantum_espresso/universal_workflow.ipynb](example_workflows/quantum_espresso/universal_workflow.ipynb) | Execute workflow defined in the Python Workflow Definition with aiida, executorlib, jobflow, pyiron_base and pure Python. |
|
|
96
|
-
|
|
97
|
-
### NFDI4Ing Scientific Workflow Requirements
|
|
98
|
-
To demonstrate the compatibility of the Python Workflow Definition to file based workflows, the workflow benchmark developed as part of [NFDI4Ing](https://github.com/BAMresearch/NFDI4IngScientificWorkflowRequirements)
|
|
99
|
-
is implemented for all three simulation codes based on a shared workflow definition.
|
|
100
|
-
* [example_workflows/nfdi/workflow.py](example_workflows/nfdi/workflow.py) Python functions
|
|
101
|
-
* [example_workflows/nfdi/workflow.json](example_workflows/nfdi/workflow.json) Workflow definition in the Python Workflow Definition.
|
|
102
|
-
|
|
103
|
-
Additional source files provided with the workflow benchmark:
|
|
104
|
-
* [example_workflows/nfdi/source/envs/preprocessing.yaml](example_workflows/nfdi/source/envs/preprocessing.yaml) Conda environment for preprocessing
|
|
105
|
-
* [example_workflows/nfdi/source/envs/processing.yaml](example_workflows/nfdi/source/envs/processing.yaml) Conda environment for processing
|
|
106
|
-
* [example_workflows/nfdi/source/envs/postprocessing.yaml](example_workflows/nfdi/source/envs/postprocessing.yaml) Conda environment for postprocessing
|
|
107
|
-
* [example_workflows/nfdi/source/macros.tex.template](example_workflows/nfdi/source/macros.tex.template) LaTeX module template
|
|
108
|
-
* [example_workflows/nfdi/source/paper.tex](example_workflows/nfdi/source/paper.tex) LaTeX paper template
|
|
109
|
-
* [example_workflows/nfdi/source/poisson.py](example_workflows/nfdi/source/poisson.py) Poisson Python script
|
|
110
|
-
* [example_workflows/nfdi/source/postprocessing.py](example_workflows/nfdi/source/postprocessing.py) Postprocessing Python script
|
|
111
|
-
* [example_workflows/nfdi/source/prepare_paper_macros.py](example_workflows/nfdi/source/prepare_paper_macros.py) LaTeX preprocessing Python script
|
|
112
|
-
* [example_workflows/nfdi/source/unit_square.geo](example_workflows/nfdi/source/unit_square.geo) Input structure
|
|
113
|
-
|
|
114
|
-
| Example | Explanation |
|
|
115
|
-
|----------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
|
|
116
|
-
| [example_workflows/nfdi/aiida.ipynb](example_workflows/nfdi/aiida.ipynb) | Define Workflow with aiida and execute it with jobflow and pyiron_base. |
|
|
117
|
-
| [example_workflows/nfdi/jobflow.ipynb](example_workflows/nfdi/jobflow.ipynb) | Define Workflow with jobflow and execute it with aiida and pyiron_base. |
|
|
118
|
-
| [example_workflows/nfdi/pyiron_base.ipynb](example_workflows/nfdi/pyiron_base.ipynb) | Define Workflow with pyiron_base and execute it with aiida and jobflow. |
|
|
119
|
-
| [example_workflows/nfdi/universal_workflow.ipynb](example_workflows/nfdi/universal_workflow.ipynb) | Execute workflow defined in the Python Workflow Definition with aiida, executorlib, jobflow, pyiron_base and pure Python. |
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
channels:
|
|
2
|
-
- conda-forge
|
|
3
|
-
dependencies:
|
|
4
|
-
- python =3.12
|
|
5
|
-
- click =8.1.8
|
|
6
|
-
- pydantic =2.12.4
|
|
7
|
-
- hatchling =1.27.0
|
|
8
|
-
- httpcore =1.0.7
|
|
9
|
-
- jobflow =0.2.1
|
|
10
|
-
- pyiron_base =0.15.10
|
|
11
|
-
- pyiron_workflow =0.15.4
|
|
12
|
-
- pygraphviz =1.14
|
|
13
|
-
- aiida-workgraph =0.7.6
|
|
14
|
-
- plumpy =0.25.0
|
|
15
|
-
- conda_subprocess =0.0.10
|
|
16
|
-
- networkx =3.4.2
|
|
17
|
-
- cwltool =3.1.20251031082601
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
conda env create -n preprocessing -f example_workflows/nfdi/source/envs/preprocessing.yaml -y
|
|
2
|
-
conda env create -n processing -f example_workflows/nfdi/source/envs/processing.yaml -y
|
|
3
|
-
conda env create -n postprocessing -f example_workflows/nfdi/source/envs/postprocessing.yaml -y
|
|
4
|
-
conda env update --name base --file example_workflows/quantum_espresso/environment.yml
|
|
5
|
-
pip install -e example_workflows/quantum_espresso/qe_xml_parser
|
|
6
|
-
pip install -e python_workflow_definition
|
|
7
|
-
verdi presto --profile-name pwd
|
|
8
|
-
mv example_workflows/quantum_espresso/espresso ~
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
channels:
|
|
2
|
-
- conda-forge
|
|
3
|
-
dependencies:
|
|
4
|
-
- python =3.12
|
|
5
|
-
- click =8.1.8
|
|
6
|
-
- pydantic =2.12.4
|
|
7
|
-
- hatchling =1.27.0
|
|
8
|
-
- httpcore =1.0.7
|
|
9
|
-
- jobflow =0.2.1
|
|
10
|
-
- pyiron_base =0.15.10
|
|
11
|
-
- pyiron_workflow =0.15.4
|
|
12
|
-
- pygraphviz =1.14
|
|
13
|
-
- aiida-workgraph =0.7.6
|
|
14
|
-
- plumpy =0.25.0
|
|
15
|
-
- conda_subprocess =0.0.10
|
|
16
|
-
- networkx =3.4.2
|
|
17
|
-
- cwltool =3.1.20251031082601
|
|
18
|
-
- python=3.12
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
title: Python Workflow Definition
|
|
2
|
-
author: Jan Janssen
|
|
3
|
-
|
|
4
|
-
execute:
|
|
5
|
-
execute_notebooks : off
|
|
6
|
-
|
|
7
|
-
repository:
|
|
8
|
-
url : https://github.com/pythonworkflow/python-workflow-definition
|
|
9
|
-
path_to_book : ""
|
|
10
|
-
|
|
11
|
-
launch_buttons:
|
|
12
|
-
notebook_interface : jupyterlab
|
|
13
|
-
binderhub_url : https://mybinder.org
|