jobflow 0.1.9__tar.gz → 0.1.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jobflow-0.1.9 → jobflow-0.1.11}/PKG-INFO +14 -11
- {jobflow-0.1.9 → jobflow-0.1.11}/README.md +4 -2
- jobflow-0.1.11/pyproject.toml +140 -0
- jobflow-0.1.11/setup.cfg +4 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/core/flow.py +11 -7
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/core/job.py +52 -39
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/core/maker.py +8 -9
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/core/reference.py +11 -16
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/core/state.py +3 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/core/store.py +5 -8
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/managers/local.py +1 -1
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/utils/dict_mods.py +1 -1
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/utils/find.py +5 -5
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/utils/graph.py +1 -1
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow.egg-info/PKG-INFO +14 -11
- jobflow-0.1.11/src/jobflow.egg-info/SOURCES.txt +31 -0
- jobflow-0.1.11/src/jobflow.egg-info/requires.txt +40 -0
- jobflow-0.1.11/tests/test_version.py +34 -0
- jobflow-0.1.9/.coveragerc +0 -16
- jobflow-0.1.9/.github/ISSUE_TEMPLATE/bug_report.md +0 -24
- jobflow-0.1.9/.github/ISSUE_TEMPLATE/config.yml +0 -5
- jobflow-0.1.9/.github/PULL_REQUEST_TEMPLATE.md +0 -49
- jobflow-0.1.9/.github/dependabot.yml +0 -10
- jobflow-0.1.9/.github/workflows/deploy.yml +0 -101
- jobflow-0.1.9/.github/workflows/deps.yml +0 -19
- jobflow-0.1.9/.github/workflows/testing.yml +0 -97
- jobflow-0.1.9/.github/workflows/update-precommit.yml +0 -34
- jobflow-0.1.9/.gitignore +0 -137
- jobflow-0.1.9/.pre-commit-config.yaml +0 -58
- jobflow-0.1.9/CHANGELOG.md +0 -147
- jobflow-0.1.9/CODE_OF_CONDUCT.md +0 -48
- jobflow-0.1.9/CONTRIBUTING.md +0 -68
- jobflow-0.1.9/RELEASING_JOBFLOW.md +0 -7
- jobflow-0.1.9/codecov.yml +0 -6
- jobflow-0.1.9/docs/_static/custom.css +0 -70
- jobflow-0.1.9/docs/_static/fonts/lato-bold-italic.woff +0 -0
- jobflow-0.1.9/docs/_static/fonts/lato-bold-italic.woff2 +0 -0
- jobflow-0.1.9/docs/_static/fonts/lato-bold.woff +0 -0
- jobflow-0.1.9/docs/_static/fonts/lato-bold.woff2 +0 -0
- jobflow-0.1.9/docs/_static/fonts/lato-normal-italic.woff +0 -0
- jobflow-0.1.9/docs/_static/fonts/lato-normal-italic.woff2 +0 -0
- jobflow-0.1.9/docs/_static/fonts/lato-normal.woff +0 -0
- jobflow-0.1.9/docs/_static/fonts/lato-normal.woff2 +0 -0
- jobflow-0.1.9/docs/_static/img/simple_flow.png +0 -0
- jobflow-0.1.9/docs/_static/orcid.svg +0 -17
- jobflow-0.1.9/docs/_templates/breadcrumbs.html +0 -3
- jobflow-0.1.9/docs/changelog.rst +0 -1
- jobflow-0.1.9/docs/conf.py +0 -162
- jobflow-0.1.9/docs/contributing.rst +0 -1
- jobflow-0.1.9/docs/contributors.rst +0 -116
- jobflow-0.1.9/docs/genindex.rst +0 -2
- jobflow-0.1.9/docs/index.rst +0 -33
- jobflow-0.1.9/docs/install.rst +0 -64
- jobflow-0.1.9/docs/jobflow.core.rst +0 -50
- jobflow-0.1.9/docs/jobflow.managers.rst +0 -17
- jobflow-0.1.9/docs/jobflow.rst +0 -10
- jobflow-0.1.9/docs/jobflow.settings.rst +0 -7
- jobflow-0.1.9/docs/jobflow.utils.rst +0 -44
- jobflow-0.1.9/docs/license.rst +0 -53
- jobflow-0.1.9/docs/tutorials/1-quickstart.nblink +0 -3
- jobflow-0.1.9/docs/tutorials/2-introduction.nblink +0 -3
- jobflow-0.1.9/docs/tutorials/3-defining-jobs.nblink +0 -3
- jobflow-0.1.9/docs/tutorials/4-creating-flows.rst +0 -2
- jobflow-0.1.9/docs/tutorials/5-dynamic-flows.rst +0 -2
- jobflow-0.1.9/docs/tutorials/6-jobflow-database.rst +0 -2
- jobflow-0.1.9/docs/tutorials/7-fireworks.rst +0 -2
- jobflow-0.1.9/docs/tutorials.rst +0 -12
- jobflow-0.1.9/examples/data_store.py +0 -36
- jobflow-0.1.9/examples/encode_decode.py +0 -34
- jobflow-0.1.9/examples/fibonacci.py +0 -25
- jobflow-0.1.9/examples/linear.py +0 -20
- jobflow-0.1.9/examples/maker.py +0 -27
- jobflow-0.1.9/examples/nesting.py +0 -44
- jobflow-0.1.9/examples/restarts.py +0 -60
- jobflow-0.1.9/examples/schema.py +0 -23
- jobflow-0.1.9/examples/websites.txt +0 -3
- jobflow-0.1.9/requirements.txt +0 -6
- jobflow-0.1.9/setup.cfg +0 -19
- jobflow-0.1.9/setup.py +0 -74
- jobflow-0.1.9/src/jobflow.egg-info/SOURCES.txt +0 -118
- jobflow-0.1.9/src/jobflow.egg-info/not-zip-safe +0 -1
- jobflow-0.1.9/src/jobflow.egg-info/requires.txt +0 -35
- jobflow-0.1.9/tests/__init__.py +0 -0
- jobflow-0.1.9/tests/conftest.py +0 -113
- jobflow-0.1.9/tests/core/__init__.py +0 -0
- jobflow-0.1.9/tests/core/test_flow.py +0 -833
- jobflow-0.1.9/tests/core/test_job.py +0 -1260
- jobflow-0.1.9/tests/core/test_maker.py +0 -210
- jobflow-0.1.9/tests/core/test_reference.py +0 -478
- jobflow-0.1.9/tests/core/test_state.py +0 -6
- jobflow-0.1.9/tests/core/test_store.py +0 -415
- jobflow-0.1.9/tests/managers/__init__.py +0 -0
- jobflow-0.1.9/tests/managers/conftest.py +0 -327
- jobflow-0.1.9/tests/managers/test_fireworks.py +0 -573
- jobflow-0.1.9/tests/managers/test_local.py +0 -363
- jobflow-0.1.9/tests/test_data/db.yaml +0 -6
- jobflow-0.1.9/tests/test_data/db_bad.yaml +0 -7
- jobflow-0.1.9/tests/test_data/db_gridfs.yaml +0 -13
- jobflow-0.1.9/tests/test_data/db_serialized.json +0 -1
- jobflow-0.1.9/tests/utils/__init__.py +0 -0
- jobflow-0.1.9/tests/utils/test_dict_mods.py +0 -156
- jobflow-0.1.9/tests/utils/test_enum.py +0 -15
- jobflow-0.1.9/tests/utils/test_find.py +0 -76
- jobflow-0.1.9/tests/utils/test_graph.py +0 -100
- jobflow-0.1.9/tests/utils/test_log.py +0 -24
- jobflow-0.1.9/tests/utils/test_uuid.py +0 -6
- jobflow-0.1.9/tutorials/1-quickstart.ipynb +0 -337
- jobflow-0.1.9/tutorials/2-introduction.ipynb +0 -509
- jobflow-0.1.9/tutorials/3-defining-jobs.ipynb +0 -222
- {jobflow-0.1.9 → jobflow-0.1.11}/LICENSE +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/__init__.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/_version.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/core/__init__.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/managers/__init__.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/managers/fireworks.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/py.typed +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/settings.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/utils/__init__.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/utils/enum.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/utils/log.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow/utils/uuid.py +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow.egg-info/dependency_links.txt +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/src/jobflow.egg-info/top_level.txt +0 -0
- {jobflow-0.1.9 → jobflow-0.1.11}/tests/test_settings.py +0 -0
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: jobflow
|
|
3
|
-
Version: 0.1.
|
|
4
|
-
Summary: jobflow is library for writing computational workflows
|
|
5
|
-
|
|
6
|
-
Author: Alex Ganose
|
|
7
|
-
Author-email: alexganose@googlemail.com
|
|
3
|
+
Version: 0.1.11
|
|
4
|
+
Summary: jobflow is a library for writing computational workflows
|
|
5
|
+
Author-email: Alex Ganose <alexganose@gmail.com>
|
|
8
6
|
License: modified BSD
|
|
9
|
-
|
|
7
|
+
Project-URL: homepage, https://materialsproject.github.io/jobflow/
|
|
8
|
+
Project-URL: repository, https://github.com/materialsproject/jobflow
|
|
9
|
+
Project-URL: documentation, https://materialsproject.github.io/jobflow/
|
|
10
|
+
Project-URL: changelog, https://github.com/materialsproject/jobflow/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: high-throughput,workflow
|
|
10
12
|
Classifier: Programming Language :: Python :: 3
|
|
11
13
|
Classifier: Programming Language :: Python :: 3.8
|
|
12
14
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -22,19 +24,22 @@ Classifier: Topic :: Scientific/Engineering
|
|
|
22
24
|
Requires-Python: >=3.8
|
|
23
25
|
Description-Content-Type: text/markdown
|
|
24
26
|
Provides-Extra: docs
|
|
25
|
-
Provides-Extra: tests
|
|
26
27
|
Provides-Extra: dev
|
|
28
|
+
Provides-Extra: tests
|
|
27
29
|
Provides-Extra: vis
|
|
28
30
|
Provides-Extra: fireworks
|
|
31
|
+
Provides-Extra: strict
|
|
29
32
|
License-File: LICENSE
|
|
30
33
|
|
|
31
34
|
# jobflow
|
|
32
35
|
|
|
33
|
-
<a href="https://github.com/materialsproject/jobflow/actions?query=workflow%3Atesting"><img alt="code coverage" src="https://img.shields.io/github/workflow/status/materialsproject/jobflow/testing?label=tests"></a>
|
|
36
|
+
<a href="https://github.com/materialsproject/jobflow/actions?query=workflow%3Atesting"><img alt="code coverage" src="https://img.shields.io/github/actions/workflow/status/materialsproject/jobflow/testing.yml?branch=main&label=tests"></a>
|
|
34
37
|
<a href="https://codecov.io/gh/materialsproject/jobflow/"><img alt="code coverage" src="https://img.shields.io/codecov/c/gh/materialsproject/jobflow/main"></a>
|
|
35
38
|
<a href="https://pypi.org/project/jobflow"><img alt="pypi version" src="https://img.shields.io/pypi/v/jobflow?color=blue"></a>
|
|
36
39
|
<img alt="supported python versions" src="https://img.shields.io/pypi/pyversions/jobflow">
|
|
37
40
|
|
|
41
|
+
[Documentation](https://materialsproject.github.io/jobflow/) | [PyPI](https://pypi.org/project/jobflow/) | [GitHub](https://github.com/materialsproject/jobflow)
|
|
42
|
+
|
|
38
43
|
Jobflow is a free, open-source library for writing and executing workflows. Complex
|
|
39
44
|
workflows can be defined using simple python functions and executed locally or on
|
|
40
45
|
arbitrary computing resources using the [FireWorks][fireworks] workflow manager.
|
|
@@ -104,7 +109,7 @@ the jobs is determined automatically and can be visualised using the flow graph.
|
|
|
104
109
|
|
|
105
110
|
## Installation
|
|
106
111
|
|
|
107
|
-
The jobflow is a Python 3.
|
|
112
|
+
The jobflow is a Python 3.8+ library and can be installed using pip.
|
|
108
113
|
|
|
109
114
|
```bash
|
|
110
115
|
pip install jobflow
|
|
@@ -157,5 +162,3 @@ Jobflow was designed by Alex Ganose, Anubhav Jain, Gian-Marco Rignanese, David W
|
|
|
157
162
|
[dynamic-flows]: https://materialsproject.github.io/jobflow/tutorials/5-dynamic-flows.html
|
|
158
163
|
[jobflow-database]: https://materialsproject.github.io/jobflow/tutorials/6-jobflow-database.html
|
|
159
164
|
[jobflow-fireworks]: https://materialsproject.github.io/jobflow/tutorials/7-fireworks.html
|
|
160
|
-
|
|
161
|
-
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
# jobflow
|
|
2
2
|
|
|
3
|
-
<a href="https://github.com/materialsproject/jobflow/actions?query=workflow%3Atesting"><img alt="code coverage" src="https://img.shields.io/github/workflow/status/materialsproject/jobflow/testing?label=tests"></a>
|
|
3
|
+
<a href="https://github.com/materialsproject/jobflow/actions?query=workflow%3Atesting"><img alt="code coverage" src="https://img.shields.io/github/actions/workflow/status/materialsproject/jobflow/testing.yml?branch=main&label=tests"></a>
|
|
4
4
|
<a href="https://codecov.io/gh/materialsproject/jobflow/"><img alt="code coverage" src="https://img.shields.io/codecov/c/gh/materialsproject/jobflow/main"></a>
|
|
5
5
|
<a href="https://pypi.org/project/jobflow"><img alt="pypi version" src="https://img.shields.io/pypi/v/jobflow?color=blue"></a>
|
|
6
6
|
<img alt="supported python versions" src="https://img.shields.io/pypi/pyversions/jobflow">
|
|
7
7
|
|
|
8
|
+
[Documentation](https://materialsproject.github.io/jobflow/) | [PyPI](https://pypi.org/project/jobflow/) | [GitHub](https://github.com/materialsproject/jobflow)
|
|
9
|
+
|
|
8
10
|
Jobflow is a free, open-source library for writing and executing workflows. Complex
|
|
9
11
|
workflows can be defined using simple python functions and executed locally or on
|
|
10
12
|
arbitrary computing resources using the [FireWorks][fireworks] workflow manager.
|
|
@@ -74,7 +76,7 @@ the jobs is determined automatically and can be visualised using the flow graph.
|
|
|
74
76
|
|
|
75
77
|
## Installation
|
|
76
78
|
|
|
77
|
-
The jobflow is a Python 3.
|
|
79
|
+
The jobflow is a Python 3.8+ library and can be installed using pip.
|
|
78
80
|
|
|
79
81
|
```bash
|
|
80
82
|
pip install jobflow
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools >= 42", "versioningit ~= 1.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "jobflow"
|
|
7
|
+
description = "jobflow is a library for writing computational workflows"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
keywords = ["high-throughput", "workflow"]
|
|
10
|
+
license = { text = "modified BSD" }
|
|
11
|
+
authors = [{ name = "Alex Ganose", email = "alexganose@gmail.com" }]
|
|
12
|
+
dynamic = ["version"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Programming Language :: Python :: 3.8",
|
|
16
|
+
"Programming Language :: Python :: 3.9",
|
|
17
|
+
"Programming Language :: Python :: 3.10",
|
|
18
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
19
|
+
"Intended Audience :: Science/Research",
|
|
20
|
+
"Intended Audience :: System Administrators",
|
|
21
|
+
"Intended Audience :: Information Technology",
|
|
22
|
+
"Operating System :: OS Independent",
|
|
23
|
+
"Topic :: Other/Nonlisted Topic",
|
|
24
|
+
"Topic :: Database :: Front-Ends",
|
|
25
|
+
"Topic :: Scientific/Engineering",
|
|
26
|
+
]
|
|
27
|
+
requires-python = ">=3.8"
|
|
28
|
+
dependencies = [
|
|
29
|
+
"monty>=2021.5.9",
|
|
30
|
+
"pydash",
|
|
31
|
+
"networkx",
|
|
32
|
+
"maggma>=0.38.1",
|
|
33
|
+
"pydantic",
|
|
34
|
+
"PyYAML",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
docs = [
|
|
39
|
+
"sphinx==6.1.3",
|
|
40
|
+
"furo==2022.12.7",
|
|
41
|
+
"myst_parser==1.0.0",
|
|
42
|
+
"ipython==8.11.0",
|
|
43
|
+
"nbsphinx==0.9.0",
|
|
44
|
+
"autodoc_pydantic==1.8.0",
|
|
45
|
+
]
|
|
46
|
+
dev = ["pre-commit>=2.12.1"]
|
|
47
|
+
tests = ["pytest==7.2.2", "pytest-cov==4.0.0"]
|
|
48
|
+
vis = ["matplotlib", "pydot"]
|
|
49
|
+
fireworks = ["FireWorks"]
|
|
50
|
+
strict = [
|
|
51
|
+
"monty==2022.9.9",
|
|
52
|
+
"networkx==3.0",
|
|
53
|
+
"pydash==6.0.2",
|
|
54
|
+
"maggma==0.50.3",
|
|
55
|
+
"pydantic==1.10.6",
|
|
56
|
+
"PyYAML==6.0",
|
|
57
|
+
"FireWorks==2.0.3",
|
|
58
|
+
"matplotlib==3.7.1",
|
|
59
|
+
"pydot==1.4.2",
|
|
60
|
+
"moto==4.1.4",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
[project.urls]
|
|
64
|
+
homepage = "https://materialsproject.github.io/jobflow/"
|
|
65
|
+
repository = "https://github.com/materialsproject/jobflow"
|
|
66
|
+
documentation = "https://materialsproject.github.io/jobflow/"
|
|
67
|
+
changelog = "https://github.com/materialsproject/jobflow/blob/main/CHANGELOG.md"
|
|
68
|
+
|
|
69
|
+
[tool.setuptools.package-data]
|
|
70
|
+
jobflow = ["py.typed"]
|
|
71
|
+
|
|
72
|
+
[tool.versioningit.vcs]
|
|
73
|
+
method = "git"
|
|
74
|
+
default-tag = "0.0.1"
|
|
75
|
+
|
|
76
|
+
[tool.flake8]
|
|
77
|
+
max-line-length = 88
|
|
78
|
+
max-doc-length = 88
|
|
79
|
+
select = "C, E, F, W, B"
|
|
80
|
+
extend-ignore = "E203, W503, E501, F401, RST21"
|
|
81
|
+
min-python-version = "3.8.0"
|
|
82
|
+
docstring-convention = "numpy"
|
|
83
|
+
rst-roles = "class, func, ref, obj"
|
|
84
|
+
|
|
85
|
+
[tool.mypy]
|
|
86
|
+
ignore_missing_imports = true
|
|
87
|
+
no_strict_optional = true
|
|
88
|
+
|
|
89
|
+
[tool.pytest.ini_options]
|
|
90
|
+
filterwarnings = [
|
|
91
|
+
"ignore:.*POTCAR.*:UserWarning",
|
|
92
|
+
"ignore:.*magmom.*:UserWarning",
|
|
93
|
+
"ignore:.*is not gzipped.*:UserWarning",
|
|
94
|
+
"ignore:.*input structure.*:UserWarning",
|
|
95
|
+
"ignore::DeprecationWarning",
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
[tool.coverage.run]
|
|
99
|
+
include = ["src/*"]
|
|
100
|
+
parallel = true
|
|
101
|
+
branch = true
|
|
102
|
+
|
|
103
|
+
[tool.coverage.paths]
|
|
104
|
+
source = ["src/"]
|
|
105
|
+
|
|
106
|
+
[tool.coverage.report]
|
|
107
|
+
skip_covered = true
|
|
108
|
+
show_missing = true
|
|
109
|
+
exclude_lines = [
|
|
110
|
+
'^\s*assert False(,|$)',
|
|
111
|
+
'if typing.TYPE_CHECKING:',
|
|
112
|
+
'^\s*@overload( |$)',
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
[tool.ruff]
|
|
116
|
+
target-version = "py38"
|
|
117
|
+
ignore-init-module-imports = true
|
|
118
|
+
select = [
|
|
119
|
+
"B", # flake8-bugbear
|
|
120
|
+
"C4", # flake8-comprehensions
|
|
121
|
+
"D", # pydocstyle
|
|
122
|
+
"E", # pycodestyle
|
|
123
|
+
"F", # pyflakes
|
|
124
|
+
"I", # isort
|
|
125
|
+
"PLE", # pylint error
|
|
126
|
+
"PLW", # pylint warning
|
|
127
|
+
"Q", # flake8-quotes
|
|
128
|
+
"RUF", # Ruff-specific rules
|
|
129
|
+
"SIM", # flake8-simplify
|
|
130
|
+
"TID", # tidy imports
|
|
131
|
+
"UP", # pyupgrade
|
|
132
|
+
"W", # pycodestyle
|
|
133
|
+
"YTT", # flake8-2020
|
|
134
|
+
]
|
|
135
|
+
pydocstyle.convention = "numpy"
|
|
136
|
+
isort.known-first-party = ["jobflow"]
|
|
137
|
+
|
|
138
|
+
[tool.ruff.per-file-ignores]
|
|
139
|
+
"__init__.py" = ["F401"]
|
|
140
|
+
"**/tests/*" = ["D"]
|
jobflow-0.1.11/setup.cfg
ADDED
|
@@ -140,7 +140,7 @@ class Flow(MSONable):
|
|
|
140
140
|
self.uuid = uuid
|
|
141
141
|
self.hosts = hosts or []
|
|
142
142
|
|
|
143
|
-
self._jobs: tuple[Flow | Job, ...] =
|
|
143
|
+
self._jobs: tuple[Flow | Job, ...] = ()
|
|
144
144
|
self.add_jobs(jobs)
|
|
145
145
|
self.output = output
|
|
146
146
|
|
|
@@ -173,7 +173,8 @@ class Flow(MSONable):
|
|
|
173
173
|
"""
|
|
174
174
|
Set the output of the Flow.
|
|
175
175
|
|
|
176
|
-
The output should be compatible with the list of Jobs/Flows contained in the
|
|
176
|
+
The output should be compatible with the list of Jobs/Flows contained in the
|
|
177
|
+
Flow.
|
|
177
178
|
|
|
178
179
|
Parameters
|
|
179
180
|
----------
|
|
@@ -406,8 +407,9 @@ class Flow(MSONable):
|
|
|
406
407
|
update
|
|
407
408
|
The updates to apply.
|
|
408
409
|
name_filter
|
|
409
|
-
A filter for the Maker name. Only Makers with a matching name will be
|
|
410
|
-
Includes partial matches, e.g. "ad" will match a Maker with the
|
|
410
|
+
A filter for the Maker name. Only Makers with a matching name will be
|
|
411
|
+
updated. Includes partial matches, e.g. "ad" will match a Maker with the
|
|
412
|
+
name "adder".
|
|
411
413
|
class_filter
|
|
412
414
|
A filter for the maker class. Only Makers with a matching class will be
|
|
413
415
|
updated. Note the class filter will match any subclasses.
|
|
@@ -533,7 +535,8 @@ class Flow(MSONable):
|
|
|
533
535
|
Use the dict mod language to apply updates. See :obj:`.DictMods` for more
|
|
534
536
|
details.
|
|
535
537
|
dynamic
|
|
536
|
-
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
538
|
+
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
539
|
+
runtime.
|
|
537
540
|
|
|
538
541
|
Examples
|
|
539
542
|
--------
|
|
@@ -587,7 +590,8 @@ class Flow(MSONable):
|
|
|
587
590
|
Which attributes of the job config to set. Can be specified as one or more
|
|
588
591
|
attributes specified by their name.
|
|
589
592
|
dynamic
|
|
590
|
-
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
593
|
+
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
594
|
+
runtime.
|
|
591
595
|
|
|
592
596
|
Examples
|
|
593
597
|
--------
|
|
@@ -678,7 +682,7 @@ class Flow(MSONable):
|
|
|
678
682
|
jobs = [jobs]
|
|
679
683
|
|
|
680
684
|
job_ids = set(self.all_uuids)
|
|
681
|
-
hosts = [self.uuid
|
|
685
|
+
hosts = [self.uuid, *self.hosts]
|
|
682
686
|
for job in jobs:
|
|
683
687
|
if job.host is not None and job.host != self.uuid:
|
|
684
688
|
raise ValueError(
|
|
@@ -49,8 +49,8 @@ class JobConfig(MSONable):
|
|
|
49
49
|
response_manager_config
|
|
50
50
|
The custom configuration to pass to a detour, addition, or replacement job.
|
|
51
51
|
Using this kwarg will automatically take precedence over the behavior of
|
|
52
|
-
``pass_manager_config`` such that a different configuration than
|
|
53
|
-
can be passed to downstream jobs.
|
|
52
|
+
``pass_manager_config`` such that a different configuration than
|
|
53
|
+
``manger_config`` can be passed to downstream jobs.
|
|
54
54
|
|
|
55
55
|
Returns
|
|
56
56
|
-------
|
|
@@ -116,9 +116,9 @@ def job(method: Callable | None = None, **job_kwargs):
|
|
|
116
116
|
OutputReference('abeb6f48-9b34-4698-ab69-e4dc2127ebe9')
|
|
117
117
|
|
|
118
118
|
.. Note::
|
|
119
|
-
Because the task has not yet been run, the output value is
|
|
120
|
-
object. References are automatically converted to their
|
|
121
|
-
(resolved) when the task runs.
|
|
119
|
+
Because the task has not yet been run, the output value is an
|
|
120
|
+
:obj:`OutputReference` object. References are automatically converted to their
|
|
121
|
+
computed values (resolved) when the task runs.
|
|
122
122
|
|
|
123
123
|
If a dictionary of values is returned, the values can be indexed in the usual
|
|
124
124
|
way.
|
|
@@ -251,10 +251,11 @@ class Job(MSONable):
|
|
|
251
251
|
The config setting for the job.
|
|
252
252
|
hosts
|
|
253
253
|
The list of UUIDs of the hosts containing the job. The object identified by one
|
|
254
|
-
UUID of the list should be contained in objects identified by its subsequent
|
|
254
|
+
UUID of the list should be contained in objects identified by its subsequent
|
|
255
|
+
elements.
|
|
255
256
|
metadata_updates
|
|
256
|
-
A list of updates for the metadata that will be applied to any Flow/Job
|
|
257
|
-
by the job.
|
|
257
|
+
A list of updates for the metadata that will be applied to any Flow/Job
|
|
258
|
+
generated by the job.
|
|
258
259
|
config_updates
|
|
259
260
|
A list of updates for the config that will be applied to any Flow/Job generated
|
|
260
261
|
by the job.
|
|
@@ -323,7 +324,7 @@ class Job(MSONable):
|
|
|
323
324
|
|
|
324
325
|
from jobflow.utils.find import contains_flow_or_job
|
|
325
326
|
|
|
326
|
-
function_args =
|
|
327
|
+
function_args = () if function_args is None else function_args
|
|
327
328
|
function_kwargs = {} if function_kwargs is None else function_kwargs
|
|
328
329
|
uuid = suuid() if uuid is None else uuid
|
|
329
330
|
metadata = {} if metadata is None else metadata
|
|
@@ -584,12 +585,14 @@ class Job(MSONable):
|
|
|
584
585
|
pass_manager_config(response.replace, passed_config)
|
|
585
586
|
|
|
586
587
|
try:
|
|
587
|
-
output = jsanitize(
|
|
588
|
-
|
|
588
|
+
output = jsanitize(
|
|
589
|
+
response.output, strict=True, enum_values=True, allow_bson=True
|
|
590
|
+
)
|
|
591
|
+
except AttributeError as err:
|
|
589
592
|
raise RuntimeError(
|
|
590
593
|
"Job output contained an object that is not MSONable and therefore "
|
|
591
594
|
"could not be serialized."
|
|
592
|
-
)
|
|
595
|
+
) from err
|
|
593
596
|
|
|
594
597
|
save = {k: "output" if v is True else v for k, v in self._kwargs.items()}
|
|
595
598
|
data = {
|
|
@@ -730,8 +733,9 @@ class Job(MSONable):
|
|
|
730
733
|
update
|
|
731
734
|
The updates to apply.
|
|
732
735
|
name_filter
|
|
733
|
-
A filter for the Maker name. Only Makers with a matching name will be
|
|
734
|
-
Includes partial matches, e.g. "ad" will match a Maker with the
|
|
736
|
+
A filter for the Maker name. Only Makers with a matching name will be
|
|
737
|
+
updated. Includes partial matches, e.g. "ad" will match a Maker with the
|
|
738
|
+
name "adder".
|
|
735
739
|
class_filter
|
|
736
740
|
A filter for the maker class. Only Makers with a matching class will be
|
|
737
741
|
updated. Note the class filter will match any subclasses.
|
|
@@ -802,7 +806,7 @@ class Job(MSONable):
|
|
|
802
806
|
nested=nested,
|
|
803
807
|
dict_mod=dict_mod,
|
|
804
808
|
)
|
|
805
|
-
|
|
809
|
+
self.function = getattr(maker, self.function.__name__)
|
|
806
810
|
elif nested:
|
|
807
811
|
# also look for makers in job args and kwargs
|
|
808
812
|
new_args = []
|
|
@@ -874,12 +878,13 @@ class Job(MSONable):
|
|
|
874
878
|
Use the dict mod language to apply updates. See :obj:`.DictMods` for more
|
|
875
879
|
details.
|
|
876
880
|
dynamic
|
|
877
|
-
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
881
|
+
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
882
|
+
runtime.
|
|
878
883
|
|
|
879
884
|
Examples
|
|
880
885
|
--------
|
|
881
|
-
Consider a simple job that makes use of a :obj:`Maker` to generate additional
|
|
882
|
-
at runtime (see :obj:`Response` options for more details):
|
|
886
|
+
Consider a simple job that makes use of a :obj:`Maker` to generate additional
|
|
887
|
+
jobs at runtime (see :obj:`Response` options for more details):
|
|
883
888
|
|
|
884
889
|
>>> @job
|
|
885
890
|
... def use_maker(maker):
|
|
@@ -902,12 +907,12 @@ class Job(MSONable):
|
|
|
902
907
|
from jobflow.utils.dict_mods import apply_mod
|
|
903
908
|
|
|
904
909
|
if dynamic:
|
|
905
|
-
dict_input =
|
|
906
|
-
update
|
|
907
|
-
name_filter
|
|
908
|
-
function_filter
|
|
909
|
-
dict_mod
|
|
910
|
-
|
|
910
|
+
dict_input = {
|
|
911
|
+
"update": update,
|
|
912
|
+
"name_filter": name_filter,
|
|
913
|
+
"function_filter": function_filter,
|
|
914
|
+
"dict_mod": dict_mod,
|
|
915
|
+
}
|
|
911
916
|
self.metadata_updates.append(dict_input)
|
|
912
917
|
|
|
913
918
|
# unwrap the functions in case the job is a decorated one
|
|
@@ -957,7 +962,8 @@ class Job(MSONable):
|
|
|
957
962
|
Which attributes of the job config to set. Can be specified as one or more
|
|
958
963
|
attributes specified by their name.
|
|
959
964
|
dynamic
|
|
960
|
-
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
965
|
+
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
966
|
+
runtime.
|
|
961
967
|
|
|
962
968
|
Examples
|
|
963
969
|
--------
|
|
@@ -976,9 +982,9 @@ class Job(MSONable):
|
|
|
976
982
|
... )
|
|
977
983
|
>>> add_job.update_config(new_config)
|
|
978
984
|
|
|
979
|
-
To only update specific attributes, the ``attributes`` argument can be
|
|
980
|
-
For example, the following will only update the "manager_config"
|
|
981
|
-
job config.
|
|
985
|
+
To only update specific attributes, the ``attributes`` argument can be
|
|
986
|
+
specified. For example, the following will only update the "manager_config"
|
|
987
|
+
attribute of the job config.
|
|
982
988
|
|
|
983
989
|
>>> add_job.update_config(new_config, attributes="manager_config")
|
|
984
990
|
|
|
@@ -1010,12 +1016,12 @@ class Job(MSONable):
|
|
|
1010
1016
|
only be set for the `test_job` and not for the generated Jobs.
|
|
1011
1017
|
"""
|
|
1012
1018
|
if dynamic:
|
|
1013
|
-
dict_input =
|
|
1014
|
-
config
|
|
1015
|
-
name_filter
|
|
1016
|
-
function_filter
|
|
1017
|
-
attributes
|
|
1018
|
-
|
|
1019
|
+
dict_input = {
|
|
1020
|
+
"config": config,
|
|
1021
|
+
"name_filter": name_filter,
|
|
1022
|
+
"function_filter": function_filter,
|
|
1023
|
+
"attributes": attributes,
|
|
1024
|
+
}
|
|
1019
1025
|
self.config_updates.append(dict_input)
|
|
1020
1026
|
|
|
1021
1027
|
# unwrap the functions in case the job is a decorated one
|
|
@@ -1033,15 +1039,15 @@ class Job(MSONable):
|
|
|
1033
1039
|
|
|
1034
1040
|
# if we get to here then we pass all the filters
|
|
1035
1041
|
if isinstance(config, dict):
|
|
1036
|
-
# convert dict specification to a JobConfig but set the attributes
|
|
1042
|
+
# convert dict specification to a JobConfig but set the attributes
|
|
1037
1043
|
if attributes is None:
|
|
1038
1044
|
attributes = list(config.keys())
|
|
1039
1045
|
|
|
1040
1046
|
attributes = [attributes] if isinstance(attributes, str) else attributes
|
|
1041
1047
|
if not set(attributes).issubset(set(config.keys())):
|
|
1042
1048
|
raise ValueError(
|
|
1043
|
-
"Specified attributes include a key that is not present in the
|
|
1044
|
-
" dictionary."
|
|
1049
|
+
"Specified attributes include a key that is not present in the "
|
|
1050
|
+
"config dictionary."
|
|
1045
1051
|
)
|
|
1046
1052
|
config = JobConfig(**config)
|
|
1047
1053
|
|
|
@@ -1098,8 +1104,15 @@ class Job(MSONable):
|
|
|
1098
1104
|
self.hosts.extend(hosts_uuids)
|
|
1099
1105
|
|
|
1100
1106
|
|
|
1107
|
+
# For type checking, the Response output type can be specified
|
|
1108
|
+
# in a type hint via this type variable.
|
|
1109
|
+
# For example, a signature `-> Response[int]` would require
|
|
1110
|
+
# that the Response.output is an int.
|
|
1111
|
+
T = typing.TypeVar("T")
|
|
1112
|
+
|
|
1113
|
+
|
|
1101
1114
|
@dataclass
|
|
1102
|
-
class Response:
|
|
1115
|
+
class Response(typing.Generic[T]):
|
|
1103
1116
|
"""
|
|
1104
1117
|
The :obj:`Response` contains the output, detours, and stop commands of a job.
|
|
1105
1118
|
|
|
@@ -1121,7 +1134,7 @@ class Response:
|
|
|
1121
1134
|
Stop executing all remaining jobs.
|
|
1122
1135
|
"""
|
|
1123
1136
|
|
|
1124
|
-
output:
|
|
1137
|
+
output: T | None = None
|
|
1125
1138
|
detour: jobflow.Flow | Job | list[Job] | list[jobflow.Flow] | None = None
|
|
1126
1139
|
addition: jobflow.Flow | Job | list[Job] | list[jobflow.Flow] | None = None
|
|
1127
1140
|
replace: jobflow.Flow | Job | list[Job] | list[jobflow.Flow] | None = None
|
|
@@ -149,8 +149,9 @@ class Maker(MSONable):
|
|
|
149
149
|
update
|
|
150
150
|
The updates to apply.
|
|
151
151
|
name_filter
|
|
152
|
-
A filter for the Maker name. Only Makers with a matching name will be
|
|
153
|
-
Includes partial matches, e.g. "ad" will match a Maker with the
|
|
152
|
+
A filter for the Maker name. Only Makers with a matching name will be
|
|
153
|
+
updated. Includes partial matches, e.g. "ad" will match a Maker with the
|
|
154
|
+
name "adder".
|
|
154
155
|
class_filter
|
|
155
156
|
A filter for the maker class. Only Makers with a matching class will be
|
|
156
157
|
updated. Note the class filter will match any subclasses.
|
|
@@ -232,7 +233,7 @@ class Maker(MSONable):
|
|
|
232
233
|
|
|
233
234
|
def recursive_call(
|
|
234
235
|
obj: Maker,
|
|
235
|
-
func: Callable[[Maker],
|
|
236
|
+
func: Callable[[Maker], Maker],
|
|
236
237
|
name_filter: str | None = None,
|
|
237
238
|
class_filter: type[Maker] | None = None,
|
|
238
239
|
nested: bool = True,
|
|
@@ -281,13 +282,11 @@ def recursive_call(
|
|
|
281
282
|
# 2. Regenerate the classes and check if they are a Maker
|
|
282
283
|
# 3. Call the functions on the deepest classes first
|
|
283
284
|
# 4. Call the function or update on the deepest classes and move up the tree
|
|
284
|
-
# 5. Finally replace the call/update the object itself
|
|
285
|
+
# 5. Finally, replace the call/update the object itself
|
|
285
286
|
|
|
286
287
|
# find all classes in the serialized maker kwargs
|
|
287
|
-
if nested
|
|
288
|
-
|
|
289
|
-
else:
|
|
290
|
-
locations = [[]] # will only look at the top level if nested=False
|
|
288
|
+
# will only look at the top level if nested=False
|
|
289
|
+
locations = find_key(d, "@class", nested=True) if nested else [[]]
|
|
291
290
|
|
|
292
291
|
for location in sorted(
|
|
293
292
|
locations, key=len, reverse=True
|
|
@@ -298,7 +297,7 @@ def recursive_call(
|
|
|
298
297
|
if _filter(nested_class):
|
|
299
298
|
# either update or call the function on the nested Maker
|
|
300
299
|
modified_class = func(nested_class)
|
|
301
|
-
if not isinstance(modified_class, Maker):
|
|
300
|
+
if not isinstance(modified_class, Maker):
|
|
302
301
|
raise ValueError(
|
|
303
302
|
"Function must return a Maker object. "
|
|
304
303
|
f"Got {type(modified_class)} instead."
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import contextlib
|
|
5
6
|
import typing
|
|
6
7
|
from typing import Any, Sequence
|
|
7
8
|
|
|
@@ -93,7 +94,7 @@ class OutputReference(MSONable):
|
|
|
93
94
|
def __init__(
|
|
94
95
|
self,
|
|
95
96
|
uuid: str,
|
|
96
|
-
attributes: tuple[tuple[str, Any], ...] =
|
|
97
|
+
attributes: tuple[tuple[str, Any], ...] = (),
|
|
97
98
|
output_schema: type[BaseModel] | None = None,
|
|
98
99
|
):
|
|
99
100
|
super().__init__()
|
|
@@ -157,12 +158,10 @@ class OutputReference(MSONable):
|
|
|
157
158
|
index = None if result is None else result["index"]
|
|
158
159
|
|
|
159
160
|
if index is not None and index not in cache[self.uuid]:
|
|
160
|
-
|
|
161
|
+
with contextlib.suppress(ValueError):
|
|
161
162
|
cache[self.uuid][index] = store.get_output(
|
|
162
163
|
self.uuid, which="last", load=True, on_missing=on_missing
|
|
163
164
|
)
|
|
164
|
-
except ValueError:
|
|
165
|
-
pass
|
|
166
165
|
|
|
167
166
|
if on_missing == OnMissing.ERROR and index not in cache[self.uuid]:
|
|
168
167
|
istr = f" ({index})" if index is not None else ""
|
|
@@ -183,12 +182,8 @@ class OutputReference(MSONable):
|
|
|
183
182
|
cache[self.uuid][index] = data
|
|
184
183
|
|
|
185
184
|
for attr_type, attr in self.attributes:
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
data = data[attr]
|
|
189
|
-
else:
|
|
190
|
-
# attribute access
|
|
191
|
-
data = getattr(data, attr)
|
|
185
|
+
# i means index else use attribute access
|
|
186
|
+
data = data[attr] if attr_type == "i" else getattr(data, attr)
|
|
192
187
|
|
|
193
188
|
return data
|
|
194
189
|
|
|
@@ -226,7 +221,7 @@ class OutputReference(MSONable):
|
|
|
226
221
|
_, subschema = validate_schema_access(self.output_schema, item)
|
|
227
222
|
|
|
228
223
|
return OutputReference(
|
|
229
|
-
self.uuid, self.attributes
|
|
224
|
+
self.uuid, (*self.attributes, ("i", item)), output_schema=subschema
|
|
230
225
|
)
|
|
231
226
|
|
|
232
227
|
def __getattr__(self, item) -> OutputReference:
|
|
@@ -242,7 +237,7 @@ class OutputReference(MSONable):
|
|
|
242
237
|
_, subschema = validate_schema_access(self.output_schema, item)
|
|
243
238
|
|
|
244
239
|
return OutputReference(
|
|
245
|
-
self.uuid, self.attributes
|
|
240
|
+
self.uuid, (*self.attributes, ("a", item)), output_schema=subschema
|
|
246
241
|
)
|
|
247
242
|
|
|
248
243
|
def __setattr__(self, attr, val):
|
|
@@ -390,13 +385,13 @@ def find_and_get_references(arg: Any) -> tuple[OutputReference, ...]:
|
|
|
390
385
|
|
|
391
386
|
if isinstance(arg, OutputReference):
|
|
392
387
|
# if the argument is a reference then stop there
|
|
393
|
-
return
|
|
388
|
+
return (arg,)
|
|
394
389
|
|
|
395
390
|
elif isinstance(arg, (float, int, str, bool)):
|
|
396
391
|
# argument is a primitive, we won't find a reference here
|
|
397
|
-
return
|
|
392
|
+
return ()
|
|
398
393
|
|
|
399
|
-
arg = jsanitize(arg, strict=True, enum_values=True)
|
|
394
|
+
arg = jsanitize(arg, strict=True, enum_values=True, allow_bson=True)
|
|
400
395
|
|
|
401
396
|
# recursively find any reference classes
|
|
402
397
|
locations = find_key_value(arg, "@class", "OutputReference")
|
|
@@ -453,7 +448,7 @@ def find_and_resolve_references(
|
|
|
453
448
|
return arg
|
|
454
449
|
|
|
455
450
|
# serialize the argument to a dictionary
|
|
456
|
-
encoded_arg = jsanitize(arg, strict=True, enum_values=True)
|
|
451
|
+
encoded_arg = jsanitize(arg, strict=True, enum_values=True, allow_bson=True)
|
|
457
452
|
|
|
458
453
|
# recursively find any reference classes
|
|
459
454
|
locations = find_key_value(encoded_arg, "@class", "OutputReference")
|
|
@@ -24,10 +24,13 @@ __all__ = ["CURRENT_JOB"]
|
|
|
24
24
|
|
|
25
25
|
@singleton
|
|
26
26
|
class State:
|
|
27
|
+
"""State of the current job and store."""
|
|
28
|
+
|
|
27
29
|
job: jobflow.Job | None = None
|
|
28
30
|
store: jobflow.JobStore | None = None
|
|
29
31
|
|
|
30
32
|
def reset(self):
|
|
33
|
+
"""Reset the current state."""
|
|
31
34
|
self.job = None
|
|
32
35
|
self.store = None
|
|
33
36
|
|