jobflow 0.1.14__tar.gz → 0.1.15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jobflow-0.1.14/src/jobflow.egg-info → jobflow-0.1.15}/PKG-INFO +19 -20
- {jobflow-0.1.14 → jobflow-0.1.15}/README.md +5 -5
- {jobflow-0.1.14 → jobflow-0.1.15}/pyproject.toml +57 -32
- jobflow-0.1.15/src/jobflow/_version.py +7 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/core/flow.py +23 -22
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/core/job.py +12 -12
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/core/maker.py +0 -2
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/core/reference.py +22 -25
- jobflow-0.1.15/src/jobflow/core/schemas.py +34 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/core/state.py +0 -4
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/core/store.py +32 -28
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/managers/fireworks.py +3 -6
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/managers/local.py +18 -16
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/settings.py +0 -2
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/utils/dict_mods.py +0 -3
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/utils/enum.py +1 -4
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/utils/find.py +5 -13
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/utils/graph.py +11 -15
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/utils/log.py +0 -2
- {jobflow-0.1.14 → jobflow-0.1.15/src/jobflow.egg-info}/PKG-INFO +19 -20
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow.egg-info/SOURCES.txt +1 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow.egg-info/requires.txt +11 -11
- {jobflow-0.1.14 → jobflow-0.1.15}/tests/test_settings.py +1 -1
- jobflow-0.1.15/tests/test_version.py +9 -0
- jobflow-0.1.14/src/jobflow/_version.py +0 -7
- jobflow-0.1.14/tests/test_version.py +0 -34
- {jobflow-0.1.14 → jobflow-0.1.15}/LICENSE +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/setup.cfg +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/__init__.py +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/core/__init__.py +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/managers/__init__.py +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/py.typed +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/utils/__init__.py +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow/utils/uuid.py +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow.egg-info/dependency_links.txt +0 -0
- {jobflow-0.1.14 → jobflow-0.1.15}/src/jobflow.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: jobflow
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.15
|
|
4
4
|
Summary: jobflow is a library for writing computational workflows
|
|
5
5
|
Author-email: Alex Ganose <alexganose@gmail.com>
|
|
6
6
|
License: modified BSD
|
|
@@ -15,27 +15,26 @@ Classifier: Intended Audience :: Science/Research
|
|
|
15
15
|
Classifier: Intended Audience :: System Administrators
|
|
16
16
|
Classifier: Operating System :: OS Independent
|
|
17
17
|
Classifier: Programming Language :: Python :: 3
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
-
Classifier: Programming Language :: Python :: 3.
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.9
|
|
22
21
|
Classifier: Topic :: Database :: Front-Ends
|
|
23
22
|
Classifier: Topic :: Other/Nonlisted Topic
|
|
24
23
|
Classifier: Topic :: Scientific/Engineering
|
|
25
|
-
Requires-Python: >=3.
|
|
24
|
+
Requires-Python: >=3.9
|
|
26
25
|
Description-Content-Type: text/markdown
|
|
27
26
|
License-File: LICENSE
|
|
28
27
|
Requires-Dist: PyYAML
|
|
29
28
|
Requires-Dist: maggma>=0.57.0
|
|
30
29
|
Requires-Dist: monty>=2023.9.25
|
|
31
30
|
Requires-Dist: networkx
|
|
32
|
-
Requires-Dist: pydantic>=2.0.1
|
|
33
31
|
Requires-Dist: pydantic-settings>=2.0.3
|
|
32
|
+
Requires-Dist: pydantic>=2.0.1
|
|
34
33
|
Requires-Dist: pydash
|
|
35
34
|
Provides-Extra: docs
|
|
36
35
|
Requires-Dist: autodoc_pydantic==2.0.1; extra == "docs"
|
|
37
36
|
Requires-Dist: furo==2023.9.10; extra == "docs"
|
|
38
|
-
Requires-Dist: ipython==8.
|
|
37
|
+
Requires-Dist: ipython==8.18.1; extra == "docs"
|
|
39
38
|
Requires-Dist: myst_parser==2.0.0; extra == "docs"
|
|
40
39
|
Requires-Dist: nbsphinx==0.9.3; extra == "docs"
|
|
41
40
|
Requires-Dist: sphinx-copybutton==0.5.2; extra == "docs"
|
|
@@ -43,9 +42,9 @@ Requires-Dist: sphinx==7.2.6; extra == "docs"
|
|
|
43
42
|
Provides-Extra: dev
|
|
44
43
|
Requires-Dist: pre-commit>=2.12.1; extra == "dev"
|
|
45
44
|
Provides-Extra: tests
|
|
45
|
+
Requires-Dist: moto==4.2.11; extra == "tests"
|
|
46
46
|
Requires-Dist: pytest-cov==4.1.0; extra == "tests"
|
|
47
|
-
Requires-Dist: pytest==7.4.
|
|
48
|
-
Requires-Dist: moto==4.2.4; extra == "tests"
|
|
47
|
+
Requires-Dist: pytest==7.4.3; extra == "tests"
|
|
49
48
|
Provides-Extra: vis
|
|
50
49
|
Requires-Dist: matplotlib; extra == "vis"
|
|
51
50
|
Requires-Dist: pydot; extra == "vis"
|
|
@@ -54,23 +53,23 @@ Requires-Dist: FireWorks; extra == "fireworks"
|
|
|
54
53
|
Provides-Extra: strict
|
|
55
54
|
Requires-Dist: FireWorks==2.0.3; extra == "strict"
|
|
56
55
|
Requires-Dist: PyYAML==6.0.1; extra == "strict"
|
|
57
|
-
Requires-Dist: maggma==0.
|
|
58
|
-
Requires-Dist: matplotlib==3.
|
|
59
|
-
Requires-Dist: monty==2023.
|
|
60
|
-
Requires-Dist: moto==4.2.
|
|
61
|
-
Requires-Dist: networkx==3.1; extra == "strict"
|
|
62
|
-
Requires-Dist: pydantic==2.
|
|
63
|
-
Requires-Dist: pydantic
|
|
56
|
+
Requires-Dist: maggma==0.58.0; extra == "strict"
|
|
57
|
+
Requires-Dist: matplotlib==3.8.2; extra == "strict"
|
|
58
|
+
Requires-Dist: monty==2023.11.3; extra == "strict"
|
|
59
|
+
Requires-Dist: moto==4.2.11; extra == "strict"
|
|
60
|
+
Requires-Dist: networkx==3.2.1; extra == "strict"
|
|
61
|
+
Requires-Dist: pydantic-settings==2.1.0; extra == "strict"
|
|
62
|
+
Requires-Dist: pydantic==2.5.2; extra == "strict"
|
|
64
63
|
Requires-Dist: pydash==7.0.6; extra == "strict"
|
|
65
64
|
Requires-Dist: pydot==1.4.2; extra == "strict"
|
|
66
65
|
Requires-Dist: typing-extensions==4.8.0; extra == "strict"
|
|
67
66
|
|
|
68
67
|
# jobflow
|
|
69
68
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
69
|
+
[](https://github.com/materialsproject/jobflow/actions?query=workflow%3Atesting)
|
|
70
|
+
[](https://codecov.io/gh/materialsproject/jobflow/)
|
|
71
|
+
[](https://pypi.org/project/jobflow/)
|
|
72
|
+

|
|
74
73
|
|
|
75
74
|
[Documentation](https://materialsproject.github.io/jobflow/) | [PyPI](https://pypi.org/project/jobflow/) | [GitHub](https://github.com/materialsproject/jobflow)
|
|
76
75
|
|
|
@@ -143,7 +142,7 @@ the jobs is determined automatically and can be visualised using the flow graph.
|
|
|
143
142
|
|
|
144
143
|
## Installation
|
|
145
144
|
|
|
146
|
-
|
|
145
|
+
`jobflow` is a Python 3.9+ library and can be installed using `pip`.
|
|
147
146
|
|
|
148
147
|
```bash
|
|
149
148
|
pip install jobflow
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# jobflow
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
[](https://github.com/materialsproject/jobflow/actions?query=workflow%3Atesting)
|
|
4
|
+
[](https://codecov.io/gh/materialsproject/jobflow/)
|
|
5
|
+
[](https://pypi.org/project/jobflow/)
|
|
6
|
+

|
|
7
7
|
|
|
8
8
|
[Documentation](https://materialsproject.github.io/jobflow/) | [PyPI](https://pypi.org/project/jobflow/) | [GitHub](https://github.com/materialsproject/jobflow)
|
|
9
9
|
|
|
@@ -76,7 +76,7 @@ the jobs is determined automatically and can be visualised using the flow graph.
|
|
|
76
76
|
|
|
77
77
|
## Installation
|
|
78
78
|
|
|
79
|
-
|
|
79
|
+
`jobflow` is a Python 3.9+ library and can be installed using `pip`.
|
|
80
80
|
|
|
81
81
|
```bash
|
|
82
82
|
pip install jobflow
|
|
@@ -17,22 +17,21 @@ classifiers = [
|
|
|
17
17
|
"Intended Audience :: System Administrators",
|
|
18
18
|
"Operating System :: OS Independent",
|
|
19
19
|
"Programming Language :: Python :: 3",
|
|
20
|
-
"Programming Language :: Python :: 3.11",
|
|
21
20
|
"Programming Language :: Python :: 3.10",
|
|
22
|
-
"Programming Language :: Python :: 3.
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
23
22
|
"Programming Language :: Python :: 3.9",
|
|
24
23
|
"Topic :: Database :: Front-Ends",
|
|
25
24
|
"Topic :: Other/Nonlisted Topic",
|
|
26
25
|
"Topic :: Scientific/Engineering",
|
|
27
26
|
]
|
|
28
|
-
requires-python = ">=3.
|
|
27
|
+
requires-python = ">=3.9"
|
|
29
28
|
dependencies = [
|
|
30
29
|
"PyYAML",
|
|
31
30
|
"maggma>=0.57.0",
|
|
32
31
|
"monty>=2023.9.25",
|
|
33
32
|
"networkx",
|
|
34
|
-
"pydantic>=2.0.1",
|
|
35
33
|
"pydantic-settings>=2.0.3",
|
|
34
|
+
"pydantic>=2.0.1",
|
|
36
35
|
"pydash",
|
|
37
36
|
]
|
|
38
37
|
|
|
@@ -40,26 +39,26 @@ dependencies = [
|
|
|
40
39
|
docs = [
|
|
41
40
|
"autodoc_pydantic==2.0.1",
|
|
42
41
|
"furo==2023.9.10",
|
|
43
|
-
"ipython==8.
|
|
42
|
+
"ipython==8.18.1",
|
|
44
43
|
"myst_parser==2.0.0",
|
|
45
44
|
"nbsphinx==0.9.3",
|
|
46
45
|
"sphinx-copybutton==0.5.2",
|
|
47
46
|
"sphinx==7.2.6",
|
|
48
47
|
]
|
|
49
48
|
dev = ["pre-commit>=2.12.1"]
|
|
50
|
-
tests = ["pytest-cov==4.1.0", "pytest==7.4.
|
|
49
|
+
tests = ["moto==4.2.11", "pytest-cov==4.1.0", "pytest==7.4.3"]
|
|
51
50
|
vis = ["matplotlib", "pydot"]
|
|
52
51
|
fireworks = ["FireWorks"]
|
|
53
52
|
strict = [
|
|
54
53
|
"FireWorks==2.0.3",
|
|
55
54
|
"PyYAML==6.0.1",
|
|
56
|
-
"maggma==0.
|
|
57
|
-
"matplotlib==3.
|
|
58
|
-
"monty==2023.
|
|
59
|
-
"moto==4.2.
|
|
60
|
-
"networkx==3.1",
|
|
61
|
-
"pydantic==2.
|
|
62
|
-
"pydantic
|
|
55
|
+
"maggma==0.58.0",
|
|
56
|
+
"matplotlib==3.8.2",
|
|
57
|
+
"monty==2023.11.3",
|
|
58
|
+
"moto==4.2.11",
|
|
59
|
+
"networkx==3.2.1",
|
|
60
|
+
"pydantic-settings==2.1.0",
|
|
61
|
+
"pydantic==2.5.2",
|
|
63
62
|
"pydash==7.0.6",
|
|
64
63
|
"pydot==1.4.2",
|
|
65
64
|
"typing-extensions==4.8.0",
|
|
@@ -83,7 +82,7 @@ max-line-length = 88
|
|
|
83
82
|
max-doc-length = 88
|
|
84
83
|
select = "C, E, F, W, B"
|
|
85
84
|
extend-ignore = "E203, W503, E501, F401, RST21"
|
|
86
|
-
min-python-version = "3.
|
|
85
|
+
min-python-version = "3.9.0"
|
|
87
86
|
docstring-convention = "numpy"
|
|
88
87
|
rst-roles = "class, func, ref, obj"
|
|
89
88
|
|
|
@@ -119,29 +118,55 @@ exclude_lines = [
|
|
|
119
118
|
]
|
|
120
119
|
|
|
121
120
|
[tool.ruff]
|
|
122
|
-
target-version = "
|
|
121
|
+
target-version = "py39"
|
|
123
122
|
ignore-init-module-imports = true
|
|
124
123
|
select = [
|
|
125
|
-
"B",
|
|
126
|
-
"C4",
|
|
127
|
-
"D",
|
|
128
|
-
"E",
|
|
129
|
-
"
|
|
130
|
-
"
|
|
131
|
-
"
|
|
132
|
-
"
|
|
133
|
-
"
|
|
134
|
-
"
|
|
135
|
-
"
|
|
136
|
-
"
|
|
137
|
-
"
|
|
138
|
-
"
|
|
139
|
-
"
|
|
124
|
+
"B", # flake8-bugbear
|
|
125
|
+
"C4", # flake8-comprehensions
|
|
126
|
+
"D", # pydocstyle
|
|
127
|
+
"E", # pycodestyle error
|
|
128
|
+
"EXE", # flake8-executable
|
|
129
|
+
"F", # pyflakes
|
|
130
|
+
"FA", # flake8-future-annotations
|
|
131
|
+
"FBT003", # boolean-positional-value-in-call
|
|
132
|
+
"FLY", # flynt
|
|
133
|
+
"I", # isort
|
|
134
|
+
"ICN", # flake8-import-conventions
|
|
135
|
+
"ISC", # flake8-implicit-str-concat
|
|
136
|
+
"PD", # pandas-vet
|
|
137
|
+
"PERF", # perflint
|
|
138
|
+
"PIE", # flake8-pie
|
|
139
|
+
"PL", # pylint
|
|
140
|
+
"PT", # flake8-pytest-style
|
|
141
|
+
"PYI", # flakes8-pyi
|
|
142
|
+
"Q", # flake8-quotes
|
|
143
|
+
"RET", # flake8-return
|
|
144
|
+
"RSE", # flake8-raise
|
|
145
|
+
"RUF", # Ruff-specific rules
|
|
146
|
+
"SIM", # flake8-simplify
|
|
147
|
+
"SLOT", # flake8-slots
|
|
148
|
+
"TCH", # flake8-type-checking
|
|
149
|
+
"TID", # flake8-tidy-imports
|
|
150
|
+
"UP", # pyupgrade
|
|
151
|
+
"W", # pycodestyle warning
|
|
152
|
+
"YTT", # flake8-2020
|
|
153
|
+
]
|
|
154
|
+
ignore = [
|
|
155
|
+
"B028",
|
|
156
|
+
"PLR0911", # too-many-return-statements
|
|
157
|
+
"PLR0912", # too-many-branches
|
|
158
|
+
"PLR0913", # too-many-arguments
|
|
159
|
+
"PLR0915", # too-many-statements
|
|
160
|
+
"PLW0603",
|
|
161
|
+
"RUF013",
|
|
140
162
|
]
|
|
141
|
-
ignore = ["B028", "PLW0603", "RUF013"]
|
|
142
163
|
pydocstyle.convention = "numpy"
|
|
143
164
|
isort.known-first-party = ["jobflow"]
|
|
144
165
|
|
|
145
166
|
[tool.ruff.per-file-ignores]
|
|
167
|
+
# F401: unused import
|
|
146
168
|
"__init__.py" = ["F401"]
|
|
147
|
-
|
|
169
|
+
# D: pydocstyle
|
|
170
|
+
# PLR2004: magic-value-comparison
|
|
171
|
+
# PT004: pytest-missing-fixture-name-underscore
|
|
172
|
+
"**/tests/*" = ["D", "PLR2004", "PT004"]
|
|
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import logging
|
|
6
6
|
import warnings
|
|
7
7
|
from copy import deepcopy
|
|
8
|
-
from typing import TYPE_CHECKING
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
9
9
|
|
|
10
10
|
from monty.json import MSONable
|
|
11
11
|
|
|
@@ -14,13 +14,13 @@ from jobflow.core.reference import find_and_get_references
|
|
|
14
14
|
from jobflow.utils import ValueEnum, contains_flow_or_job, suuid
|
|
15
15
|
|
|
16
16
|
if TYPE_CHECKING:
|
|
17
|
-
from
|
|
17
|
+
from collections.abc import Iterator, Sequence
|
|
18
|
+
from typing import Any, Callable
|
|
18
19
|
|
|
19
20
|
from networkx import DiGraph
|
|
20
21
|
|
|
21
22
|
from jobflow import Job
|
|
22
23
|
|
|
23
|
-
__all__ = ["JobOrder", "Flow", "get_flow"]
|
|
24
24
|
|
|
25
25
|
logger = logging.getLogger(__name__)
|
|
26
26
|
|
|
@@ -166,7 +166,7 @@ class Flow(MSONable):
|
|
|
166
166
|
raise TypeError(
|
|
167
167
|
f"Flow can only contain Job or Flow objects, not {type(value).__name__}"
|
|
168
168
|
)
|
|
169
|
-
jobs = list(self
|
|
169
|
+
jobs = list(self)
|
|
170
170
|
jobs[idx] = value # type: ignore[index, assignment]
|
|
171
171
|
self.jobs = tuple(jobs)
|
|
172
172
|
|
|
@@ -188,10 +188,10 @@ class Flow(MSONable):
|
|
|
188
188
|
|
|
189
189
|
def __sub__(self, other: Flow | Job) -> Flow:
|
|
190
190
|
"""Remove a job or subflow from the flow."""
|
|
191
|
-
if other not in self
|
|
191
|
+
if other not in self:
|
|
192
192
|
raise ValueError(f"{other!r} not found in flow")
|
|
193
193
|
new_flow = deepcopy(self)
|
|
194
|
-
new_flow.jobs = tuple([job for job in new_flow
|
|
194
|
+
new_flow.jobs = tuple([job for job in new_flow if job != other])
|
|
195
195
|
return new_flow
|
|
196
196
|
|
|
197
197
|
def __repr__(self, level: int = 0, prefix: str = "") -> str:
|
|
@@ -201,8 +201,8 @@ class Flow(MSONable):
|
|
|
201
201
|
_prefix = f"{prefix}." if prefix else ""
|
|
202
202
|
job_reprs = "\n".join(
|
|
203
203
|
f"{indent}{_prefix}{i}. "
|
|
204
|
-
f"{
|
|
205
|
-
for i,
|
|
204
|
+
f"{jb.__repr__(level + 1, f'{_prefix}{i}') if isinstance(jb, Flow) else jb}"
|
|
205
|
+
for i, jb in enumerate(self, 1)
|
|
206
206
|
)
|
|
207
207
|
return f"Flow({name=}, {uuid=})\n{job_reprs}"
|
|
208
208
|
|
|
@@ -298,7 +298,7 @@ class Flow(MSONable):
|
|
|
298
298
|
The uuids of all Jobs in the Flow (including nested Flows).
|
|
299
299
|
"""
|
|
300
300
|
uuids: list[str] = []
|
|
301
|
-
for job in self
|
|
301
|
+
for job in self:
|
|
302
302
|
if isinstance(job, Flow):
|
|
303
303
|
uuids.extend(job.job_uuids)
|
|
304
304
|
else:
|
|
@@ -316,7 +316,7 @@ class Flow(MSONable):
|
|
|
316
316
|
The uuids of all Jobs and Flows in the Flow (including nested Flows).
|
|
317
317
|
"""
|
|
318
318
|
uuids: list[str] = []
|
|
319
|
-
for job in self
|
|
319
|
+
for job in self:
|
|
320
320
|
if isinstance(job, Flow):
|
|
321
321
|
uuids.extend(job.all_uuids)
|
|
322
322
|
uuids.append(job.uuid)
|
|
@@ -336,7 +336,7 @@ class Flow(MSONable):
|
|
|
336
336
|
|
|
337
337
|
import networkx as nx
|
|
338
338
|
|
|
339
|
-
graph = nx.compose_all([job.graph for job in self
|
|
339
|
+
graph = nx.compose_all([job.graph for job in self])
|
|
340
340
|
|
|
341
341
|
for node in graph:
|
|
342
342
|
node_props = graph.nodes[node]
|
|
@@ -346,7 +346,7 @@ class Flow(MSONable):
|
|
|
346
346
|
if self.order == JobOrder.LINEAR:
|
|
347
347
|
# add fake edges between jobs to force linear order
|
|
348
348
|
edges = []
|
|
349
|
-
for job_a, job_b in nx.utils.pairwise(self
|
|
349
|
+
for job_a, job_b in nx.utils.pairwise(self):
|
|
350
350
|
if isinstance(job_a, Flow):
|
|
351
351
|
leaves = [v for v, d in job_a.graph.out_degree() if d == 0]
|
|
352
352
|
else:
|
|
@@ -474,7 +474,7 @@ class Flow(MSONable):
|
|
|
474
474
|
>>> flow.update_kwargs({"number": 10}, name_filter="add")
|
|
475
475
|
>>> flow.update_kwargs({"number": 10}, function_filter=add)
|
|
476
476
|
"""
|
|
477
|
-
for job in self
|
|
477
|
+
for job in self:
|
|
478
478
|
job.update_kwargs(
|
|
479
479
|
update,
|
|
480
480
|
name_filter=name_filter,
|
|
@@ -573,7 +573,7 @@ class Flow(MSONable):
|
|
|
573
573
|
... {"number": 10}, class_filter=AddMaker, nested=False
|
|
574
574
|
... )
|
|
575
575
|
"""
|
|
576
|
-
for job in self
|
|
576
|
+
for job in self:
|
|
577
577
|
job.update_maker_kwargs(
|
|
578
578
|
update,
|
|
579
579
|
name_filter=name_filter,
|
|
@@ -598,7 +598,7 @@ class Flow(MSONable):
|
|
|
598
598
|
else:
|
|
599
599
|
self.name += append_str
|
|
600
600
|
|
|
601
|
-
for job in self
|
|
601
|
+
for job in self:
|
|
602
602
|
job.append_name(append_str, prepend=prepend)
|
|
603
603
|
|
|
604
604
|
def update_metadata(
|
|
@@ -647,7 +647,7 @@ class Flow(MSONable):
|
|
|
647
647
|
|
|
648
648
|
>>> flow.update_metadata({"tag": "addition_job"})
|
|
649
649
|
"""
|
|
650
|
-
for job in self
|
|
650
|
+
for job in self:
|
|
651
651
|
job.update_metadata(
|
|
652
652
|
update,
|
|
653
653
|
name_filter=name_filter,
|
|
@@ -717,7 +717,7 @@ class Flow(MSONable):
|
|
|
717
717
|
|
|
718
718
|
>>> flow.update_config({"manager_config": {"_fworker": "myfworker"}})
|
|
719
719
|
"""
|
|
720
|
-
for job in self
|
|
720
|
+
for job in self:
|
|
721
721
|
job.update_config(
|
|
722
722
|
config,
|
|
723
723
|
name_filter=name_filter,
|
|
@@ -756,8 +756,8 @@ class Flow(MSONable):
|
|
|
756
756
|
self.hosts.extend(hosts_uuids)
|
|
757
757
|
else:
|
|
758
758
|
hosts_uuids = [self.uuid]
|
|
759
|
-
for
|
|
760
|
-
|
|
759
|
+
for job in self:
|
|
760
|
+
job.add_hosts_uuids(hosts_uuids, prepend=prepend)
|
|
761
761
|
|
|
762
762
|
def add_jobs(self, jobs: Job | Flow | Sequence[Flow | Job]) -> None:
|
|
763
763
|
"""
|
|
@@ -794,7 +794,8 @@ class Flow(MSONable):
|
|
|
794
794
|
f"current Flow ({self.uuid})"
|
|
795
795
|
)
|
|
796
796
|
job_ids.add(job.uuid)
|
|
797
|
-
job.
|
|
797
|
+
if job.host != self.uuid:
|
|
798
|
+
job.add_hosts_uuids(hosts)
|
|
798
799
|
self._jobs += tuple(jobs)
|
|
799
800
|
|
|
800
801
|
def remove_jobs(self, indices: int | list[int]):
|
|
@@ -810,12 +811,12 @@ class Flow(MSONable):
|
|
|
810
811
|
"""
|
|
811
812
|
if not isinstance(indices, (list, tuple)):
|
|
812
813
|
indices = [indices]
|
|
813
|
-
if any(
|
|
814
|
+
if any(idx < 0 or idx >= len(self) for idx in indices):
|
|
814
815
|
raise ValueError(
|
|
815
816
|
"Only indices between 0 and the number of the jobs are accepted"
|
|
816
817
|
)
|
|
817
818
|
|
|
818
|
-
new_jobs = tuple(
|
|
819
|
+
new_jobs = tuple(job for idx, job in enumerate(self) if idx not in indices)
|
|
819
820
|
uuids: set = set()
|
|
820
821
|
for job in new_jobs:
|
|
821
822
|
if isinstance(job, Flow):
|
|
@@ -13,7 +13,8 @@ from jobflow.core.reference import OnMissing, OutputReference
|
|
|
13
13
|
from jobflow.utils.uuid import suuid
|
|
14
14
|
|
|
15
15
|
if typing.TYPE_CHECKING:
|
|
16
|
-
from
|
|
16
|
+
from collections.abc import Hashable, Sequence
|
|
17
|
+
from typing import Any, Callable
|
|
17
18
|
|
|
18
19
|
from networkx import DiGraph
|
|
19
20
|
from pydantic import BaseModel
|
|
@@ -22,8 +23,6 @@ if typing.TYPE_CHECKING:
|
|
|
22
23
|
|
|
23
24
|
logger = logging.getLogger(__name__)
|
|
24
25
|
|
|
25
|
-
__all__ = ["job", "Job", "Response", "JobConfig", "store_inputs"]
|
|
26
|
-
|
|
27
26
|
|
|
28
27
|
@dataclass
|
|
29
28
|
class JobConfig(MSONable):
|
|
@@ -560,6 +559,7 @@ class Job(MSONable):
|
|
|
560
559
|
|
|
561
560
|
from jobflow import CURRENT_JOB
|
|
562
561
|
from jobflow.core.flow import get_flow
|
|
562
|
+
from jobflow.core.schemas import JobStoreDocument
|
|
563
563
|
|
|
564
564
|
index_str = f", {self.index}" if self.index != 1 else ""
|
|
565
565
|
logger.info(f"Starting job - {self.name} ({self.uuid}{index_str})")
|
|
@@ -633,15 +633,15 @@ class Job(MSONable):
|
|
|
633
633
|
) from err
|
|
634
634
|
|
|
635
635
|
save = {k: "output" if v is True else v for k, v in self._kwargs.items()}
|
|
636
|
-
data =
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
636
|
+
data: JobStoreDocument = JobStoreDocument(
|
|
637
|
+
uuid=self.uuid,
|
|
638
|
+
index=self.index,
|
|
639
|
+
output=output,
|
|
640
|
+
completed_at=datetime.now().isoformat(),
|
|
641
|
+
metadata=self.metadata,
|
|
642
|
+
hosts=self.hosts,
|
|
643
|
+
name=self.name,
|
|
644
|
+
)
|
|
645
645
|
store.update(data, key=["uuid", "index"], save=save)
|
|
646
646
|
|
|
647
647
|
CURRENT_JOB.reset()
|
|
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import contextlib
|
|
6
6
|
import typing
|
|
7
|
-
from typing import Any
|
|
7
|
+
from typing import Any
|
|
8
8
|
|
|
9
9
|
from monty.json import MontyDecoder, MontyEncoder, MSONable, jsanitize
|
|
10
10
|
from pydantic import BaseModel
|
|
@@ -13,15 +13,9 @@ from pydantic.v1.utils import lenient_issubclass
|
|
|
13
13
|
from jobflow.utils.enum import ValueEnum
|
|
14
14
|
|
|
15
15
|
if typing.TYPE_CHECKING:
|
|
16
|
-
import
|
|
16
|
+
from collections.abc import Sequence
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
"OnMissing",
|
|
20
|
-
"OutputReference",
|
|
21
|
-
"resolve_references",
|
|
22
|
-
"find_and_resolve_references",
|
|
23
|
-
"find_and_get_references",
|
|
24
|
-
]
|
|
18
|
+
import jobflow
|
|
25
19
|
|
|
26
20
|
|
|
27
21
|
class OnMissing(ValueEnum):
|
|
@@ -95,14 +89,14 @@ class OutputReference(MSONable):
|
|
|
95
89
|
uuid: str,
|
|
96
90
|
attributes: tuple[tuple[str, Any], ...] = (),
|
|
97
91
|
output_schema: type[BaseModel] = None,
|
|
98
|
-
):
|
|
92
|
+
) -> None:
|
|
99
93
|
super().__init__()
|
|
100
94
|
self.uuid = uuid
|
|
101
95
|
self.attributes = attributes
|
|
102
96
|
self.output_schema = output_schema
|
|
103
97
|
|
|
104
98
|
for attr_type, attr in attributes:
|
|
105
|
-
if attr_type not in
|
|
99
|
+
if attr_type not in {"a", "i"}:
|
|
106
100
|
raise ValueError(
|
|
107
101
|
f"Unrecognised attribute type '{attr_type}' for attribute '{attr}'"
|
|
108
102
|
)
|
|
@@ -165,11 +159,12 @@ class OutputReference(MSONable):
|
|
|
165
159
|
if on_missing == OnMissing.ERROR and index not in cache[self.uuid]:
|
|
166
160
|
istr = f" ({index})" if index is not None else ""
|
|
167
161
|
raise ValueError(
|
|
168
|
-
f"Could not resolve reference - {self.uuid}{istr} not in store or
|
|
162
|
+
f"Could not resolve reference - {self.uuid}{istr} not in store or "
|
|
163
|
+
f"{index=}, {cache=}"
|
|
169
164
|
)
|
|
170
|
-
|
|
165
|
+
if on_missing == OnMissing.NONE and index not in cache[self.uuid]:
|
|
171
166
|
return None
|
|
172
|
-
|
|
167
|
+
if on_missing == OnMissing.PASS and index not in cache[self.uuid]:
|
|
173
168
|
return self
|
|
174
169
|
|
|
175
170
|
data = cache[self.uuid][index]
|
|
@@ -182,7 +177,11 @@ class OutputReference(MSONable):
|
|
|
182
177
|
|
|
183
178
|
for attr_type, attr in self.attributes:
|
|
184
179
|
# i means index else use attribute access
|
|
185
|
-
data =
|
|
180
|
+
data = (
|
|
181
|
+
data[attr]
|
|
182
|
+
if attr_type == "i" or isinstance(data, dict)
|
|
183
|
+
else getattr(data, attr)
|
|
184
|
+
)
|
|
186
185
|
|
|
187
186
|
return data
|
|
188
187
|
|
|
@@ -206,12 +205,11 @@ class OutputReference(MSONable):
|
|
|
206
205
|
if inplace:
|
|
207
206
|
self.uuid = uuid
|
|
208
207
|
return self
|
|
209
|
-
|
|
210
|
-
from copy import deepcopy
|
|
208
|
+
from copy import deepcopy
|
|
211
209
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
210
|
+
new_reference = deepcopy(self)
|
|
211
|
+
new_reference.uuid = uuid
|
|
212
|
+
return new_reference
|
|
215
213
|
|
|
216
214
|
def __getitem__(self, item) -> OutputReference:
|
|
217
215
|
"""Index the reference."""
|
|
@@ -269,7 +267,7 @@ class OutputReference(MSONable):
|
|
|
269
267
|
"""Return a hash of the reference."""
|
|
270
268
|
return hash(str(self))
|
|
271
269
|
|
|
272
|
-
def __eq__(self, other:
|
|
270
|
+
def __eq__(self, other: object) -> bool:
|
|
273
271
|
"""Test for equality against another reference."""
|
|
274
272
|
if isinstance(other, OutputReference):
|
|
275
273
|
return (
|
|
@@ -291,7 +289,7 @@ class OutputReference(MSONable):
|
|
|
291
289
|
"""Serialize the reference as a dict."""
|
|
292
290
|
schema = self.output_schema
|
|
293
291
|
schema_dict = MontyEncoder().default(schema) if schema is not None else None
|
|
294
|
-
|
|
292
|
+
return {
|
|
295
293
|
"@module": self.__class__.__module__,
|
|
296
294
|
"@class": type(self).__name__,
|
|
297
295
|
"@version": None,
|
|
@@ -299,7 +297,6 @@ class OutputReference(MSONable):
|
|
|
299
297
|
"attributes": self.attributes,
|
|
300
298
|
"output_schema": schema_dict,
|
|
301
299
|
}
|
|
302
|
-
return data
|
|
303
300
|
|
|
304
301
|
|
|
305
302
|
def resolve_references(
|
|
@@ -382,7 +379,7 @@ def find_and_get_references(arg: Any) -> tuple[OutputReference, ...]:
|
|
|
382
379
|
# if the argument is a reference then stop there
|
|
383
380
|
return (arg,)
|
|
384
381
|
|
|
385
|
-
|
|
382
|
+
if isinstance(arg, (float, int, str, bool)):
|
|
386
383
|
# argument is a primitive, we won't find a reference here
|
|
387
384
|
return ()
|
|
388
385
|
|
|
@@ -438,7 +435,7 @@ def find_and_resolve_references(
|
|
|
438
435
|
# if the argument is a reference then stop there
|
|
439
436
|
return arg.resolve(store, cache=cache, on_missing=on_missing)
|
|
440
437
|
|
|
441
|
-
|
|
438
|
+
if isinstance(arg, (float, int, str, bool)):
|
|
442
439
|
# argument is a primitive, we won't find a reference here
|
|
443
440
|
return arg
|
|
444
441
|
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""A Pydantic model for Jobstore document."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class JobStoreDocument(BaseModel):
|
|
9
|
+
"""A Pydantic model for Jobstore document."""
|
|
10
|
+
|
|
11
|
+
uuid: str = Field(
|
|
12
|
+
None, description="An unique identifier for the job. Generated automatically."
|
|
13
|
+
)
|
|
14
|
+
index: int = Field(
|
|
15
|
+
None,
|
|
16
|
+
description="The index of the job (number of times the job has been replaced).",
|
|
17
|
+
)
|
|
18
|
+
output: Any = Field(
|
|
19
|
+
None,
|
|
20
|
+
description="This is a reference to the future job output.",
|
|
21
|
+
)
|
|
22
|
+
completed_at: str = Field(None, description="The time the job was completed.")
|
|
23
|
+
metadata: dict = Field(
|
|
24
|
+
None,
|
|
25
|
+
description="Metadata information supplied by the user.",
|
|
26
|
+
)
|
|
27
|
+
hosts: list[str] = Field(
|
|
28
|
+
None,
|
|
29
|
+
description="The list of UUIDs of the hosts containing the job.",
|
|
30
|
+
)
|
|
31
|
+
name: str = Field(
|
|
32
|
+
None,
|
|
33
|
+
description="The name of the job.",
|
|
34
|
+
)
|