jobflow 0.1.16__py3-none-any.whl → 0.1.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jobflow/core/flow.py +6 -7
- jobflow/core/job.py +47 -13
- jobflow/core/maker.py +1 -3
- jobflow/core/reference.py +1 -1
- jobflow/core/store.py +7 -6
- jobflow/managers/fireworks.py +15 -0
- jobflow/managers/local.py +22 -10
- jobflow/settings.py +5 -0
- jobflow/utils/__init__.py +2 -0
- jobflow/utils/uid.py +115 -0
- jobflow/utils/uuid.py +5 -0
- {jobflow-0.1.16.dist-info → jobflow-0.1.18.dist-info}/METADATA +39 -22
- jobflow-0.1.18.dist-info/RECORD +28 -0
- {jobflow-0.1.16.dist-info → jobflow-0.1.18.dist-info}/WHEEL +1 -1
- jobflow-0.1.16.dist-info/RECORD +0 -27
- {jobflow-0.1.16.dist-info → jobflow-0.1.18.dist-info}/LICENSE +0 -0
- {jobflow-0.1.16.dist-info → jobflow-0.1.18.dist-info}/top_level.txt +0 -0
jobflow/core/flow.py
CHANGED
|
@@ -11,7 +11,7 @@ from monty.json import MSONable
|
|
|
11
11
|
|
|
12
12
|
import jobflow
|
|
13
13
|
from jobflow.core.reference import find_and_get_references
|
|
14
|
-
from jobflow.utils import ValueEnum, contains_flow_or_job,
|
|
14
|
+
from jobflow.utils import ValueEnum, contains_flow_or_job, suid
|
|
15
15
|
|
|
16
16
|
if TYPE_CHECKING:
|
|
17
17
|
from collections.abc import Iterator, Sequence
|
|
@@ -135,7 +135,7 @@ class Flow(MSONable):
|
|
|
135
135
|
jobs = [jobs]
|
|
136
136
|
|
|
137
137
|
if uuid is None:
|
|
138
|
-
uuid =
|
|
138
|
+
uuid = suid()
|
|
139
139
|
|
|
140
140
|
self.name = name
|
|
141
141
|
self.order = order
|
|
@@ -158,9 +158,8 @@ class Flow(MSONable):
|
|
|
158
158
|
self, idx: int | slice, value: Flow | Job | Sequence[Flow | Job]
|
|
159
159
|
) -> None:
|
|
160
160
|
"""Set the job(s) or subflow(s) at the given index/slice."""
|
|
161
|
-
if (
|
|
162
|
-
|
|
163
|
-
or isinstance(value, (tuple, list))
|
|
161
|
+
if not isinstance(value, (Flow, jobflow.Job, tuple, list)) or (
|
|
162
|
+
isinstance(value, (tuple, list))
|
|
164
163
|
and not all(isinstance(v, (Flow, jobflow.Job)) for v in value)
|
|
165
164
|
):
|
|
166
165
|
raise TypeError(
|
|
@@ -583,7 +582,7 @@ class Flow(MSONable):
|
|
|
583
582
|
dict_mod=dict_mod,
|
|
584
583
|
)
|
|
585
584
|
|
|
586
|
-
def append_name(self, append_str: str, prepend: bool = False):
|
|
585
|
+
def append_name(self, append_str: str, prepend: bool = False, dynamic: bool = True):
|
|
587
586
|
"""
|
|
588
587
|
Append a string to the name of the flow and all jobs contained in it.
|
|
589
588
|
|
|
@@ -600,7 +599,7 @@ class Flow(MSONable):
|
|
|
600
599
|
self.name += append_str
|
|
601
600
|
|
|
602
601
|
for job in self:
|
|
603
|
-
job.append_name(append_str, prepend=prepend)
|
|
602
|
+
job.append_name(append_str, prepend=prepend, dynamic=dynamic)
|
|
604
603
|
|
|
605
604
|
def update_metadata(
|
|
606
605
|
self,
|
jobflow/core/job.py
CHANGED
|
@@ -6,14 +6,17 @@ import logging
|
|
|
6
6
|
import typing
|
|
7
7
|
import warnings
|
|
8
8
|
from dataclasses import dataclass, field
|
|
9
|
+
from typing import cast, overload
|
|
9
10
|
|
|
10
11
|
from monty.json import MSONable, jsanitize
|
|
12
|
+
from typing_extensions import Self
|
|
11
13
|
|
|
12
14
|
from jobflow.core.reference import OnMissing, OutputReference
|
|
13
|
-
from jobflow.utils.
|
|
15
|
+
from jobflow.utils.uid import suid
|
|
14
16
|
|
|
15
17
|
if typing.TYPE_CHECKING:
|
|
16
18
|
from collections.abc import Hashable, Sequence
|
|
19
|
+
from pathlib import Path
|
|
17
20
|
from typing import Any, Callable
|
|
18
21
|
|
|
19
22
|
from networkx import DiGraph
|
|
@@ -65,7 +68,19 @@ class JobConfig(MSONable):
|
|
|
65
68
|
response_manager_config: dict = field(default_factory=dict)
|
|
66
69
|
|
|
67
70
|
|
|
68
|
-
|
|
71
|
+
@overload
|
|
72
|
+
def job(method: Callable = None) -> Callable[..., Job]:
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@overload
|
|
77
|
+
def job(method: Callable = None, **job_kwargs) -> Callable[..., Callable[..., Job]]:
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def job(
|
|
82
|
+
method: Callable = None, **job_kwargs
|
|
83
|
+
) -> Callable[..., Job] | Callable[..., Callable[..., Job]]:
|
|
69
84
|
"""
|
|
70
85
|
Wrap a function to produce a :obj:`Job`.
|
|
71
86
|
|
|
@@ -313,6 +328,7 @@ class Job(MSONable):
|
|
|
313
328
|
hosts: list[str] = None,
|
|
314
329
|
metadata_updates: list[dict[str, Any]] = None,
|
|
315
330
|
config_updates: list[dict[str, Any]] = None,
|
|
331
|
+
name_updates: list[dict[str, Any]] = None,
|
|
316
332
|
**kwargs,
|
|
317
333
|
):
|
|
318
334
|
from copy import deepcopy
|
|
@@ -321,7 +337,7 @@ class Job(MSONable):
|
|
|
321
337
|
|
|
322
338
|
function_args = () if function_args is None else function_args
|
|
323
339
|
function_kwargs = {} if function_kwargs is None else function_kwargs
|
|
324
|
-
uuid =
|
|
340
|
+
uuid = suid() if uuid is None else uuid
|
|
325
341
|
metadata = {} if metadata is None else metadata
|
|
326
342
|
config = JobConfig() if config is None else config
|
|
327
343
|
|
|
@@ -337,6 +353,7 @@ class Job(MSONable):
|
|
|
337
353
|
self.config = config
|
|
338
354
|
self.hosts = hosts or []
|
|
339
355
|
self.metadata_updates = metadata_updates or []
|
|
356
|
+
self.name_updates = name_updates or []
|
|
340
357
|
self.config_updates = config_updates or []
|
|
341
358
|
self._kwargs = kwargs
|
|
342
359
|
|
|
@@ -526,7 +543,7 @@ class Job(MSONable):
|
|
|
526
543
|
self.uuid = uuid
|
|
527
544
|
self.output = self.output.set_uuid(uuid)
|
|
528
545
|
|
|
529
|
-
def run(self, store: jobflow.JobStore) -> Response:
|
|
546
|
+
def run(self, store: jobflow.JobStore, job_dir: Path = None) -> Response:
|
|
530
547
|
"""
|
|
531
548
|
Run the job.
|
|
532
549
|
|
|
@@ -581,7 +598,9 @@ class Job(MSONable):
|
|
|
581
598
|
function = types.MethodType(function, bound)
|
|
582
599
|
|
|
583
600
|
response = function(*self.function_args, **self.function_kwargs)
|
|
584
|
-
response = Response.from_job_returns(
|
|
601
|
+
response = Response.from_job_returns(
|
|
602
|
+
response, self.output_schema, job_dir=job_dir
|
|
603
|
+
)
|
|
585
604
|
|
|
586
605
|
if response.replace is not None:
|
|
587
606
|
response.replace = prepare_replace(response.replace, self)
|
|
@@ -604,6 +623,8 @@ class Job(MSONable):
|
|
|
604
623
|
new_jobs.update_metadata(**metadata_update, dynamic=True)
|
|
605
624
|
for config_update in self.config_updates:
|
|
606
625
|
new_jobs.update_config(**config_update, dynamic=True)
|
|
626
|
+
for name_update in self.name_updates:
|
|
627
|
+
new_jobs.append_name(**name_update, dynamic=True)
|
|
607
628
|
|
|
608
629
|
if self.config.response_manager_config:
|
|
609
630
|
passed_config = self.config.response_manager_config
|
|
@@ -872,7 +893,7 @@ class Job(MSONable):
|
|
|
872
893
|
dict_mod=dict_mod,
|
|
873
894
|
)
|
|
874
895
|
|
|
875
|
-
def append_name(self, append_str: str, prepend: bool = False):
|
|
896
|
+
def append_name(self, append_str: str, prepend: bool = False, dynamic: bool = True):
|
|
876
897
|
"""
|
|
877
898
|
Append a string to the name of the job.
|
|
878
899
|
|
|
@@ -882,12 +903,18 @@ class Job(MSONable):
|
|
|
882
903
|
A string to append.
|
|
883
904
|
prepend
|
|
884
905
|
Prepend the name rather than appending it.
|
|
906
|
+
dynamic
|
|
907
|
+
The updates will be propagated to Jobs/Flows dynamically generated at
|
|
908
|
+
runtime.
|
|
885
909
|
"""
|
|
886
910
|
if prepend:
|
|
887
911
|
self.name = append_str + self.name
|
|
888
912
|
else:
|
|
889
913
|
self.name += append_str
|
|
890
914
|
|
|
915
|
+
if dynamic:
|
|
916
|
+
self.name_updates.append({"append_str": append_str, "prepend": prepend})
|
|
917
|
+
|
|
891
918
|
def update_metadata(
|
|
892
919
|
self,
|
|
893
920
|
update: dict[str, Any],
|
|
@@ -1134,8 +1161,8 @@ class Job(MSONable):
|
|
|
1134
1161
|
prepend
|
|
1135
1162
|
Insert the UUIDs at the beginning of the list rather than extending it.
|
|
1136
1163
|
"""
|
|
1137
|
-
if
|
|
1138
|
-
hosts_uuids = [hosts_uuids]
|
|
1164
|
+
if isinstance(hosts_uuids, str):
|
|
1165
|
+
hosts_uuids = [hosts_uuids]
|
|
1139
1166
|
if prepend:
|
|
1140
1167
|
self.hosts[0:0] = hosts_uuids
|
|
1141
1168
|
else:
|
|
@@ -1170,6 +1197,8 @@ class Response(typing.Generic[T]):
|
|
|
1170
1197
|
Stop any children of the current flow.
|
|
1171
1198
|
stop_jobflow
|
|
1172
1199
|
Stop executing all remaining jobs.
|
|
1200
|
+
job_dir
|
|
1201
|
+
The directory where the job was run.
|
|
1173
1202
|
"""
|
|
1174
1203
|
|
|
1175
1204
|
output: T = None
|
|
@@ -1179,13 +1208,15 @@ class Response(typing.Generic[T]):
|
|
|
1179
1208
|
stored_data: dict[Hashable, Any] = None
|
|
1180
1209
|
stop_children: bool = False
|
|
1181
1210
|
stop_jobflow: bool = False
|
|
1211
|
+
job_dir: str | Path = None
|
|
1182
1212
|
|
|
1183
1213
|
@classmethod
|
|
1184
1214
|
def from_job_returns(
|
|
1185
1215
|
cls,
|
|
1186
1216
|
job_returns: Any | None,
|
|
1187
1217
|
output_schema: type[BaseModel] = None,
|
|
1188
|
-
|
|
1218
|
+
job_dir: str | Path = None,
|
|
1219
|
+
) -> Self:
|
|
1189
1220
|
"""
|
|
1190
1221
|
Generate a :obj:`Response` from the outputs of a :obj:`Job`.
|
|
1191
1222
|
|
|
@@ -1199,6 +1230,8 @@ class Response(typing.Generic[T]):
|
|
|
1199
1230
|
output_schema
|
|
1200
1231
|
A pydantic model associated with the job. Used to enforce a schema for the
|
|
1201
1232
|
outputs.
|
|
1233
|
+
job_dir
|
|
1234
|
+
The directory where the job was run.
|
|
1202
1235
|
|
|
1203
1236
|
Raises
|
|
1204
1237
|
------
|
|
@@ -1215,17 +1248,18 @@ class Response(typing.Generic[T]):
|
|
|
1215
1248
|
# only apply output schema if there is no replace.
|
|
1216
1249
|
job_returns.output = apply_schema(job_returns.output, output_schema)
|
|
1217
1250
|
|
|
1218
|
-
|
|
1251
|
+
job_returns.job_dir = job_dir
|
|
1252
|
+
return cast(Self, job_returns)
|
|
1219
1253
|
|
|
1220
1254
|
if isinstance(job_returns, (list, tuple)):
|
|
1221
1255
|
# check that a Response object is not given as one of many outputs
|
|
1222
|
-
for
|
|
1223
|
-
if isinstance(
|
|
1256
|
+
for resp in job_returns:
|
|
1257
|
+
if isinstance(resp, Response):
|
|
1224
1258
|
raise ValueError(
|
|
1225
1259
|
"Response cannot be returned in combination with other outputs."
|
|
1226
1260
|
)
|
|
1227
1261
|
|
|
1228
|
-
return cls(output=apply_schema(job_returns, output_schema))
|
|
1262
|
+
return cls(output=apply_schema(job_returns, output_schema), job_dir=job_dir)
|
|
1229
1263
|
|
|
1230
1264
|
|
|
1231
1265
|
def apply_schema(output: Any, schema: type[BaseModel] | None):
|
jobflow/core/maker.py
CHANGED
|
@@ -269,9 +269,7 @@ def recursive_call(
|
|
|
269
269
|
return False
|
|
270
270
|
if name_filter is not None and name_filter not in nested_obj.name:
|
|
271
271
|
return False
|
|
272
|
-
|
|
273
|
-
return False
|
|
274
|
-
return True
|
|
272
|
+
return class_filter is None or isinstance(nested_obj, class_filter)
|
|
275
273
|
|
|
276
274
|
d = obj.as_dict()
|
|
277
275
|
|
jobflow/core/reference.py
CHANGED
jobflow/core/store.py
CHANGED
|
@@ -17,6 +17,7 @@ if typing.TYPE_CHECKING:
|
|
|
17
17
|
from typing import Any, Optional, Union
|
|
18
18
|
|
|
19
19
|
from maggma.core import Sort
|
|
20
|
+
from typing_extensions import Self
|
|
20
21
|
|
|
21
22
|
from jobflow.core.schemas import JobStoreDocument
|
|
22
23
|
|
|
@@ -545,9 +546,9 @@ class JobStore(Store):
|
|
|
545
546
|
)
|
|
546
547
|
|
|
547
548
|
@classmethod
|
|
548
|
-
def from_file(cls
|
|
549
|
+
def from_file(cls, db_file: str | Path, **kwargs) -> Self:
|
|
549
550
|
"""
|
|
550
|
-
Create
|
|
551
|
+
Create a JobStore from a database file.
|
|
551
552
|
|
|
552
553
|
Two options are supported for the database file. The file should be in json or
|
|
553
554
|
yaml format.
|
|
@@ -555,7 +556,7 @@ class JobStore(Store):
|
|
|
555
556
|
The simplest format is a monty dumped version of the store, generated using:
|
|
556
557
|
|
|
557
558
|
>>> from monty.serialization import dumpfn
|
|
558
|
-
>>> dumpfn("job_store.yaml"
|
|
559
|
+
>>> dumpfn(job_store, "job_store.yaml")
|
|
559
560
|
|
|
560
561
|
Alternatively, the file can contain the keys docs_store, additional_stores and
|
|
561
562
|
any other keyword arguments supported by the :obj:`JobStore` constructor. The
|
|
@@ -605,7 +606,7 @@ class JobStore(Store):
|
|
|
605
606
|
return cls.from_dict_spec(store_info, **kwargs)
|
|
606
607
|
|
|
607
608
|
@classmethod
|
|
608
|
-
def from_dict_spec(cls
|
|
609
|
+
def from_dict_spec(cls, spec: dict, **kwargs) -> Self:
|
|
609
610
|
"""
|
|
610
611
|
Create an JobStore from a dict specification.
|
|
611
612
|
|
|
@@ -786,7 +787,7 @@ def _filter_blobs(
|
|
|
786
787
|
|
|
787
788
|
|
|
788
789
|
def _get_blob_info(obj: Any, store_name: str) -> dict[str, str]:
|
|
789
|
-
from jobflow.utils.
|
|
790
|
+
from jobflow.utils.uid import suid
|
|
790
791
|
|
|
791
792
|
class_name = ""
|
|
792
793
|
module_name = ""
|
|
@@ -797,6 +798,6 @@ def _get_blob_info(obj: Any, store_name: str) -> dict[str, str]:
|
|
|
797
798
|
return {
|
|
798
799
|
"@class": class_name,
|
|
799
800
|
"@module": module_name,
|
|
800
|
-
"blob_uuid":
|
|
801
|
+
"blob_uuid": suid(),
|
|
801
802
|
"store": store_name,
|
|
802
803
|
}
|
jobflow/managers/fireworks.py
CHANGED
|
@@ -5,6 +5,8 @@ from __future__ import annotations
|
|
|
5
5
|
import typing
|
|
6
6
|
|
|
7
7
|
from fireworks import FiretaskBase, Firework, FWAction, Workflow, explicit_serialize
|
|
8
|
+
from fireworks.utilities.fw_serializers import recursive_serialize, serialize_fw
|
|
9
|
+
from monty.json import jsanitize
|
|
8
10
|
|
|
9
11
|
if typing.TYPE_CHECKING:
|
|
10
12
|
from collections.abc import Sequence
|
|
@@ -197,3 +199,16 @@ class JobFiretask(FiretaskBase):
|
|
|
197
199
|
defuse_workflow=response.stop_jobflow,
|
|
198
200
|
defuse_children=response.stop_children,
|
|
199
201
|
)
|
|
202
|
+
|
|
203
|
+
@serialize_fw
|
|
204
|
+
@recursive_serialize
|
|
205
|
+
def to_dict(self) -> dict:
|
|
206
|
+
"""
|
|
207
|
+
Serialize version of the FireTask.
|
|
208
|
+
|
|
209
|
+
Overrides the original method to explicitly jsanitize the Job
|
|
210
|
+
to handle cases not properly handled by fireworks, like a Callable.
|
|
211
|
+
"""
|
|
212
|
+
d = dict(self)
|
|
213
|
+
d["job"] = jsanitize(d["job"].as_dict())
|
|
214
|
+
return d
|
jobflow/managers/local.py
CHANGED
|
@@ -16,11 +16,12 @@ logger = logging.getLogger(__name__)
|
|
|
16
16
|
def run_locally(
|
|
17
17
|
flow: jobflow.Flow | jobflow.Job | list[jobflow.Job],
|
|
18
18
|
log: bool = True,
|
|
19
|
-
store: jobflow.JobStore = None,
|
|
19
|
+
store: jobflow.JobStore | None = None,
|
|
20
20
|
create_folders: bool = False,
|
|
21
21
|
root_dir: str | Path | None = None,
|
|
22
22
|
ensure_success: bool = False,
|
|
23
23
|
allow_external_references: bool = False,
|
|
24
|
+
raise_immediately: bool = False,
|
|
24
25
|
) -> dict[str, dict[int, jobflow.Response]]:
|
|
25
26
|
"""
|
|
26
27
|
Run a :obj:`Job` or :obj:`Flow` locally.
|
|
@@ -46,6 +47,10 @@ def run_locally(
|
|
|
46
47
|
allow_external_references : bool
|
|
47
48
|
If False all the references to other outputs should be from other Jobs
|
|
48
49
|
of the Flow.
|
|
50
|
+
raise_immediately : bool
|
|
51
|
+
If True, raise an exception immediately if a job fails. If False, continue
|
|
52
|
+
running the flow and only raise an exception at the end if the flow did not
|
|
53
|
+
finish running successfully.
|
|
49
54
|
|
|
50
55
|
Returns
|
|
51
56
|
-------
|
|
@@ -53,7 +58,7 @@ def run_locally(
|
|
|
53
58
|
The responses of the jobs, as a dict of ``{uuid: {index: response}}``.
|
|
54
59
|
"""
|
|
55
60
|
from collections import defaultdict
|
|
56
|
-
from datetime import datetime
|
|
61
|
+
from datetime import datetime, timezone
|
|
57
62
|
from pathlib import Path
|
|
58
63
|
from random import randint
|
|
59
64
|
|
|
@@ -102,14 +107,19 @@ def run_locally(
|
|
|
102
107
|
errored.add(job.uuid)
|
|
103
108
|
return None, False
|
|
104
109
|
|
|
105
|
-
|
|
110
|
+
if raise_immediately:
|
|
106
111
|
response = job.run(store=store)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
112
|
+
else:
|
|
113
|
+
try:
|
|
114
|
+
response = job.run(store=store)
|
|
115
|
+
except Exception:
|
|
116
|
+
import traceback
|
|
117
|
+
|
|
118
|
+
logger.info(
|
|
119
|
+
f"{job.name} failed with exception:\n{traceback.format_exc()}"
|
|
120
|
+
)
|
|
121
|
+
errored.add(job.uuid)
|
|
122
|
+
return None, False
|
|
113
123
|
|
|
114
124
|
responses[job.uuid][job.index] = response
|
|
115
125
|
|
|
@@ -142,7 +152,7 @@ def run_locally(
|
|
|
142
152
|
|
|
143
153
|
def _get_job_dir():
|
|
144
154
|
if create_folders:
|
|
145
|
-
time_now = datetime.
|
|
155
|
+
time_now = datetime.now(tz=timezone.utc).strftime(SETTINGS.DIRECTORY_FORMAT)
|
|
146
156
|
job_dir = root_dir / f"job_{time_now}-{randint(10000, 99999)}"
|
|
147
157
|
job_dir.mkdir()
|
|
148
158
|
return job_dir
|
|
@@ -155,6 +165,8 @@ def run_locally(
|
|
|
155
165
|
with cd(job_dir):
|
|
156
166
|
response, jobflow_stopped = _run_job(job, parents)
|
|
157
167
|
|
|
168
|
+
if response is not None:
|
|
169
|
+
response.job_dir = job_dir
|
|
158
170
|
encountered_bad_response = encountered_bad_response or response is None
|
|
159
171
|
if jobflow_stopped:
|
|
160
172
|
return False
|
jobflow/settings.py
CHANGED
|
@@ -117,6 +117,11 @@ class JobflowSettings(BaseSettings):
|
|
|
117
117
|
"%Y-%m-%d-%H-%M-%S-%f",
|
|
118
118
|
description="Date stamp format used to create directories",
|
|
119
119
|
)
|
|
120
|
+
|
|
121
|
+
UID_TYPE: str = Field(
|
|
122
|
+
"uuid4", description="Type of unique identifier to use to track jobs. "
|
|
123
|
+
)
|
|
124
|
+
|
|
120
125
|
model_config = SettingsConfigDict(env_prefix="jobflow_")
|
|
121
126
|
|
|
122
127
|
@model_validator(mode="before")
|
jobflow/utils/__init__.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Utility functions for logging, enumerations and manipulating dictionaries."""
|
|
2
|
+
|
|
2
3
|
from jobflow.utils.enum import ValueEnum
|
|
3
4
|
from jobflow.utils.find import (
|
|
4
5
|
contains_flow_or_job,
|
|
@@ -7,4 +8,5 @@ from jobflow.utils.find import (
|
|
|
7
8
|
update_in_dictionary,
|
|
8
9
|
)
|
|
9
10
|
from jobflow.utils.log import initialize_logger
|
|
11
|
+
from jobflow.utils.uid import suid
|
|
10
12
|
from jobflow.utils.uuid import suuid
|
jobflow/utils/uid.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Tools for generating UUIDs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from uuid import UUID
|
|
6
|
+
|
|
7
|
+
try: # pragma: no cover
|
|
8
|
+
from ulid import ULID
|
|
9
|
+
except ImportError: # pragma: no cover
|
|
10
|
+
err_msg = (
|
|
11
|
+
"The ulid package is not installed. "
|
|
12
|
+
"Install it with `pip install jobflow[ulid]` or `pip install python-ulid`."
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
class ULID: # type: ignore[no-redef]
|
|
16
|
+
"""Fake ULID class for raising import error."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, *args, **kwargs):
|
|
19
|
+
raise ImportError(err_msg)
|
|
20
|
+
|
|
21
|
+
def from_str(self, *args, **kwargs):
|
|
22
|
+
"""Raise import error."""
|
|
23
|
+
raise ImportError(err_msg)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def suid(id_type: str | None = None) -> str:
|
|
27
|
+
"""Generate a string UUID (universally unique identifier).
|
|
28
|
+
|
|
29
|
+
Since the timestamp of the IDs are important for sorting,
|
|
30
|
+
only id types that include a timestamp are supported.
|
|
31
|
+
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
uuid_type:
|
|
35
|
+
The type of UUID to generate.
|
|
36
|
+
In the future, ``uuid7`` and ``ulid`` may be supported.
|
|
37
|
+
|
|
38
|
+
Returns
|
|
39
|
+
-------
|
|
40
|
+
str
|
|
41
|
+
A UUID.
|
|
42
|
+
"""
|
|
43
|
+
import uuid
|
|
44
|
+
|
|
45
|
+
from jobflow import SETTINGS
|
|
46
|
+
|
|
47
|
+
if id_type is None:
|
|
48
|
+
id_type = SETTINGS.UID_TYPE
|
|
49
|
+
|
|
50
|
+
funcs = {
|
|
51
|
+
"uuid1": uuid.uuid1,
|
|
52
|
+
"uuid4": uuid.uuid4,
|
|
53
|
+
"ulid": ULID,
|
|
54
|
+
}
|
|
55
|
+
if id_type not in funcs:
|
|
56
|
+
raise ValueError(f"UUID type {id_type} not supported.")
|
|
57
|
+
return str(funcs[id_type]())
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_timestamp_from_uid(uid: str) -> float:
|
|
61
|
+
"""
|
|
62
|
+
Get the time that a UID was generated.
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
uuid
|
|
67
|
+
A UUID.
|
|
68
|
+
|
|
69
|
+
Returns
|
|
70
|
+
-------
|
|
71
|
+
float
|
|
72
|
+
The time stamp from the UUID.
|
|
73
|
+
"""
|
|
74
|
+
id_type = _get_id_type(uid)
|
|
75
|
+
if id_type == "uuid4":
|
|
76
|
+
raise ValueError(
|
|
77
|
+
"UUID4 is randomly generated and not associated with a time stamp."
|
|
78
|
+
)
|
|
79
|
+
funcs = {
|
|
80
|
+
"uuid1": lambda uuid: (UUID(uuid).time - 0x01B21DD213814000) / 1e7,
|
|
81
|
+
"ulid": lambda uuid: ULID.from_str(uuid).timestamp,
|
|
82
|
+
}
|
|
83
|
+
return funcs[id_type](uid)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _get_id_type(uid: str) -> str:
|
|
87
|
+
"""
|
|
88
|
+
Get the type of a UUID.
|
|
89
|
+
|
|
90
|
+
Parameters
|
|
91
|
+
----------
|
|
92
|
+
uuid
|
|
93
|
+
A UUID.
|
|
94
|
+
|
|
95
|
+
Returns
|
|
96
|
+
-------
|
|
97
|
+
str
|
|
98
|
+
The type of the UUID.
|
|
99
|
+
"""
|
|
100
|
+
try:
|
|
101
|
+
version = UUID(uid).version
|
|
102
|
+
return {
|
|
103
|
+
1: "uuid1",
|
|
104
|
+
4: "uuid4",
|
|
105
|
+
}[version]
|
|
106
|
+
except ValueError:
|
|
107
|
+
pass
|
|
108
|
+
|
|
109
|
+
try:
|
|
110
|
+
ULID.from_str(uid)
|
|
111
|
+
return "ulid"
|
|
112
|
+
except ValueError:
|
|
113
|
+
pass
|
|
114
|
+
|
|
115
|
+
raise ValueError(f"ID type for {uid} not recognized.")
|
jobflow/utils/uuid.py
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
"""Tools for generating UUIDs."""
|
|
2
2
|
|
|
3
|
+
from monty.dev import deprecated
|
|
3
4
|
|
|
5
|
+
|
|
6
|
+
@deprecated(
|
|
7
|
+
message="The UUID system will be replace with UID that contains both UUID and ULID."
|
|
8
|
+
)
|
|
4
9
|
def suuid() -> str:
|
|
5
10
|
"""
|
|
6
11
|
Generate a string UUID (universally unique identifier).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: jobflow
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.18
|
|
4
4
|
Summary: jobflow is a library for writing computational workflows
|
|
5
5
|
Author-email: Alex Ganose <a.ganose@imperial.ac.uk>
|
|
6
6
|
License: modified BSD
|
|
@@ -33,49 +33,59 @@ Requires-Dist: pydantic >=2.0.1
|
|
|
33
33
|
Requires-Dist: pydash
|
|
34
34
|
Provides-Extra: dev
|
|
35
35
|
Requires-Dist: pre-commit >=2.12.1 ; extra == 'dev'
|
|
36
|
+
Requires-Dist: typing-extensions ; (python_version < "3.11") and extra == 'dev'
|
|
36
37
|
Provides-Extra: docs
|
|
37
|
-
Requires-Dist: autodoc-pydantic ==2.0
|
|
38
|
-
Requires-Dist: furo ==
|
|
39
|
-
Requires-Dist: ipython ==8.
|
|
40
|
-
Requires-Dist: myst-parser ==
|
|
41
|
-
Requires-Dist: nbsphinx ==0.9.
|
|
38
|
+
Requires-Dist: autodoc-pydantic ==2.1.0 ; extra == 'docs'
|
|
39
|
+
Requires-Dist: furo ==2024.5.6 ; extra == 'docs'
|
|
40
|
+
Requires-Dist: ipython ==8.26.0 ; extra == 'docs'
|
|
41
|
+
Requires-Dist: myst-parser ==3.0.1 ; extra == 'docs'
|
|
42
|
+
Requires-Dist: nbsphinx ==0.9.4 ; extra == 'docs'
|
|
42
43
|
Requires-Dist: sphinx-copybutton ==0.5.2 ; extra == 'docs'
|
|
43
|
-
Requires-Dist: sphinx ==7.
|
|
44
|
+
Requires-Dist: sphinx ==7.4.4 ; extra == 'docs'
|
|
44
45
|
Provides-Extra: fireworks
|
|
45
46
|
Requires-Dist: FireWorks ; extra == 'fireworks'
|
|
46
47
|
Provides-Extra: strict
|
|
47
48
|
Requires-Dist: FireWorks ==2.0.3 ; extra == 'strict'
|
|
48
49
|
Requires-Dist: PyYAML ==6.0.1 ; extra == 'strict'
|
|
49
|
-
Requires-Dist: maggma ==0.
|
|
50
|
-
Requires-Dist: matplotlib ==3.
|
|
51
|
-
Requires-Dist: monty ==
|
|
52
|
-
Requires-Dist: moto ==4.2.
|
|
50
|
+
Requires-Dist: maggma ==0.69.0 ; extra == 'strict'
|
|
51
|
+
Requires-Dist: matplotlib ==3.9.1 ; extra == 'strict'
|
|
52
|
+
Requires-Dist: monty ==2024.7.12 ; extra == 'strict'
|
|
53
|
+
Requires-Dist: moto ==4.2.13 ; extra == 'strict'
|
|
53
54
|
Requires-Dist: networkx ==3.2.1 ; extra == 'strict'
|
|
54
|
-
Requires-Dist: pydantic-settings ==2.
|
|
55
|
-
Requires-Dist: pydantic ==2.
|
|
56
|
-
Requires-Dist: pydash ==
|
|
55
|
+
Requires-Dist: pydantic-settings ==2.3.4 ; extra == 'strict'
|
|
56
|
+
Requires-Dist: pydantic ==2.8.2 ; extra == 'strict'
|
|
57
|
+
Requires-Dist: pydash ==8.0.1 ; extra == 'strict'
|
|
57
58
|
Requires-Dist: pydot ==2.0.0 ; extra == 'strict'
|
|
58
|
-
Requires-Dist:
|
|
59
|
+
Requires-Dist: python-ulid ==2.7.0 ; extra == 'strict'
|
|
60
|
+
Requires-Dist: typing-extensions ==4.12.2 ; extra == 'strict'
|
|
59
61
|
Provides-Extra: tests
|
|
60
|
-
Requires-Dist: moto ==4.2.
|
|
61
|
-
Requires-Dist: pytest-cov ==
|
|
62
|
-
Requires-Dist: pytest ==
|
|
62
|
+
Requires-Dist: moto ==4.2.13 ; extra == 'tests'
|
|
63
|
+
Requires-Dist: pytest-cov ==5.0.0 ; extra == 'tests'
|
|
64
|
+
Requires-Dist: pytest ==8.2.2 ; extra == 'tests'
|
|
65
|
+
Provides-Extra: ulid
|
|
66
|
+
Requires-Dist: python-ulid ; extra == 'ulid'
|
|
63
67
|
Provides-Extra: vis
|
|
64
68
|
Requires-Dist: matplotlib ; extra == 'vis'
|
|
65
69
|
Requires-Dist: pydot ; extra == 'vis'
|
|
66
70
|
|
|
67
|
-
|
|
71
|
+
<div align="center">
|
|
72
|
+
|
|
73
|
+
# 
|
|
68
74
|
|
|
69
75
|
[](https://github.com/materialsproject/jobflow/actions?query=workflow%3Atesting)
|
|
70
76
|
[](https://codecov.io/gh/materialsproject/jobflow/)
|
|
71
77
|
[](https://pypi.org/project/jobflow/)
|
|
72
78
|

|
|
79
|
+
[](https://doi.org/10.21105/joss.05995)
|
|
80
|
+
|
|
81
|
+
</div>
|
|
73
82
|
|
|
74
|
-
[Documentation](https://materialsproject.github.io/jobflow/) | [PyPI](https://pypi.org/project/jobflow/) | [GitHub](https://github.com/materialsproject/jobflow)
|
|
83
|
+
[Documentation](https://materialsproject.github.io/jobflow/) | [PyPI](https://pypi.org/project/jobflow/) | [GitHub](https://github.com/materialsproject/jobflow) | [Paper](https://doi.org/10.21105/joss.05995)
|
|
75
84
|
|
|
76
85
|
Jobflow is a free, open-source library for writing and executing workflows. Complex
|
|
77
86
|
workflows can be defined using simple python functions and executed locally or on
|
|
78
|
-
arbitrary computing resources using the [FireWorks][fireworks]
|
|
87
|
+
arbitrary computing resources using the [jobflow-remote][jfr] or [FireWorks][fireworks]
|
|
88
|
+
workflow managers.
|
|
79
89
|
|
|
80
90
|
Some features that distinguish jobflow are dynamic workflows, easy compositing and
|
|
81
91
|
connecting of workflows, and the ability to store workflow outputs across multiple
|
|
@@ -94,7 +104,7 @@ Some of its features include:
|
|
|
94
104
|
way to build complex workflows.
|
|
95
105
|
- Integration with multiple databases (MongoDB, S3, GridFS, and more) through the
|
|
96
106
|
[Maggma][maggma] package.
|
|
97
|
-
- Support for the [FireWorks][fireworks] workflow management
|
|
107
|
+
- Support for the [jobflow-remote][jfr] and [FireWorks][fireworks] workflow management systems, allowing workflow
|
|
98
108
|
execution on multicore machines or through a queue, on a single machine or multiple
|
|
99
109
|
machines.
|
|
100
110
|
- Support for dynamic workflows — workflows that modify themselves or create new ones
|
|
@@ -176,12 +186,19 @@ We maintain a list of all contributors [here][contributors].
|
|
|
176
186
|
|
|
177
187
|
jobflow is released under a modified BSD license; the full text can be found [here][license].
|
|
178
188
|
|
|
189
|
+
## Citation
|
|
190
|
+
|
|
191
|
+
If you use Jobflow in your work, please cite it as follows:
|
|
192
|
+
|
|
193
|
+
- "Jobflow: Computational Workflows Made Simple", A.S. Rosen, M. Gallant, J. George, J. Riebesell, H. Sahasrabuddhe, J.X. Shen, M. Wen, M.L. Evans, G. Petretto, D. Waroquiers, G.‑M. Rignanese, K.A. Persson, A. Jain, A.M. Ganose, _Journal of Open Source Software_, 9(93), 5995 (2024) DOI: [10.21105/joss.05995](https://doi.org/10.21105/joss.05995)
|
|
194
|
+
|
|
179
195
|
## Acknowledgements
|
|
180
196
|
|
|
181
197
|
Jobflow was designed by Alex Ganose, Anubhav Jain, Gian-Marco Rignanese, David Waroquiers, and Guido Petretto. Alex Ganose implemented the first version of the package. Later versions have benefited from the contributions of several research groups. A full list of contributors is available [here](https://materialsproject.github.io/jobflow/contributors.html).
|
|
182
198
|
|
|
183
199
|
[maggma]: https://materialsproject.github.io/maggma/
|
|
184
200
|
[fireworks]: https://materialsproject.github.io/fireworks/
|
|
201
|
+
[jfr]: https://matgenix.github.io/jobflow-remote
|
|
185
202
|
[help-forum]: https://matsci.org/c/fireworks
|
|
186
203
|
[issues]: https://github.com/materialsproject/jobflow/issues
|
|
187
204
|
[changelog]: https://materialsproject.github.io/jobflow/changelog.html
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
jobflow/__init__.py,sha256=l7o10BaqEQWw5aZziWRg40PsIAgQ4lrlluXs9hIv2mg,570
|
|
2
|
+
jobflow/_version.py,sha256=Ym07PBD7sAmpqVpX8tuzWma3P_Hv6KXbDKXWkw8OwaI,205
|
|
3
|
+
jobflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
jobflow/settings.py,sha256=2tG3FKGTkCDgguwQZrgSt36Ll86nn4ET8ukBr2IN8Ys,5614
|
|
5
|
+
jobflow/core/__init__.py,sha256=3sx5t1Gysejc4c_fPrhvCjPUg0p_384Zko8ms2c_NnY,98
|
|
6
|
+
jobflow/core/flow.py,sha256=SEWCcw8Vs-2jRK_py720nD5SmSHh2vNtOh-LW4vu500,28939
|
|
7
|
+
jobflow/core/job.py,sha256=ZzlWMaTNNM5d3OaE7KDDi2jWyKNfdNNdGnnCOWKQRH8,47595
|
|
8
|
+
jobflow/core/maker.py,sha256=V48-StHQn7S0pFlxm281hQ0TDxQEm7BqQZ6ZuGRrB6A,11134
|
|
9
|
+
jobflow/core/reference.py,sha256=hLuuxQcXwMCEQk-N3jpvxnWfxJZ4OFtxK4Fb-PXMJNk,17090
|
|
10
|
+
jobflow/core/schemas.py,sha256=Oi5-PnZpI8S9jSY7Q4f8H7xUybbRZDXlgugeVewVsrA,968
|
|
11
|
+
jobflow/core/state.py,sha256=IGJTtmpotDKEcgDEnsT5x20ZeyvQT68Mr3teTjkgYnM,709
|
|
12
|
+
jobflow/core/store.py,sha256=PAtAhZTLU1b17OD66rqlo8_JP1p-IzH8c2WJwWd1Q2o,26997
|
|
13
|
+
jobflow/managers/__init__.py,sha256=KkA5cVDe2os2_2aTa8eiB9SnkGLZNybcci-Lo4tbaWM,55
|
|
14
|
+
jobflow/managers/fireworks.py,sha256=xKPcL1BX59jnF6LpyEwcsxvCCShLqep-9UHRyv9FmYE,7146
|
|
15
|
+
jobflow/managers/local.py,sha256=hVTYwwC5oxF_lMMw85xNDECI3wu-rGzJUJkQtPB_TxY,5880
|
|
16
|
+
jobflow/utils/__init__.py,sha256=XrA5IQ2Zm6unFRzgA58Ip9CIPchnlXm8Ei-Ny20f6NA,364
|
|
17
|
+
jobflow/utils/dict_mods.py,sha256=g50aMw-mK3RjXp_hHJBR9xUaWRYXoqqmPTMCPDDluz4,6052
|
|
18
|
+
jobflow/utils/enum.py,sha256=_buiq0NuwyYe82Ajd5-8Pjx9J63XcDDeIvLDS4evM1s,713
|
|
19
|
+
jobflow/utils/find.py,sha256=J57_xQ9boWVVqpbSfTECuZ3TinEXKN24LO4YWIB-7T4,6149
|
|
20
|
+
jobflow/utils/graph.py,sha256=UjwEklPvFz3oTiSY3E_ElYZz21ePY0xLaT7nqWDVfpI,6598
|
|
21
|
+
jobflow/utils/log.py,sha256=tIMpsI4JTlkpxjBZfWqZ0qkEkIxk1-RBasz8JhDcF7E,692
|
|
22
|
+
jobflow/utils/uid.py,sha256=hNkpJ5AYhKd_sPWE_iGPcn6YD_AyizKX_swWksFr-_M,2537
|
|
23
|
+
jobflow/utils/uuid.py,sha256=-cN3NpxiZuQMjl_0wq5h8ZjTYpf5o4-o2QQ5g9E7vQM,406
|
|
24
|
+
jobflow-0.1.18.dist-info/LICENSE,sha256=jUEiENfZNQZh9RE9ixtUWgVkLRD85ScZ6iv1WREf19w,2418
|
|
25
|
+
jobflow-0.1.18.dist-info/METADATA,sha256=w4Umy_IlAZmdPyFZ6srSDj_E8cYUoqp_eZaCvyCMwrM,10003
|
|
26
|
+
jobflow-0.1.18.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
|
|
27
|
+
jobflow-0.1.18.dist-info/top_level.txt,sha256=IanNooU88OupQPDrWnT0rbL3E27P2wEy7Jsfx9_j8zc,8
|
|
28
|
+
jobflow-0.1.18.dist-info/RECORD,,
|
jobflow-0.1.16.dist-info/RECORD
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
jobflow/__init__.py,sha256=l7o10BaqEQWw5aZziWRg40PsIAgQ4lrlluXs9hIv2mg,570
|
|
2
|
-
jobflow/_version.py,sha256=Ym07PBD7sAmpqVpX8tuzWma3P_Hv6KXbDKXWkw8OwaI,205
|
|
3
|
-
jobflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
jobflow/settings.py,sha256=UaLqno9bBoTcpWhzke8y4exZ_byMn0mPFhra5ediG98,5499
|
|
5
|
-
jobflow/core/__init__.py,sha256=3sx5t1Gysejc4c_fPrhvCjPUg0p_384Zko8ms2c_NnY,98
|
|
6
|
-
jobflow/core/flow.py,sha256=5OIRoNCjqt9ThSWM_92P9b88zf5tNzXkQYOwwK2heqI,28914
|
|
7
|
-
jobflow/core/job.py,sha256=ppngMjEcctyBxT1vi-us7dnHL2ILKJuMbo3irvfIBcE,46491
|
|
8
|
-
jobflow/core/maker.py,sha256=DKfFXe91v9rRFtgUrm8FMiEcLArmb6jnK7nysuDdZts,11185
|
|
9
|
-
jobflow/core/reference.py,sha256=sPhV3_q8Ru-8xIoryKY8Y9oQN-1cAGVvnY08tgSAxwk,17090
|
|
10
|
-
jobflow/core/schemas.py,sha256=Oi5-PnZpI8S9jSY7Q4f8H7xUybbRZDXlgugeVewVsrA,968
|
|
11
|
-
jobflow/core/state.py,sha256=IGJTtmpotDKEcgDEnsT5x20ZeyvQT68Mr3teTjkgYnM,709
|
|
12
|
-
jobflow/core/store.py,sha256=KyM6kQms01uAxYOboIuhyaFj2x75xd3-0LOpNdicjcA,26974
|
|
13
|
-
jobflow/managers/__init__.py,sha256=KkA5cVDe2os2_2aTa8eiB9SnkGLZNybcci-Lo4tbaWM,55
|
|
14
|
-
jobflow/managers/fireworks.py,sha256=5IKDkE-dppvbhDWTfJKCMmqvxg50zBgCqm6qUqsVZtc,6654
|
|
15
|
-
jobflow/managers/local.py,sha256=VJ36WANrRtWml1YLrz4nYZLfD3pVL1K0JkpajE589W4,5347
|
|
16
|
-
jobflow/utils/__init__.py,sha256=meuvfuk05U594rx4YB6BoBnoQxBMjCA2hKX3TSfZsB8,328
|
|
17
|
-
jobflow/utils/dict_mods.py,sha256=g50aMw-mK3RjXp_hHJBR9xUaWRYXoqqmPTMCPDDluz4,6052
|
|
18
|
-
jobflow/utils/enum.py,sha256=_buiq0NuwyYe82Ajd5-8Pjx9J63XcDDeIvLDS4evM1s,713
|
|
19
|
-
jobflow/utils/find.py,sha256=J57_xQ9boWVVqpbSfTECuZ3TinEXKN24LO4YWIB-7T4,6149
|
|
20
|
-
jobflow/utils/graph.py,sha256=UjwEklPvFz3oTiSY3E_ElYZz21ePY0xLaT7nqWDVfpI,6598
|
|
21
|
-
jobflow/utils/log.py,sha256=tIMpsI4JTlkpxjBZfWqZ0qkEkIxk1-RBasz8JhDcF7E,692
|
|
22
|
-
jobflow/utils/uuid.py,sha256=lVgo8e8gUB7HLSR0H_9uZH-OPkVBaOT39atAnNKYAaI,268
|
|
23
|
-
jobflow-0.1.16.dist-info/LICENSE,sha256=jUEiENfZNQZh9RE9ixtUWgVkLRD85ScZ6iv1WREf19w,2418
|
|
24
|
-
jobflow-0.1.16.dist-info/METADATA,sha256=5yXZgT67ziHmUVmP0E6XI7JYjXcf0znPoyE8-BUj8F4,9053
|
|
25
|
-
jobflow-0.1.16.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
26
|
-
jobflow-0.1.16.dist-info/top_level.txt,sha256=IanNooU88OupQPDrWnT0rbL3E27P2wEy7Jsfx9_j8zc,8
|
|
27
|
-
jobflow-0.1.16.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|