sirocco 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sirocco/__init__.py +5 -0
- sirocco/core.py +689 -0
- sirocco/parsing/__init__.py +5 -0
- sirocco/parsing/_utils.py +24 -0
- sirocco/parsing/_yaml_data_models.py +361 -0
- sirocco-0.0.0.dev0.dist-info/METADATA +52 -0
- sirocco-0.0.0.dev0.dist-info/RECORD +8 -0
- sirocco-0.0.0.dev0.dist-info/WHEEL +4 -0
sirocco/__init__.py
ADDED
sirocco/core.py
ADDED
|
@@ -0,0 +1,689 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from itertools import chain
|
|
5
|
+
from os.path import expandvars
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
from isoduration import parse_duration
|
|
10
|
+
from isoduration.types import Duration
|
|
11
|
+
|
|
12
|
+
from sirocco.parsing._utils import TimeUtils
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from collections.abc import Generator
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _DataBase:
|
|
19
|
+
def __init__(
|
|
20
|
+
self,
|
|
21
|
+
name: str,
|
|
22
|
+
type: str, # noqa: A002
|
|
23
|
+
src: str,
|
|
24
|
+
lag: list[Duration],
|
|
25
|
+
date: list[datetime],
|
|
26
|
+
arg_option: str | None,
|
|
27
|
+
*,
|
|
28
|
+
available: bool,
|
|
29
|
+
):
|
|
30
|
+
self._name = name
|
|
31
|
+
|
|
32
|
+
self._src = src
|
|
33
|
+
self._path = Path(expandvars(self._src))
|
|
34
|
+
|
|
35
|
+
self._type = type
|
|
36
|
+
if self._type not in ["file", "dir"]:
|
|
37
|
+
msg = f"Data type {self._type!r} not supported. Please use 'file' or 'dir'."
|
|
38
|
+
raise ValueError(msg)
|
|
39
|
+
|
|
40
|
+
if len(lag) > 0 and len(date) > 0:
|
|
41
|
+
msg = "Either 'lag' or 'date' can be nonempty. Not both."
|
|
42
|
+
raise ValueError(msg)
|
|
43
|
+
|
|
44
|
+
# COMMENT I think we should just disallow multiple lags, and enforce the user to write multiple lags
|
|
45
|
+
# I am not sure how this work with icon as it does not need positional arguments
|
|
46
|
+
# or rather how does it work with plugins
|
|
47
|
+
if arg_option is not None and (len(lag) > 1 or len(date) > 1):
|
|
48
|
+
msg = (
|
|
49
|
+
"You cannot give an arg_option when multiple lags and dates are given. "
|
|
50
|
+
"They must be positional arguments, since passing them to one option is ambiguous."
|
|
51
|
+
)
|
|
52
|
+
raise ValueError(msg)
|
|
53
|
+
|
|
54
|
+
self._lag = lag
|
|
55
|
+
self._date = date
|
|
56
|
+
self._arg_option = arg_option
|
|
57
|
+
self._available = available
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def name(self) -> str:
|
|
61
|
+
"""The name of this data instance that is used as identifier."""
|
|
62
|
+
return self._name
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def type(self) -> str:
|
|
66
|
+
"""The data type."""
|
|
67
|
+
return self._type
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def src(self) -> str:
|
|
71
|
+
return self._src
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def path(self) -> Path:
|
|
75
|
+
return self._path
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def lag(self) -> list[Duration]:
|
|
79
|
+
return self._lag
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def date(self) -> list[datetime]:
|
|
83
|
+
return self._date
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def arg_option(self) -> str | None:
|
|
87
|
+
return self._arg_option
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def available(self) -> bool:
|
|
91
|
+
return self._available
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class Data(_DataBase):
|
|
95
|
+
def __init__(
|
|
96
|
+
self,
|
|
97
|
+
name: str,
|
|
98
|
+
type: str, # noqa: A002
|
|
99
|
+
src: str,
|
|
100
|
+
lag: list[Duration],
|
|
101
|
+
date: list[datetime],
|
|
102
|
+
arg_option: str | None,
|
|
103
|
+
*,
|
|
104
|
+
available: bool,
|
|
105
|
+
):
|
|
106
|
+
super().__init__(name, type, src, lag, date, arg_option, available=available)
|
|
107
|
+
self._task: Task | None = None
|
|
108
|
+
|
|
109
|
+
def unroll(self, unrolled_task: UnrolledTask) -> Generator[UnrolledData, None, None]:
|
|
110
|
+
if len(self._date) == 0 and len(self._lag) == 0:
|
|
111
|
+
yield UnrolledData.from_data(self, unrolled_task, unrolled_task.unrolled_date)
|
|
112
|
+
|
|
113
|
+
for lag in self._lag:
|
|
114
|
+
lagged_date = unrolled_task.unrolled_date + lag
|
|
115
|
+
if (
|
|
116
|
+
lagged_date >= unrolled_task.unrolled_cycle.start_date
|
|
117
|
+
and lagged_date <= unrolled_task.unrolled_cycle.end_date
|
|
118
|
+
):
|
|
119
|
+
yield UnrolledData.from_data(self, unrolled_task, lagged_date)
|
|
120
|
+
|
|
121
|
+
for date in self._date:
|
|
122
|
+
yield UnrolledData.from_data(self, unrolled_task, date)
|
|
123
|
+
|
|
124
|
+
def __repr__(self) -> str:
|
|
125
|
+
if self.task is None:
|
|
126
|
+
identifier = f"{self.__class__.__name__} '{self.name}'"
|
|
127
|
+
else:
|
|
128
|
+
identifier = f"{self.__class__.__name__} '{self.name}' attached task '{self.task}'"
|
|
129
|
+
return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def task(self) -> Task | None:
|
|
133
|
+
return self._task
|
|
134
|
+
|
|
135
|
+
@task.setter
|
|
136
|
+
def task(self, task: Task):
|
|
137
|
+
if self._task is not None:
|
|
138
|
+
msg = f"Data {self} was already assigned to task {self._task}. Cannot assign task to task {task}."
|
|
139
|
+
raise ValueError(msg)
|
|
140
|
+
self._task = task
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class UnrolledData(_DataBase):
|
|
144
|
+
"""
|
|
145
|
+
Data that are created during the unrolling of a cycle.
|
|
146
|
+
This class should be only initiated through unrolling a cycle.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
@classmethod
|
|
150
|
+
def from_data(cls, data: Data, unrolled_task: UnrolledTask, unrolled_date: datetime):
|
|
151
|
+
return cls(
|
|
152
|
+
unrolled_task,
|
|
153
|
+
unrolled_date,
|
|
154
|
+
data.name,
|
|
155
|
+
data.type,
|
|
156
|
+
data.src,
|
|
157
|
+
data.lag,
|
|
158
|
+
data.date,
|
|
159
|
+
data.arg_option,
|
|
160
|
+
available=data.available,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def __init__(
|
|
164
|
+
self,
|
|
165
|
+
unrolled_task: UnrolledTask,
|
|
166
|
+
unrolled_date: datetime,
|
|
167
|
+
name: str,
|
|
168
|
+
type: str, # noqa: A002
|
|
169
|
+
src: str,
|
|
170
|
+
lag: list[Duration],
|
|
171
|
+
date: list[datetime],
|
|
172
|
+
arg_option: str | None,
|
|
173
|
+
*,
|
|
174
|
+
available: bool,
|
|
175
|
+
):
|
|
176
|
+
super().__init__(name, type, src, lag, date, arg_option, available=available)
|
|
177
|
+
self._unrolled_task = unrolled_task
|
|
178
|
+
self._unrolled_date = unrolled_date
|
|
179
|
+
|
|
180
|
+
def __repr__(self) -> str:
|
|
181
|
+
if self.unrolled_task is None:
|
|
182
|
+
identifier = f"{self.__class__.__name__} '{self.name}' with date {self.unrolled_date}"
|
|
183
|
+
else:
|
|
184
|
+
identifier = f"{self.__class__.__name__} '{self.name}' with date {self.unrolled_date} attached to task {self.unrolled_task}"
|
|
185
|
+
return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
|
|
186
|
+
|
|
187
|
+
@property
|
|
188
|
+
def unrolled_date(self) -> datetime:
|
|
189
|
+
return self._unrolled_date
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def unrolled_task(self) -> UnrolledTask:
|
|
193
|
+
return self._unrolled_task
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class _DependencyBase:
|
|
197
|
+
def __init__(self, depend_on_task_name: str, lag: list[Duration], date: list[datetime], cycle_name: str | None):
|
|
198
|
+
self._depend_on_task_name = depend_on_task_name
|
|
199
|
+
if len(lag) > 0 and len(date) > 0:
|
|
200
|
+
msg = "Only one key 'lag' or 'date' is allowed. Not both."
|
|
201
|
+
raise ValueError(msg)
|
|
202
|
+
|
|
203
|
+
self._lag = lag
|
|
204
|
+
self._date = date
|
|
205
|
+
self._cycle_name = cycle_name
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def depend_on_task_name(self) -> str:
|
|
209
|
+
return self._depend_on_task_name
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def lag(self) -> list[Duration]:
|
|
213
|
+
return self._lag
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def date(self) -> list[datetime]:
|
|
217
|
+
return self._date
|
|
218
|
+
|
|
219
|
+
@property
|
|
220
|
+
def cycle_name(self) -> str | None:
|
|
221
|
+
return self._cycle_name
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class Dependency(_DependencyBase):
|
|
225
|
+
def __init__(self, depend_on_task_name: str, lag: list[Duration], date: list[datetime], cycle_name: str | None):
|
|
226
|
+
super().__init__(depend_on_task_name, lag, date, cycle_name)
|
|
227
|
+
self._task: Task | None = None
|
|
228
|
+
|
|
229
|
+
def unroll(self, unrolled_task: UnrolledTask) -> Generator[UnrolledDependency, None, None]:
|
|
230
|
+
if len(self._date) == 0 and len(self._lag) == 0:
|
|
231
|
+
yield UnrolledDependency.from_dependency(self, unrolled_task, unrolled_task.unrolled_date)
|
|
232
|
+
|
|
233
|
+
for lag in self._lag:
|
|
234
|
+
lagged_date = unrolled_task.unrolled_date + lag
|
|
235
|
+
if (
|
|
236
|
+
lagged_date >= unrolled_task.unrolled_cycle.start_date
|
|
237
|
+
and lagged_date <= unrolled_task.unrolled_cycle.end_date
|
|
238
|
+
):
|
|
239
|
+
yield UnrolledDependency.from_dependency(self, unrolled_task, lagged_date)
|
|
240
|
+
|
|
241
|
+
for date in self._date:
|
|
242
|
+
yield UnrolledDependency.from_dependency(self, unrolled_task, date)
|
|
243
|
+
|
|
244
|
+
@property
|
|
245
|
+
def task(self) -> Task | None:
|
|
246
|
+
return self._task
|
|
247
|
+
|
|
248
|
+
@task.setter
|
|
249
|
+
def task(self, task: Task):
|
|
250
|
+
if self.task is not None:
|
|
251
|
+
msg = f"Dependency was already assigned to task {self.task}. Cannot assign to task {task}."
|
|
252
|
+
raise ValueError(msg)
|
|
253
|
+
self._task = task
|
|
254
|
+
|
|
255
|
+
def __repr__(self) -> str:
|
|
256
|
+
if self._cycle_name is None:
|
|
257
|
+
identifier = f"{self.__class__.__name__} on task '{self.depend_on_task_name}' attached to task {self.task}"
|
|
258
|
+
else:
|
|
259
|
+
identifier = f"{self.__class__.__name__} on task '{self.depend_on_task_name}' in cycle '{self.cycle_name}' attached to task {self.task}"
|
|
260
|
+
return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class UnrolledDependency(_DependencyBase):
|
|
264
|
+
"""
|
|
265
|
+
This class should be only initiated through unrolling a cycle.
|
|
266
|
+
"""
|
|
267
|
+
|
|
268
|
+
@classmethod
|
|
269
|
+
def from_dependency(cls, depend: Dependency, unrolled_task: UnrolledTask, unrolled_date: datetime):
|
|
270
|
+
return cls(unrolled_task, unrolled_date, depend.depend_on_task_name, depend.lag, depend.date, depend.cycle_name)
|
|
271
|
+
|
|
272
|
+
def __init__(
|
|
273
|
+
self,
|
|
274
|
+
unrolled_task: UnrolledTask,
|
|
275
|
+
unrolled_date: datetime,
|
|
276
|
+
depend_on_task_name: str,
|
|
277
|
+
lag: list[Duration],
|
|
278
|
+
date: list[datetime],
|
|
279
|
+
cycle_name: str | None,
|
|
280
|
+
):
|
|
281
|
+
super().__init__(depend_on_task_name, lag, date, cycle_name)
|
|
282
|
+
self._unrolled_task = unrolled_task
|
|
283
|
+
self._unrolled_date = unrolled_date
|
|
284
|
+
|
|
285
|
+
def __repr__(self) -> str:
|
|
286
|
+
if self._cycle_name is None:
|
|
287
|
+
identifier = (
|
|
288
|
+
f"{self.__class__.__name__} on task '{self.depend_on_task_name}' with date {self.unrolled_date}"
|
|
289
|
+
)
|
|
290
|
+
else:
|
|
291
|
+
identifier = f"{self.__class__.__name__} on task '{self.depend_on_task_name}' in cycle '{self.cycle_name}' with date {self.unrolled_date}"
|
|
292
|
+
return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
|
|
293
|
+
|
|
294
|
+
@property
|
|
295
|
+
def depend_on_task(self) -> UnrolledTask:
|
|
296
|
+
"""
|
|
297
|
+
throws error if not found
|
|
298
|
+
"""
|
|
299
|
+
# for now we only support looking in the same cycle
|
|
300
|
+
workflow = self._unrolled_task.unrolled_cycle.workflow
|
|
301
|
+
if self._cycle_name is None:
|
|
302
|
+
tasks_to_search = [
|
|
303
|
+
cycle.unrolled_tasks for cycle in workflow.unrolled_cycles if cycle.unrolled_date == self._unrolled_date
|
|
304
|
+
]
|
|
305
|
+
potential_tasks = [
|
|
306
|
+
task for task in chain.from_iterable(tasks_to_search) if task.name == self._depend_on_task_name
|
|
307
|
+
]
|
|
308
|
+
if len(potential_tasks) > 1:
|
|
309
|
+
msg = (
|
|
310
|
+
f"Found multiple instances of the task '{self._depend_on_task_name}' with date {self._unrolled_date}"
|
|
311
|
+
" for dependency of the task {self._unrolled_task}. Please specify a cycle name."
|
|
312
|
+
)
|
|
313
|
+
raise ValueError(msg)
|
|
314
|
+
if len(potential_tasks) == 0:
|
|
315
|
+
msg = (
|
|
316
|
+
f"Found no instance of the task '{self._depend_on_task_name}' with date {self._unrolled_date}"
|
|
317
|
+
f" for dependency attached to task {self._unrolled_task}."
|
|
318
|
+
)
|
|
319
|
+
raise ValueError(msg)
|
|
320
|
+
return potential_tasks[0]
|
|
321
|
+
|
|
322
|
+
cycle = workflow.unrolled_cycles_map[(self._cycle_name, self._unrolled_date)]
|
|
323
|
+
return cycle.unrolled_tasks_map[self._depend_on_task_name]
|
|
324
|
+
|
|
325
|
+
@property
|
|
326
|
+
def unrolled_task(self) -> UnrolledTask:
|
|
327
|
+
return self._unrolled_task
|
|
328
|
+
|
|
329
|
+
@property
|
|
330
|
+
def unrolled_date(self) -> datetime:
|
|
331
|
+
return self._unrolled_date
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
class _TaskBase:
|
|
335
|
+
"""
|
|
336
|
+
Common class for Task and UnrolledTask to reduce code duplications
|
|
337
|
+
"""
|
|
338
|
+
|
|
339
|
+
def __init__(
|
|
340
|
+
self,
|
|
341
|
+
name: str,
|
|
342
|
+
command: str,
|
|
343
|
+
inputs: list[Data],
|
|
344
|
+
outputs: list[Data],
|
|
345
|
+
depends: list[Dependency],
|
|
346
|
+
command_option: str | None,
|
|
347
|
+
):
|
|
348
|
+
self._name = name
|
|
349
|
+
self._command = expandvars(command)
|
|
350
|
+
self._inputs = inputs
|
|
351
|
+
self._outputs = outputs
|
|
352
|
+
self._depends = depends
|
|
353
|
+
self._command_option = command_option
|
|
354
|
+
|
|
355
|
+
@property
|
|
356
|
+
def name(self) -> str:
|
|
357
|
+
return self._name
|
|
358
|
+
|
|
359
|
+
@property
|
|
360
|
+
def command(self) -> str:
|
|
361
|
+
return self._command
|
|
362
|
+
|
|
363
|
+
@property
|
|
364
|
+
def inputs(self) -> list[Data]:
|
|
365
|
+
return self._inputs
|
|
366
|
+
|
|
367
|
+
@property
|
|
368
|
+
def outputs(self) -> list[Data]:
|
|
369
|
+
return self._outputs
|
|
370
|
+
|
|
371
|
+
@property
|
|
372
|
+
def command_option(self) -> str | None:
|
|
373
|
+
return self._command_option
|
|
374
|
+
|
|
375
|
+
@property
|
|
376
|
+
def depends(self) -> list[Dependency]:
|
|
377
|
+
return self._depends
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
class Task(_TaskBase):
|
|
381
|
+
"""A task that is created during the unrolling of a cycle."""
|
|
382
|
+
|
|
383
|
+
def __init__(
|
|
384
|
+
self,
|
|
385
|
+
name: str,
|
|
386
|
+
command: str,
|
|
387
|
+
inputs: list[Data],
|
|
388
|
+
outputs: list[Data],
|
|
389
|
+
depends: list[Dependency],
|
|
390
|
+
command_option: str | None,
|
|
391
|
+
):
|
|
392
|
+
super().__init__(name, command, inputs, outputs, depends, command_option)
|
|
393
|
+
for input_ in inputs:
|
|
394
|
+
input_.task = self
|
|
395
|
+
for output in outputs:
|
|
396
|
+
output.task = self
|
|
397
|
+
for depend in depends:
|
|
398
|
+
depend.task = self
|
|
399
|
+
self._cycle: Cycle | None = None
|
|
400
|
+
|
|
401
|
+
def __repr__(self) -> str:
|
|
402
|
+
identifier = f"Task '{self.name}'"
|
|
403
|
+
if self.cycle is not None:
|
|
404
|
+
identifier += f" in cycle {self.cycle.name}"
|
|
405
|
+
return super().__repr__().replace("Task", identifier)
|
|
406
|
+
|
|
407
|
+
def unroll(self, unrolled_cycle: UnrolledCycle) -> Generator[tuple[str, UnrolledTask], None, None]:
|
|
408
|
+
# an unrolled task is just one task, since the date is determined
|
|
409
|
+
# by the cycle, but we keep the pattern for consistency
|
|
410
|
+
unrolled_task = UnrolledTask.from_task(self, unrolled_cycle)
|
|
411
|
+
yield unrolled_task.name, unrolled_task
|
|
412
|
+
|
|
413
|
+
@property
|
|
414
|
+
def cycle(self) -> Cycle | None:
|
|
415
|
+
return self._cycle
|
|
416
|
+
|
|
417
|
+
@cycle.setter
|
|
418
|
+
def cycle(self, cycle: Cycle):
|
|
419
|
+
if self._cycle is not None:
|
|
420
|
+
msg = f"Task {self} was already assigned to cycle {self._cycle}. Cannot assign task to cycle {cycle}."
|
|
421
|
+
raise ValueError(msg)
|
|
422
|
+
self._cycle = cycle
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
class UnrolledTask(_TaskBase):
|
|
426
|
+
"""
|
|
427
|
+
This class should be only initiated through unrolling a cycle.
|
|
428
|
+
"""
|
|
429
|
+
|
|
430
|
+
@classmethod
|
|
431
|
+
def from_task(cls, task: Task, unrolled_cycle: UnrolledCycle):
|
|
432
|
+
return cls(
|
|
433
|
+
unrolled_cycle, task.name, task.command, task.inputs, task.outputs, task.depends, task.command_option
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
def __init__(
|
|
437
|
+
self,
|
|
438
|
+
unrolled_cycle: UnrolledCycle,
|
|
439
|
+
name: str,
|
|
440
|
+
command: str,
|
|
441
|
+
inputs: list[Data],
|
|
442
|
+
outputs: list[Data],
|
|
443
|
+
depends: list[Dependency],
|
|
444
|
+
command_option: str | None,
|
|
445
|
+
):
|
|
446
|
+
super().__init__(name, command, inputs, outputs, depends, command_option)
|
|
447
|
+
self._unrolled_cycle = unrolled_cycle
|
|
448
|
+
self._unrolled_inputs = list(self.unroll_inputs())
|
|
449
|
+
self._unrolled_outputs = list(self.unroll_outputs())
|
|
450
|
+
self._unrolled_depends = list(self.unroll_depends())
|
|
451
|
+
|
|
452
|
+
def __repr__(self) -> str:
|
|
453
|
+
if self.unrolled_cycle is None:
|
|
454
|
+
identifier = f"Task '{self.name}' with date {self.unrolled_date}"
|
|
455
|
+
else:
|
|
456
|
+
identifier = f"Task '{self.name}' in cycle {self.unrolled_cycle.name} with date {self.unrolled_date}"
|
|
457
|
+
return super().__repr__().replace("Task", identifier)
|
|
458
|
+
|
|
459
|
+
def unroll_inputs(self) -> Generator[UnrolledData, None, None]:
|
|
460
|
+
"""
|
|
461
|
+
Outputs the inputs together with a unique identifier within the task
|
|
462
|
+
"""
|
|
463
|
+
for input_ in self._inputs:
|
|
464
|
+
yield from input_.unroll(self)
|
|
465
|
+
|
|
466
|
+
def unroll_outputs(self) -> Generator[UnrolledData, None, None]:
|
|
467
|
+
for output in self._outputs:
|
|
468
|
+
yield from output.unroll(self)
|
|
469
|
+
|
|
470
|
+
def unroll_depends(self) -> Generator[UnrolledDependency, None, None]:
|
|
471
|
+
for depend in self._depends:
|
|
472
|
+
yield from depend.unroll(self)
|
|
473
|
+
|
|
474
|
+
@property
|
|
475
|
+
def unrolled_inputs(self) -> list[UnrolledData]:
|
|
476
|
+
return self._unrolled_inputs
|
|
477
|
+
|
|
478
|
+
@property
|
|
479
|
+
def unrolled_outputs(self) -> list[UnrolledData]:
|
|
480
|
+
return self._unrolled_outputs
|
|
481
|
+
|
|
482
|
+
@property
|
|
483
|
+
def unrolled_depends(self) -> list[UnrolledDependency]:
|
|
484
|
+
return self._unrolled_depends
|
|
485
|
+
|
|
486
|
+
@property
|
|
487
|
+
def unrolled_date(self) -> datetime:
|
|
488
|
+
return self._unrolled_cycle.unrolled_date
|
|
489
|
+
|
|
490
|
+
@property
|
|
491
|
+
def unrolled_cycle(self) -> UnrolledCycle:
|
|
492
|
+
return self._unrolled_cycle
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
class _CycleBase:
|
|
496
|
+
def __init__(
|
|
497
|
+
self,
|
|
498
|
+
name: str,
|
|
499
|
+
tasks: list[Task],
|
|
500
|
+
start_date: str | datetime,
|
|
501
|
+
end_date: str | datetime,
|
|
502
|
+
period: str | Duration | None = None,
|
|
503
|
+
):
|
|
504
|
+
self._name = name
|
|
505
|
+
self._tasks = tasks
|
|
506
|
+
self._start_date = start_date if isinstance(start_date, datetime) else datetime.fromisoformat(start_date)
|
|
507
|
+
self._end_date = end_date if isinstance(end_date, datetime) else datetime.fromisoformat(end_date)
|
|
508
|
+
|
|
509
|
+
if self._start_date > self._end_date:
|
|
510
|
+
msg = "For cycle {self} the start_date {start_date} lies after given end_date {end_date}."
|
|
511
|
+
raise ValueError(msg)
|
|
512
|
+
|
|
513
|
+
self._period = period if period is None or isinstance(period, Duration) else parse_duration(period)
|
|
514
|
+
if self._period is not None and TimeUtils.duration_is_less_equal_zero(self._period):
|
|
515
|
+
msg = f"For cycle {self} the period {period} is negative or zero."
|
|
516
|
+
raise ValueError(msg)
|
|
517
|
+
|
|
518
|
+
task_names = set()
|
|
519
|
+
for task in self._tasks:
|
|
520
|
+
if task.name in task_names:
|
|
521
|
+
msg = f"List of tasks does contain tasks with duplicate names. The task name '{task.name}' has been found twice."
|
|
522
|
+
raise ValueError(msg)
|
|
523
|
+
task_names.add(task.name)
|
|
524
|
+
|
|
525
|
+
@property
|
|
526
|
+
def name(self) -> str:
|
|
527
|
+
return self._name
|
|
528
|
+
|
|
529
|
+
@property
|
|
530
|
+
def start_date(self) -> datetime:
|
|
531
|
+
return self._start_date
|
|
532
|
+
|
|
533
|
+
@property
|
|
534
|
+
def end_date(self) -> datetime:
|
|
535
|
+
return self._end_date
|
|
536
|
+
|
|
537
|
+
@property
|
|
538
|
+
def period(self) -> Duration | None:
|
|
539
|
+
return self._period
|
|
540
|
+
|
|
541
|
+
@property
|
|
542
|
+
def tasks(self) -> list[Task]:
|
|
543
|
+
return self._tasks
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
class Cycle(_CycleBase):
|
|
547
|
+
def __init__(
|
|
548
|
+
self,
|
|
549
|
+
name: str,
|
|
550
|
+
tasks: list[Task],
|
|
551
|
+
start_date: str | datetime,
|
|
552
|
+
end_date: str | datetime,
|
|
553
|
+
period: str | Duration | None,
|
|
554
|
+
):
|
|
555
|
+
super().__init__(name, tasks, start_date, end_date, period)
|
|
556
|
+
for task in self._tasks:
|
|
557
|
+
task.cycle = self
|
|
558
|
+
|
|
559
|
+
self._workflow: Workflow | None = None
|
|
560
|
+
|
|
561
|
+
def __repr__(self) -> str:
|
|
562
|
+
if self.workflow is None:
|
|
563
|
+
identifier = f"Cycle '{self.name}'"
|
|
564
|
+
else:
|
|
565
|
+
identifier = f"Cycle '{self.name}' in workflow {self.workflow.name}"
|
|
566
|
+
return super().__repr__().replace("Cycle", identifier)
|
|
567
|
+
|
|
568
|
+
def unroll(self) -> Generator[tuple[str, datetime, UnrolledCycle], None, None]:
|
|
569
|
+
if self._workflow is None:
|
|
570
|
+
msg = f"Cannot unroll cycle {self} because it was not attached to a workflow before."
|
|
571
|
+
raise ValueError(msg)
|
|
572
|
+
current_date = self._start_date
|
|
573
|
+
while current_date <= self._end_date:
|
|
574
|
+
unrolled_cycle = UnrolledCycle.from_cycle(self, current_date, self._workflow)
|
|
575
|
+
yield unrolled_cycle.name, unrolled_cycle.unrolled_date, unrolled_cycle
|
|
576
|
+
if self._period is None:
|
|
577
|
+
break
|
|
578
|
+
else:
|
|
579
|
+
current_date += self._period
|
|
580
|
+
|
|
581
|
+
@property
|
|
582
|
+
def workflow(self) -> Workflow | None:
|
|
583
|
+
return self._workflow
|
|
584
|
+
|
|
585
|
+
@workflow.setter
|
|
586
|
+
def workflow(self, workflow: Workflow):
|
|
587
|
+
if self._workflow is not None:
|
|
588
|
+
msg = f"Cycle {self} was already assigned to workflow {self._workflow}. Cannot assign cycle to workflow {workflow}."
|
|
589
|
+
raise ValueError(msg)
|
|
590
|
+
self._workflow = workflow
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
class UnrolledCycle(_CycleBase):
|
|
594
|
+
"""
|
|
595
|
+
This class should be only initiated through unrolling a cycle.
|
|
596
|
+
"""
|
|
597
|
+
|
|
598
|
+
@classmethod
|
|
599
|
+
def from_cycle(cls, cycle: Cycle, unrolled_date: datetime, workflow: Workflow):
|
|
600
|
+
return cls(unrolled_date, cycle.name, cycle.tasks, cycle.start_date, cycle.end_date, cycle.period, workflow)
|
|
601
|
+
|
|
602
|
+
def __init__(
|
|
603
|
+
self,
|
|
604
|
+
unrolled_date: datetime,
|
|
605
|
+
name: str,
|
|
606
|
+
tasks: list[Task],
|
|
607
|
+
start_date: str | datetime,
|
|
608
|
+
end_date: str | datetime,
|
|
609
|
+
period: str | Duration | None,
|
|
610
|
+
workflow: Workflow,
|
|
611
|
+
):
|
|
612
|
+
super().__init__(name, tasks, start_date, end_date, period)
|
|
613
|
+
|
|
614
|
+
self._unrolled_date = unrolled_date
|
|
615
|
+
|
|
616
|
+
self._unrolled_tasks_map = dict(self.unroll_tasks())
|
|
617
|
+
self._workflow = workflow
|
|
618
|
+
|
|
619
|
+
def __repr__(self) -> str:
|
|
620
|
+
if self.workflow is None:
|
|
621
|
+
identifier = f"UnrolledCycle '{self.name}' with date {self.unrolled_date}"
|
|
622
|
+
else:
|
|
623
|
+
identifier = f"UnrolledCycle '{self.name}' in workflow {self.workflow.name} with date {self.unrolled_date}"
|
|
624
|
+
return super().__repr__().replace("UnrolledCycle", identifier)
|
|
625
|
+
|
|
626
|
+
def unroll_tasks(self) -> Generator[tuple[str, UnrolledTask], None, None]:
|
|
627
|
+
for task in self._tasks:
|
|
628
|
+
yield from task.unroll(self)
|
|
629
|
+
|
|
630
|
+
@property
|
|
631
|
+
def unrolled_tasks(self) -> list[UnrolledTask]:
|
|
632
|
+
return list(self._unrolled_tasks_map.values())
|
|
633
|
+
|
|
634
|
+
@property
|
|
635
|
+
def unrolled_tasks_map(self) -> dict[str, UnrolledTask]:
|
|
636
|
+
return self._unrolled_tasks_map
|
|
637
|
+
|
|
638
|
+
@property
|
|
639
|
+
def unrolled_date(self) -> datetime:
|
|
640
|
+
return self._unrolled_date
|
|
641
|
+
|
|
642
|
+
@property
|
|
643
|
+
def workflow(self) -> Workflow:
|
|
644
|
+
return self._workflow
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
class Workflow:
|
|
648
|
+
def __init__(self, name: str, cycles: list[Cycle]):
|
|
649
|
+
self._name = name
|
|
650
|
+
self._cycles = cycles
|
|
651
|
+
for cycle in self._cycles:
|
|
652
|
+
cycle.workflow = self
|
|
653
|
+
self._validate_cycles()
|
|
654
|
+
self._unrolled_cycles_map = {(name, date): cycle for name, date, cycle in self.unroll_cycles()}
|
|
655
|
+
|
|
656
|
+
unrolled_outputs = []
|
|
657
|
+
for unrolled_cycle in self.unrolled_cycles:
|
|
658
|
+
for unrolled_task in unrolled_cycle.unrolled_tasks:
|
|
659
|
+
unrolled_outputs.extend(unrolled_task.unrolled_outputs)
|
|
660
|
+
self._unrolled_outputs = unrolled_outputs
|
|
661
|
+
|
|
662
|
+
def _validate_cycles(self):
|
|
663
|
+
"""Checks if the defined workflow is correctly referencing key names."""
|
|
664
|
+
cycle_names = set()
|
|
665
|
+
for cycle in self._cycles:
|
|
666
|
+
if cycle.name in cycle_names:
|
|
667
|
+
msg = f"List of cycles does contain cycles with duplicate names. The cycle name '{cycle.name}' has been found twice."
|
|
668
|
+
raise ValueError(msg)
|
|
669
|
+
cycle_names.add(cycle.name)
|
|
670
|
+
|
|
671
|
+
def unroll_cycles(self) -> Generator[tuple[str, datetime, UnrolledCycle], None, None]:
|
|
672
|
+
for cycle in self._cycles:
|
|
673
|
+
yield from cycle.unroll()
|
|
674
|
+
|
|
675
|
+
@property
|
|
676
|
+
def name(self) -> str:
|
|
677
|
+
return self._name
|
|
678
|
+
|
|
679
|
+
@property
|
|
680
|
+
def cycles(self) -> list[Cycle]:
|
|
681
|
+
return self._cycles
|
|
682
|
+
|
|
683
|
+
@property
|
|
684
|
+
def unrolled_cycles(self) -> list[UnrolledCycle]:
|
|
685
|
+
return list(self._unrolled_cycles_map.values())
|
|
686
|
+
|
|
687
|
+
@property
|
|
688
|
+
def unrolled_cycles_map(self) -> dict[tuple[str, datetime], UnrolledCycle]:
|
|
689
|
+
return self._unrolled_cycles_map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from isoduration.types import Duration
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class TimeUtils:
|
|
5
|
+
@staticmethod
|
|
6
|
+
def duration_is_less_equal_zero(duration: Duration) -> bool:
|
|
7
|
+
if (
|
|
8
|
+
duration.date.years == 0
|
|
9
|
+
and duration.date.months == 0
|
|
10
|
+
and duration.date.days == 0
|
|
11
|
+
and duration.time.hours == 0
|
|
12
|
+
and duration.time.minutes == 0
|
|
13
|
+
and duration.time.seconds == 0
|
|
14
|
+
or (
|
|
15
|
+
duration.date.years < 0
|
|
16
|
+
or duration.date.months < 0
|
|
17
|
+
or duration.date.days < 0
|
|
18
|
+
or duration.time.hours < 0
|
|
19
|
+
or duration.time.minutes < 0
|
|
20
|
+
or duration.time.seconds < 0
|
|
21
|
+
)
|
|
22
|
+
):
|
|
23
|
+
return True
|
|
24
|
+
return False
|
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from os.path import expandvars
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from isoduration import parse_duration
|
|
10
|
+
from isoduration.types import Duration # pydantic needs type # noqa: TCH002
|
|
11
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
12
|
+
|
|
13
|
+
from sirocco import core
|
|
14
|
+
|
|
15
|
+
from ._utils import TimeUtils
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _NamedBaseModel(BaseModel):
|
|
19
|
+
"""Base class for all classes with a key that specifies their name.
|
|
20
|
+
|
|
21
|
+
For example:
|
|
22
|
+
|
|
23
|
+
.. yaml
|
|
24
|
+
|
|
25
|
+
- property_name:
|
|
26
|
+
property: true
|
|
27
|
+
|
|
28
|
+
When parsing with this as parent class it is converted to
|
|
29
|
+
`{"name": "propery_name", "property": True}`.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
name: str
|
|
33
|
+
|
|
34
|
+
def __init__(self, /, **data):
|
|
35
|
+
name_and_spec = {}
|
|
36
|
+
# - my_name:
|
|
37
|
+
# ...
|
|
38
|
+
if len(data) != 1:
|
|
39
|
+
msg = f"Expected dict with one element of the form {{'name': specification}} but got {data}."
|
|
40
|
+
raise ValueError(msg)
|
|
41
|
+
name_and_spec["name"] = next(iter(data.keys()))
|
|
42
|
+
# if no specification specified e.g. "- my_name:"
|
|
43
|
+
if (spec := next(iter(data.values()))) is not None:
|
|
44
|
+
name_and_spec.update(spec)
|
|
45
|
+
|
|
46
|
+
super().__init__(**name_and_spec)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class _LagDateBaseModel(BaseModel):
|
|
50
|
+
"""Base class for all classes containg a list of dates or time lags."""
|
|
51
|
+
|
|
52
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
53
|
+
date: list[datetime] = [] # this is safe in pydantic
|
|
54
|
+
lag: list[Duration] = [] # this is safe in pydantic
|
|
55
|
+
|
|
56
|
+
@model_validator(mode="before")
|
|
57
|
+
@classmethod
|
|
58
|
+
def check_lag_xor_date_is_set(cls, data: Any) -> Any:
|
|
59
|
+
if "lag" in data and "date" in data:
|
|
60
|
+
msg = "Only one key 'lag' or 'date' is allowed. Not both."
|
|
61
|
+
raise ValueError(msg)
|
|
62
|
+
return data
|
|
63
|
+
|
|
64
|
+
@field_validator("lag", mode="before")
|
|
65
|
+
@classmethod
|
|
66
|
+
def convert_durations(cls, value) -> list[Duration]:
|
|
67
|
+
if value is None:
|
|
68
|
+
return []
|
|
69
|
+
values = value if isinstance(value, list) else [value]
|
|
70
|
+
return [parse_duration(value) for value in values]
|
|
71
|
+
|
|
72
|
+
@field_validator("date", mode="before")
|
|
73
|
+
@classmethod
|
|
74
|
+
def convert_datetimes(cls, value) -> list[datetime]:
|
|
75
|
+
if value is None:
|
|
76
|
+
return []
|
|
77
|
+
values = value if isinstance(value, list) else [value]
|
|
78
|
+
return [datetime.fromisoformat(value) for value in values]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class ConfigTask(_NamedBaseModel):
|
|
82
|
+
"""
|
|
83
|
+
To create an instance of a task defined in a workflow file
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
command: str
|
|
87
|
+
command_option: str | None = None
|
|
88
|
+
host: str | None = None
|
|
89
|
+
account: str | None = None
|
|
90
|
+
plugin: str | None = None
|
|
91
|
+
config: str | None = None
|
|
92
|
+
uenv: dict | None = None
|
|
93
|
+
nodes: int | None = None
|
|
94
|
+
walltime: str | None = None
|
|
95
|
+
src: str | None = None
|
|
96
|
+
conda_env: str | None = None
|
|
97
|
+
|
|
98
|
+
def __init__(self, /, **data):
|
|
99
|
+
# We have to treat root special as it does not typically define a command
|
|
100
|
+
if "ROOT" in data and "command" not in data["ROOT"]:
|
|
101
|
+
data["ROOT"]["command"] = "ROOT_PLACEHOLDER"
|
|
102
|
+
super().__init__(**data)
|
|
103
|
+
|
|
104
|
+
@field_validator("command")
|
|
105
|
+
@classmethod
|
|
106
|
+
def expand_env_vars(cls, value: str) -> str:
|
|
107
|
+
"""Expands any environment variables in the value"""
|
|
108
|
+
return expandvars(value)
|
|
109
|
+
|
|
110
|
+
@field_validator("walltime")
|
|
111
|
+
@classmethod
|
|
112
|
+
def convert_to_struct_time(cls, value: str | None) -> time.struct_time | None:
|
|
113
|
+
"""Converts a string of form "%H:%M:%S" to a time.time_struct"""
|
|
114
|
+
return None if value is None else time.strptime(value, "%H:%M:%S")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class _DataBaseModel(_NamedBaseModel):
|
|
118
|
+
"""
|
|
119
|
+
To create an instance of a data defined in a workflow file.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
type: str
|
|
123
|
+
src: str
|
|
124
|
+
format: str | None = None
|
|
125
|
+
|
|
126
|
+
@field_validator("type")
|
|
127
|
+
@classmethod
|
|
128
|
+
def is_file_or_dir(cls, value: str) -> str:
|
|
129
|
+
"""."""
|
|
130
|
+
if value not in ["file", "dir"]:
|
|
131
|
+
msg = "Must be one of 'file' or 'dir'."
|
|
132
|
+
raise ValueError(msg)
|
|
133
|
+
return value
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def available(self) -> bool:
|
|
137
|
+
return isinstance(self, ConfigAvailableData)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class ConfigAvailableData(_DataBaseModel):
|
|
141
|
+
pass
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class ConfigGeneratedData(_DataBaseModel):
|
|
145
|
+
pass
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class ConfigData(BaseModel):
|
|
149
|
+
available: list[ConfigAvailableData]
|
|
150
|
+
generated: list[ConfigGeneratedData]
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class ConfigCycleTaskDepend(_NamedBaseModel, _LagDateBaseModel):
|
|
154
|
+
"""
|
|
155
|
+
To create an instance of a input or output in a task in a cycle defined in a workflow file.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
name: str # name of the task it depends on
|
|
159
|
+
cycle_name: str | None = None
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class ConfigCycleTaskInput(_NamedBaseModel, _LagDateBaseModel):
|
|
163
|
+
"""
|
|
164
|
+
To create an instance of an input in a task in a cycle defined in a workflow file.
|
|
165
|
+
|
|
166
|
+
For example:
|
|
167
|
+
|
|
168
|
+
.. yaml
|
|
169
|
+
|
|
170
|
+
- my_input:
|
|
171
|
+
date: ...
|
|
172
|
+
lag: ...
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
arg_option: str | None = None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class ConfigCycleTaskOutput(_NamedBaseModel):
|
|
179
|
+
"""
|
|
180
|
+
To create an instance of an output in a task in a cycle defined in a workflow file.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class ConfigCycleTask(_NamedBaseModel):
|
|
185
|
+
"""
|
|
186
|
+
To create an instance of a task in a cycle defined in a workflow file.
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
inputs: list[ConfigCycleTaskInput | str] | None = Field(default_factory=list)
|
|
190
|
+
outputs: list[ConfigCycleTaskOutput | str] | None = Field(default_factory=list)
|
|
191
|
+
depends: list[ConfigCycleTaskDepend | str] | None = Field(default_factory=list)
|
|
192
|
+
|
|
193
|
+
@field_validator("inputs", mode="before")
|
|
194
|
+
@classmethod
|
|
195
|
+
def convert_cycle_task_inputs(cls, values) -> list[ConfigCycleTaskInput]:
|
|
196
|
+
inputs = []
|
|
197
|
+
if values is None:
|
|
198
|
+
return inputs
|
|
199
|
+
for value in values:
|
|
200
|
+
if isinstance(value, str):
|
|
201
|
+
inputs.append({value: None})
|
|
202
|
+
elif isinstance(value, dict):
|
|
203
|
+
inputs.append(value)
|
|
204
|
+
return inputs
|
|
205
|
+
|
|
206
|
+
@field_validator("outputs", mode="before")
|
|
207
|
+
@classmethod
|
|
208
|
+
def convert_cycle_task_outputs(cls, values) -> list[ConfigCycleTaskOutput]:
|
|
209
|
+
outputs = []
|
|
210
|
+
if values is None:
|
|
211
|
+
return outputs
|
|
212
|
+
for value in values:
|
|
213
|
+
if isinstance(value, str):
|
|
214
|
+
outputs.append({value: None})
|
|
215
|
+
elif isinstance(value, dict):
|
|
216
|
+
outputs.append(value)
|
|
217
|
+
return outputs
|
|
218
|
+
|
|
219
|
+
@field_validator("depends", mode="before")
|
|
220
|
+
@classmethod
|
|
221
|
+
def convert_cycle_task_depends(cls, values) -> list[ConfigCycleTaskDepend]:
|
|
222
|
+
depends = []
|
|
223
|
+
if values is None:
|
|
224
|
+
return depends
|
|
225
|
+
for value in values:
|
|
226
|
+
if isinstance(value, str):
|
|
227
|
+
depends.append({value: None})
|
|
228
|
+
elif isinstance(value, dict):
|
|
229
|
+
depends.append(value)
|
|
230
|
+
|
|
231
|
+
return depends
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class ConfigCycle(_NamedBaseModel):
|
|
235
|
+
"""
|
|
236
|
+
To create an instance of a cycle defined in a workflow file.
|
|
237
|
+
"""
|
|
238
|
+
|
|
239
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
240
|
+
name: str
|
|
241
|
+
tasks: list[ConfigCycleTask]
|
|
242
|
+
start_date: datetime | None = None
|
|
243
|
+
end_date: datetime | None = None
|
|
244
|
+
period: Duration | None = None
|
|
245
|
+
|
|
246
|
+
@field_validator("start_date", "end_date", mode="before")
|
|
247
|
+
@classmethod
|
|
248
|
+
def convert_datetime(cls, value) -> None | datetime:
|
|
249
|
+
return None if value is None else datetime.fromisoformat(value)
|
|
250
|
+
|
|
251
|
+
@field_validator("period", mode="before")
|
|
252
|
+
@classmethod
|
|
253
|
+
def convert_duration(cls, value):
|
|
254
|
+
return None if value is None else parse_duration(value)
|
|
255
|
+
|
|
256
|
+
@model_validator(mode="after")
|
|
257
|
+
def check_start_date_before_end_date(self) -> ConfigCycle:
|
|
258
|
+
if self.start_date is not None and self.end_date is not None and self.start_date > self.end_date:
|
|
259
|
+
msg = "For cycle {self._name!r} the start_date {start_date!r} lies after given end_date {end_date!r}."
|
|
260
|
+
raise ValueError(msg)
|
|
261
|
+
return self
|
|
262
|
+
|
|
263
|
+
@model_validator(mode="after")
|
|
264
|
+
def check_period_is_not_negative_or_zero(self) -> ConfigCycle:
|
|
265
|
+
if self.period is not None and TimeUtils.duration_is_less_equal_zero(self.period):
|
|
266
|
+
msg = f"For cycle {self.name!r} the period {self.period!r} is negative or zero."
|
|
267
|
+
raise ValueError(msg)
|
|
268
|
+
return self
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class ConfigWorkflow(BaseModel):
|
|
272
|
+
name: str | None = None
|
|
273
|
+
start_date: datetime
|
|
274
|
+
end_date: datetime
|
|
275
|
+
cycles: list[ConfigCycle]
|
|
276
|
+
tasks: list[ConfigTask]
|
|
277
|
+
data: ConfigData
|
|
278
|
+
data_dict: dict = {}
|
|
279
|
+
task_dict: dict = {}
|
|
280
|
+
|
|
281
|
+
@field_validator("start_date", "end_date", mode="before")
|
|
282
|
+
@classmethod
|
|
283
|
+
def convert_datetime(cls, value) -> None | datetime:
|
|
284
|
+
return None if value is None else datetime.fromisoformat(value)
|
|
285
|
+
|
|
286
|
+
@model_validator(mode="after")
|
|
287
|
+
def check_start_date_before_end_date(self) -> ConfigWorkflow:
|
|
288
|
+
if self.start_date is not None and self.end_date is not None and self.start_date > self.end_date:
|
|
289
|
+
msg = "For workflow {self._name!r} the start_date {start_date!r} lies after given end_date {end_date!r}."
|
|
290
|
+
raise ValueError(msg)
|
|
291
|
+
return self
|
|
292
|
+
|
|
293
|
+
def to_core_workflow(self):
|
|
294
|
+
self.data_dict = {data.name: data for data in self.data.available} | {
|
|
295
|
+
data.name: data for data in self.data.generated
|
|
296
|
+
}
|
|
297
|
+
self.task_dict = {task.name: task for task in self.tasks}
|
|
298
|
+
|
|
299
|
+
core_cycles = [self._to_core_cycle(cycle) for cycle in self.cycles]
|
|
300
|
+
return core.Workflow(self.name, core_cycles)
|
|
301
|
+
|
|
302
|
+
def _to_core_cycle(self, cycle: ConfigCycle) -> core.Cycle:
|
|
303
|
+
core_tasks = [self._to_core_task(task) for task in cycle.tasks]
|
|
304
|
+
start_date = self.start_date if cycle.start_date is None else cycle.start_date
|
|
305
|
+
end_date = self.end_date if cycle.end_date is None else cycle.end_date
|
|
306
|
+
return core.Cycle(cycle.name, core_tasks, start_date, end_date, cycle.period)
|
|
307
|
+
|
|
308
|
+
def _to_core_task(self, cycle_task: ConfigCycleTask) -> core.Task:
|
|
309
|
+
inputs = []
|
|
310
|
+
outputs = []
|
|
311
|
+
dependencies = []
|
|
312
|
+
|
|
313
|
+
for input_ in cycle_task.inputs:
|
|
314
|
+
if (data := self.data_dict.get(input_.name)) is None:
|
|
315
|
+
msg = f"Task {cycle_task.name!r} has input {input_.name!r} that is not specied in the data section."
|
|
316
|
+
raise ValueError(msg)
|
|
317
|
+
core_data = core.Data(
|
|
318
|
+
input_.name, data.type, data.src, input_.lag, input_.date, input_.arg_option, available=data.available
|
|
319
|
+
)
|
|
320
|
+
inputs.append(core_data)
|
|
321
|
+
|
|
322
|
+
for output in cycle_task.outputs:
|
|
323
|
+
if (data := self.data_dict.get(output.name)) is None:
|
|
324
|
+
msg = f"Task {cycle_task.name!r} has output {output.name!r} that is not specied in the data section."
|
|
325
|
+
raise ValueError(msg)
|
|
326
|
+
core_data = core.Data(output.name, data.type, data.src, [], [], None, available=False)
|
|
327
|
+
outputs.append(core_data)
|
|
328
|
+
|
|
329
|
+
for depend in cycle_task.depends:
|
|
330
|
+
core_dependency = core.Dependency(depend.name, depend.lag, depend.date, depend.cycle_name)
|
|
331
|
+
dependencies.append(core_dependency)
|
|
332
|
+
|
|
333
|
+
return core.Task(
|
|
334
|
+
cycle_task.name,
|
|
335
|
+
self.task_dict[cycle_task.name].command,
|
|
336
|
+
inputs,
|
|
337
|
+
outputs,
|
|
338
|
+
dependencies,
|
|
339
|
+
self.task_dict[cycle_task.name].command_option,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def load_workflow_config(workflow_config: str) -> ConfigWorkflow:
|
|
344
|
+
"""
|
|
345
|
+
Loads a python representation of a workflow config file.
|
|
346
|
+
|
|
347
|
+
:param workflow_config: the string to the config yaml file containing the workflow definition
|
|
348
|
+
"""
|
|
349
|
+
from pydantic_yaml import parse_yaml_raw_as
|
|
350
|
+
|
|
351
|
+
config_path = Path(workflow_config)
|
|
352
|
+
|
|
353
|
+
content = config_path.read_text()
|
|
354
|
+
|
|
355
|
+
parsed_workflow = parse_yaml_raw_as(ConfigWorkflow, content)
|
|
356
|
+
|
|
357
|
+
# If name was not specified, then we use filename without file extension
|
|
358
|
+
if parsed_workflow.name is None:
|
|
359
|
+
parsed_workflow.name = config_path.stem
|
|
360
|
+
|
|
361
|
+
return parsed_workflow
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: sirocco
|
|
3
|
+
Version: 0.0.0.dev0
|
|
4
|
+
Summary: A library for creating weather and climate workflows.
|
|
5
|
+
Project-URL: Repository, https://github.com/C2SM/Sirocco.git
|
|
6
|
+
Project-URL: Changelog, https://github.com/C2SM/Sirocco/blob/main/CHANGELOG.md
|
|
7
|
+
Author-email: Matthieu Leclair <matthieu.leclair@env.ethz.ch>, Julian Geiger <julian.geiger@psi.ch>, Alexander Goscinski <alexander.goscinski@psi.ch>
|
|
8
|
+
Keywords: icon,aiida,aiida-workgraph,workflow
|
|
9
|
+
Classifier: Development Status :: 1 - Planning
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Requires-Dist: aiida-core>=2.5
|
|
18
|
+
Requires-Dist: isoduration
|
|
19
|
+
Requires-Dist: pydantic
|
|
20
|
+
Requires-Dist: pydantic-yaml
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# Sirocco
|
|
24
|
+
|
|
25
|
+
Sirocco is a library for creating weather and climate workflows from a yaml
|
|
26
|
+
format inspired by [cylc](https://cylc.github.io/) using
|
|
27
|
+
[AiiDA](https://www.aiida.net/) as workflow engine.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
To install it please use
|
|
32
|
+
|
|
33
|
+
``` bash
|
|
34
|
+
pip install -e .
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Developer tools
|
|
38
|
+
|
|
39
|
+
To manage the repo we use [hatch]{.title-ref} please install it
|
|
40
|
+
|
|
41
|
+
``` bash
|
|
42
|
+
pip install hatch
|
|
43
|
+
hatch test # run tests
|
|
44
|
+
hatch fmt # run formatting
|
|
45
|
+
hatch run docs:build # build docs
|
|
46
|
+
hatch run docs:serve # live preview of doc for development
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Resources
|
|
50
|
+
|
|
51
|
+
- <https://aiida-workgraph.readthedocs.io/en/latest/>
|
|
52
|
+
- <https://github.com/aiidateam/aiida-workgraph>
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
sirocco/__init__.py,sha256=ZEwAZxBF5K4YiYOhxbKPX-FowH0-NAhXQL8Hd5A64rg,73
|
|
2
|
+
sirocco/core.py,sha256=OI_pjW7knhAt4udLcIxki6H9MnbVace5AC-v2bhI8Q8,22835
|
|
3
|
+
sirocco/parsing/__init__.py,sha256=TWB2cUAKdMohdSB1fOm4jvUHsZeaKx2DBuGFI5SqMFk,95
|
|
4
|
+
sirocco/parsing/_utils.py,sha256=lx6HPi6V-19DBv2efU_KUYPqXChCST5X9Sx3CFcRPfc,746
|
|
5
|
+
sirocco/parsing/_yaml_data_models.py,sha256=uMD7we3mP-K1qlHOwGBb5fsAskzPZ-FvwTNYJ2y9TLo,11874
|
|
6
|
+
sirocco-0.0.0.dev0.dist-info/METADATA,sha256=e2eH0eyjPjHj8fAHyD6ojRiupRGQ30t886Kiqwaq-O0,1628
|
|
7
|
+
sirocco-0.0.0.dev0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
8
|
+
sirocco-0.0.0.dev0.dist-info/RECORD,,
|