sirocco 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sirocco/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ from . import parsing
2
+
3
+ __all__ = ["parsing"]
4
+
5
+ __version__ = "0.0.0-dev0"
sirocco/core.py ADDED
@@ -0,0 +1,689 @@
1
+ from __future__ import annotations
2
+
3
+ from datetime import datetime
4
+ from itertools import chain
5
+ from os.path import expandvars
6
+ from pathlib import Path
7
+ from typing import TYPE_CHECKING
8
+
9
+ from isoduration import parse_duration
10
+ from isoduration.types import Duration
11
+
12
+ from sirocco.parsing._utils import TimeUtils
13
+
14
+ if TYPE_CHECKING:
15
+ from collections.abc import Generator
16
+
17
+
18
+ class _DataBase:
19
+ def __init__(
20
+ self,
21
+ name: str,
22
+ type: str, # noqa: A002
23
+ src: str,
24
+ lag: list[Duration],
25
+ date: list[datetime],
26
+ arg_option: str | None,
27
+ *,
28
+ available: bool,
29
+ ):
30
+ self._name = name
31
+
32
+ self._src = src
33
+ self._path = Path(expandvars(self._src))
34
+
35
+ self._type = type
36
+ if self._type not in ["file", "dir"]:
37
+ msg = f"Data type {self._type!r} not supported. Please use 'file' or 'dir'."
38
+ raise ValueError(msg)
39
+
40
+ if len(lag) > 0 and len(date) > 0:
41
+ msg = "Either 'lag' or 'date' can be nonempty. Not both."
42
+ raise ValueError(msg)
43
+
44
+ # COMMENT I think we should just disallow multiple lags, and enforce the user to write multiple lags
45
+ # I am not sure how this work with icon as it does not need positional arguments
46
+ # or rather how does it work with plugins
47
+ if arg_option is not None and (len(lag) > 1 or len(date) > 1):
48
+ msg = (
49
+ "You cannot give an arg_option when multiple lags and dates are given. "
50
+ "They must be positional arguments, since passing them to one option is ambiguous."
51
+ )
52
+ raise ValueError(msg)
53
+
54
+ self._lag = lag
55
+ self._date = date
56
+ self._arg_option = arg_option
57
+ self._available = available
58
+
59
+ @property
60
+ def name(self) -> str:
61
+ """The name of this data instance that is used as identifier."""
62
+ return self._name
63
+
64
+ @property
65
+ def type(self) -> str:
66
+ """The data type."""
67
+ return self._type
68
+
69
+ @property
70
+ def src(self) -> str:
71
+ return self._src
72
+
73
+ @property
74
+ def path(self) -> Path:
75
+ return self._path
76
+
77
+ @property
78
+ def lag(self) -> list[Duration]:
79
+ return self._lag
80
+
81
+ @property
82
+ def date(self) -> list[datetime]:
83
+ return self._date
84
+
85
+ @property
86
+ def arg_option(self) -> str | None:
87
+ return self._arg_option
88
+
89
+ @property
90
+ def available(self) -> bool:
91
+ return self._available
92
+
93
+
94
+ class Data(_DataBase):
95
+ def __init__(
96
+ self,
97
+ name: str,
98
+ type: str, # noqa: A002
99
+ src: str,
100
+ lag: list[Duration],
101
+ date: list[datetime],
102
+ arg_option: str | None,
103
+ *,
104
+ available: bool,
105
+ ):
106
+ super().__init__(name, type, src, lag, date, arg_option, available=available)
107
+ self._task: Task | None = None
108
+
109
+ def unroll(self, unrolled_task: UnrolledTask) -> Generator[UnrolledData, None, None]:
110
+ if len(self._date) == 0 and len(self._lag) == 0:
111
+ yield UnrolledData.from_data(self, unrolled_task, unrolled_task.unrolled_date)
112
+
113
+ for lag in self._lag:
114
+ lagged_date = unrolled_task.unrolled_date + lag
115
+ if (
116
+ lagged_date >= unrolled_task.unrolled_cycle.start_date
117
+ and lagged_date <= unrolled_task.unrolled_cycle.end_date
118
+ ):
119
+ yield UnrolledData.from_data(self, unrolled_task, lagged_date)
120
+
121
+ for date in self._date:
122
+ yield UnrolledData.from_data(self, unrolled_task, date)
123
+
124
+ def __repr__(self) -> str:
125
+ if self.task is None:
126
+ identifier = f"{self.__class__.__name__} '{self.name}'"
127
+ else:
128
+ identifier = f"{self.__class__.__name__} '{self.name}' attached task '{self.task}'"
129
+ return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
130
+
131
+ @property
132
+ def task(self) -> Task | None:
133
+ return self._task
134
+
135
+ @task.setter
136
+ def task(self, task: Task):
137
+ if self._task is not None:
138
+ msg = f"Data {self} was already assigned to task {self._task}. Cannot assign task to task {task}."
139
+ raise ValueError(msg)
140
+ self._task = task
141
+
142
+
143
+ class UnrolledData(_DataBase):
144
+ """
145
+ Data that are created during the unrolling of a cycle.
146
+ This class should be only initiated through unrolling a cycle.
147
+ """
148
+
149
+ @classmethod
150
+ def from_data(cls, data: Data, unrolled_task: UnrolledTask, unrolled_date: datetime):
151
+ return cls(
152
+ unrolled_task,
153
+ unrolled_date,
154
+ data.name,
155
+ data.type,
156
+ data.src,
157
+ data.lag,
158
+ data.date,
159
+ data.arg_option,
160
+ available=data.available,
161
+ )
162
+
163
+ def __init__(
164
+ self,
165
+ unrolled_task: UnrolledTask,
166
+ unrolled_date: datetime,
167
+ name: str,
168
+ type: str, # noqa: A002
169
+ src: str,
170
+ lag: list[Duration],
171
+ date: list[datetime],
172
+ arg_option: str | None,
173
+ *,
174
+ available: bool,
175
+ ):
176
+ super().__init__(name, type, src, lag, date, arg_option, available=available)
177
+ self._unrolled_task = unrolled_task
178
+ self._unrolled_date = unrolled_date
179
+
180
+ def __repr__(self) -> str:
181
+ if self.unrolled_task is None:
182
+ identifier = f"{self.__class__.__name__} '{self.name}' with date {self.unrolled_date}"
183
+ else:
184
+ identifier = f"{self.__class__.__name__} '{self.name}' with date {self.unrolled_date} attached to task {self.unrolled_task}"
185
+ return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
186
+
187
+ @property
188
+ def unrolled_date(self) -> datetime:
189
+ return self._unrolled_date
190
+
191
+ @property
192
+ def unrolled_task(self) -> UnrolledTask:
193
+ return self._unrolled_task
194
+
195
+
196
+ class _DependencyBase:
197
+ def __init__(self, depend_on_task_name: str, lag: list[Duration], date: list[datetime], cycle_name: str | None):
198
+ self._depend_on_task_name = depend_on_task_name
199
+ if len(lag) > 0 and len(date) > 0:
200
+ msg = "Only one key 'lag' or 'date' is allowed. Not both."
201
+ raise ValueError(msg)
202
+
203
+ self._lag = lag
204
+ self._date = date
205
+ self._cycle_name = cycle_name
206
+
207
+ @property
208
+ def depend_on_task_name(self) -> str:
209
+ return self._depend_on_task_name
210
+
211
+ @property
212
+ def lag(self) -> list[Duration]:
213
+ return self._lag
214
+
215
+ @property
216
+ def date(self) -> list[datetime]:
217
+ return self._date
218
+
219
+ @property
220
+ def cycle_name(self) -> str | None:
221
+ return self._cycle_name
222
+
223
+
224
+ class Dependency(_DependencyBase):
225
+ def __init__(self, depend_on_task_name: str, lag: list[Duration], date: list[datetime], cycle_name: str | None):
226
+ super().__init__(depend_on_task_name, lag, date, cycle_name)
227
+ self._task: Task | None = None
228
+
229
+ def unroll(self, unrolled_task: UnrolledTask) -> Generator[UnrolledDependency, None, None]:
230
+ if len(self._date) == 0 and len(self._lag) == 0:
231
+ yield UnrolledDependency.from_dependency(self, unrolled_task, unrolled_task.unrolled_date)
232
+
233
+ for lag in self._lag:
234
+ lagged_date = unrolled_task.unrolled_date + lag
235
+ if (
236
+ lagged_date >= unrolled_task.unrolled_cycle.start_date
237
+ and lagged_date <= unrolled_task.unrolled_cycle.end_date
238
+ ):
239
+ yield UnrolledDependency.from_dependency(self, unrolled_task, lagged_date)
240
+
241
+ for date in self._date:
242
+ yield UnrolledDependency.from_dependency(self, unrolled_task, date)
243
+
244
+ @property
245
+ def task(self) -> Task | None:
246
+ return self._task
247
+
248
+ @task.setter
249
+ def task(self, task: Task):
250
+ if self.task is not None:
251
+ msg = f"Dependency was already assigned to task {self.task}. Cannot assign to task {task}."
252
+ raise ValueError(msg)
253
+ self._task = task
254
+
255
+ def __repr__(self) -> str:
256
+ if self._cycle_name is None:
257
+ identifier = f"{self.__class__.__name__} on task '{self.depend_on_task_name}' attached to task {self.task}"
258
+ else:
259
+ identifier = f"{self.__class__.__name__} on task '{self.depend_on_task_name}' in cycle '{self.cycle_name}' attached to task {self.task}"
260
+ return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
261
+
262
+
263
+ class UnrolledDependency(_DependencyBase):
264
+ """
265
+ This class should be only initiated through unrolling a cycle.
266
+ """
267
+
268
+ @classmethod
269
+ def from_dependency(cls, depend: Dependency, unrolled_task: UnrolledTask, unrolled_date: datetime):
270
+ return cls(unrolled_task, unrolled_date, depend.depend_on_task_name, depend.lag, depend.date, depend.cycle_name)
271
+
272
+ def __init__(
273
+ self,
274
+ unrolled_task: UnrolledTask,
275
+ unrolled_date: datetime,
276
+ depend_on_task_name: str,
277
+ lag: list[Duration],
278
+ date: list[datetime],
279
+ cycle_name: str | None,
280
+ ):
281
+ super().__init__(depend_on_task_name, lag, date, cycle_name)
282
+ self._unrolled_task = unrolled_task
283
+ self._unrolled_date = unrolled_date
284
+
285
+ def __repr__(self) -> str:
286
+ if self._cycle_name is None:
287
+ identifier = (
288
+ f"{self.__class__.__name__} on task '{self.depend_on_task_name}' with date {self.unrolled_date}"
289
+ )
290
+ else:
291
+ identifier = f"{self.__class__.__name__} on task '{self.depend_on_task_name}' in cycle '{self.cycle_name}' with date {self.unrolled_date}"
292
+ return super().__repr__().replace(f"{self.__class__.__name__}", identifier)
293
+
294
+ @property
295
+ def depend_on_task(self) -> UnrolledTask:
296
+ """
297
+ throws error if not found
298
+ """
299
+ # for now we only support looking in the same cycle
300
+ workflow = self._unrolled_task.unrolled_cycle.workflow
301
+ if self._cycle_name is None:
302
+ tasks_to_search = [
303
+ cycle.unrolled_tasks for cycle in workflow.unrolled_cycles if cycle.unrolled_date == self._unrolled_date
304
+ ]
305
+ potential_tasks = [
306
+ task for task in chain.from_iterable(tasks_to_search) if task.name == self._depend_on_task_name
307
+ ]
308
+ if len(potential_tasks) > 1:
309
+ msg = (
310
+ f"Found multiple instances of the task '{self._depend_on_task_name}' with date {self._unrolled_date}"
311
+ " for dependency of the task {self._unrolled_task}. Please specify a cycle name."
312
+ )
313
+ raise ValueError(msg)
314
+ if len(potential_tasks) == 0:
315
+ msg = (
316
+ f"Found no instance of the task '{self._depend_on_task_name}' with date {self._unrolled_date}"
317
+ f" for dependency attached to task {self._unrolled_task}."
318
+ )
319
+ raise ValueError(msg)
320
+ return potential_tasks[0]
321
+
322
+ cycle = workflow.unrolled_cycles_map[(self._cycle_name, self._unrolled_date)]
323
+ return cycle.unrolled_tasks_map[self._depend_on_task_name]
324
+
325
+ @property
326
+ def unrolled_task(self) -> UnrolledTask:
327
+ return self._unrolled_task
328
+
329
+ @property
330
+ def unrolled_date(self) -> datetime:
331
+ return self._unrolled_date
332
+
333
+
334
+ class _TaskBase:
335
+ """
336
+ Common class for Task and UnrolledTask to reduce code duplications
337
+ """
338
+
339
+ def __init__(
340
+ self,
341
+ name: str,
342
+ command: str,
343
+ inputs: list[Data],
344
+ outputs: list[Data],
345
+ depends: list[Dependency],
346
+ command_option: str | None,
347
+ ):
348
+ self._name = name
349
+ self._command = expandvars(command)
350
+ self._inputs = inputs
351
+ self._outputs = outputs
352
+ self._depends = depends
353
+ self._command_option = command_option
354
+
355
+ @property
356
+ def name(self) -> str:
357
+ return self._name
358
+
359
+ @property
360
+ def command(self) -> str:
361
+ return self._command
362
+
363
+ @property
364
+ def inputs(self) -> list[Data]:
365
+ return self._inputs
366
+
367
+ @property
368
+ def outputs(self) -> list[Data]:
369
+ return self._outputs
370
+
371
+ @property
372
+ def command_option(self) -> str | None:
373
+ return self._command_option
374
+
375
+ @property
376
+ def depends(self) -> list[Dependency]:
377
+ return self._depends
378
+
379
+
380
+ class Task(_TaskBase):
381
+ """A task that is created during the unrolling of a cycle."""
382
+
383
+ def __init__(
384
+ self,
385
+ name: str,
386
+ command: str,
387
+ inputs: list[Data],
388
+ outputs: list[Data],
389
+ depends: list[Dependency],
390
+ command_option: str | None,
391
+ ):
392
+ super().__init__(name, command, inputs, outputs, depends, command_option)
393
+ for input_ in inputs:
394
+ input_.task = self
395
+ for output in outputs:
396
+ output.task = self
397
+ for depend in depends:
398
+ depend.task = self
399
+ self._cycle: Cycle | None = None
400
+
401
+ def __repr__(self) -> str:
402
+ identifier = f"Task '{self.name}'"
403
+ if self.cycle is not None:
404
+ identifier += f" in cycle {self.cycle.name}"
405
+ return super().__repr__().replace("Task", identifier)
406
+
407
+ def unroll(self, unrolled_cycle: UnrolledCycle) -> Generator[tuple[str, UnrolledTask], None, None]:
408
+ # an unrolled task is just one task, since the date is determined
409
+ # by the cycle, but we keep the pattern for consistency
410
+ unrolled_task = UnrolledTask.from_task(self, unrolled_cycle)
411
+ yield unrolled_task.name, unrolled_task
412
+
413
+ @property
414
+ def cycle(self) -> Cycle | None:
415
+ return self._cycle
416
+
417
+ @cycle.setter
418
+ def cycle(self, cycle: Cycle):
419
+ if self._cycle is not None:
420
+ msg = f"Task {self} was already assigned to cycle {self._cycle}. Cannot assign task to cycle {cycle}."
421
+ raise ValueError(msg)
422
+ self._cycle = cycle
423
+
424
+
425
+ class UnrolledTask(_TaskBase):
426
+ """
427
+ This class should be only initiated through unrolling a cycle.
428
+ """
429
+
430
+ @classmethod
431
+ def from_task(cls, task: Task, unrolled_cycle: UnrolledCycle):
432
+ return cls(
433
+ unrolled_cycle, task.name, task.command, task.inputs, task.outputs, task.depends, task.command_option
434
+ )
435
+
436
+ def __init__(
437
+ self,
438
+ unrolled_cycle: UnrolledCycle,
439
+ name: str,
440
+ command: str,
441
+ inputs: list[Data],
442
+ outputs: list[Data],
443
+ depends: list[Dependency],
444
+ command_option: str | None,
445
+ ):
446
+ super().__init__(name, command, inputs, outputs, depends, command_option)
447
+ self._unrolled_cycle = unrolled_cycle
448
+ self._unrolled_inputs = list(self.unroll_inputs())
449
+ self._unrolled_outputs = list(self.unroll_outputs())
450
+ self._unrolled_depends = list(self.unroll_depends())
451
+
452
+ def __repr__(self) -> str:
453
+ if self.unrolled_cycle is None:
454
+ identifier = f"Task '{self.name}' with date {self.unrolled_date}"
455
+ else:
456
+ identifier = f"Task '{self.name}' in cycle {self.unrolled_cycle.name} with date {self.unrolled_date}"
457
+ return super().__repr__().replace("Task", identifier)
458
+
459
+ def unroll_inputs(self) -> Generator[UnrolledData, None, None]:
460
+ """
461
+ Outputs the inputs together with a unique identifier within the task
462
+ """
463
+ for input_ in self._inputs:
464
+ yield from input_.unroll(self)
465
+
466
+ def unroll_outputs(self) -> Generator[UnrolledData, None, None]:
467
+ for output in self._outputs:
468
+ yield from output.unroll(self)
469
+
470
+ def unroll_depends(self) -> Generator[UnrolledDependency, None, None]:
471
+ for depend in self._depends:
472
+ yield from depend.unroll(self)
473
+
474
+ @property
475
+ def unrolled_inputs(self) -> list[UnrolledData]:
476
+ return self._unrolled_inputs
477
+
478
+ @property
479
+ def unrolled_outputs(self) -> list[UnrolledData]:
480
+ return self._unrolled_outputs
481
+
482
+ @property
483
+ def unrolled_depends(self) -> list[UnrolledDependency]:
484
+ return self._unrolled_depends
485
+
486
+ @property
487
+ def unrolled_date(self) -> datetime:
488
+ return self._unrolled_cycle.unrolled_date
489
+
490
+ @property
491
+ def unrolled_cycle(self) -> UnrolledCycle:
492
+ return self._unrolled_cycle
493
+
494
+
495
+ class _CycleBase:
496
+ def __init__(
497
+ self,
498
+ name: str,
499
+ tasks: list[Task],
500
+ start_date: str | datetime,
501
+ end_date: str | datetime,
502
+ period: str | Duration | None = None,
503
+ ):
504
+ self._name = name
505
+ self._tasks = tasks
506
+ self._start_date = start_date if isinstance(start_date, datetime) else datetime.fromisoformat(start_date)
507
+ self._end_date = end_date if isinstance(end_date, datetime) else datetime.fromisoformat(end_date)
508
+
509
+ if self._start_date > self._end_date:
510
+ msg = "For cycle {self} the start_date {start_date} lies after given end_date {end_date}."
511
+ raise ValueError(msg)
512
+
513
+ self._period = period if period is None or isinstance(period, Duration) else parse_duration(period)
514
+ if self._period is not None and TimeUtils.duration_is_less_equal_zero(self._period):
515
+ msg = f"For cycle {self} the period {period} is negative or zero."
516
+ raise ValueError(msg)
517
+
518
+ task_names = set()
519
+ for task in self._tasks:
520
+ if task.name in task_names:
521
+ msg = f"List of tasks does contain tasks with duplicate names. The task name '{task.name}' has been found twice."
522
+ raise ValueError(msg)
523
+ task_names.add(task.name)
524
+
525
+ @property
526
+ def name(self) -> str:
527
+ return self._name
528
+
529
+ @property
530
+ def start_date(self) -> datetime:
531
+ return self._start_date
532
+
533
+ @property
534
+ def end_date(self) -> datetime:
535
+ return self._end_date
536
+
537
+ @property
538
+ def period(self) -> Duration | None:
539
+ return self._period
540
+
541
+ @property
542
+ def tasks(self) -> list[Task]:
543
+ return self._tasks
544
+
545
+
546
+ class Cycle(_CycleBase):
547
+ def __init__(
548
+ self,
549
+ name: str,
550
+ tasks: list[Task],
551
+ start_date: str | datetime,
552
+ end_date: str | datetime,
553
+ period: str | Duration | None,
554
+ ):
555
+ super().__init__(name, tasks, start_date, end_date, period)
556
+ for task in self._tasks:
557
+ task.cycle = self
558
+
559
+ self._workflow: Workflow | None = None
560
+
561
+ def __repr__(self) -> str:
562
+ if self.workflow is None:
563
+ identifier = f"Cycle '{self.name}'"
564
+ else:
565
+ identifier = f"Cycle '{self.name}' in workflow {self.workflow.name}"
566
+ return super().__repr__().replace("Cycle", identifier)
567
+
568
+ def unroll(self) -> Generator[tuple[str, datetime, UnrolledCycle], None, None]:
569
+ if self._workflow is None:
570
+ msg = f"Cannot unroll cycle {self} because it was not attached to a workflow before."
571
+ raise ValueError(msg)
572
+ current_date = self._start_date
573
+ while current_date <= self._end_date:
574
+ unrolled_cycle = UnrolledCycle.from_cycle(self, current_date, self._workflow)
575
+ yield unrolled_cycle.name, unrolled_cycle.unrolled_date, unrolled_cycle
576
+ if self._period is None:
577
+ break
578
+ else:
579
+ current_date += self._period
580
+
581
+ @property
582
+ def workflow(self) -> Workflow | None:
583
+ return self._workflow
584
+
585
+ @workflow.setter
586
+ def workflow(self, workflow: Workflow):
587
+ if self._workflow is not None:
588
+ msg = f"Cycle {self} was already assigned to workflow {self._workflow}. Cannot assign cycle to workflow {workflow}."
589
+ raise ValueError(msg)
590
+ self._workflow = workflow
591
+
592
+
593
+ class UnrolledCycle(_CycleBase):
594
+ """
595
+ This class should be only initiated through unrolling a cycle.
596
+ """
597
+
598
+ @classmethod
599
+ def from_cycle(cls, cycle: Cycle, unrolled_date: datetime, workflow: Workflow):
600
+ return cls(unrolled_date, cycle.name, cycle.tasks, cycle.start_date, cycle.end_date, cycle.period, workflow)
601
+
602
+ def __init__(
603
+ self,
604
+ unrolled_date: datetime,
605
+ name: str,
606
+ tasks: list[Task],
607
+ start_date: str | datetime,
608
+ end_date: str | datetime,
609
+ period: str | Duration | None,
610
+ workflow: Workflow,
611
+ ):
612
+ super().__init__(name, tasks, start_date, end_date, period)
613
+
614
+ self._unrolled_date = unrolled_date
615
+
616
+ self._unrolled_tasks_map = dict(self.unroll_tasks())
617
+ self._workflow = workflow
618
+
619
+ def __repr__(self) -> str:
620
+ if self.workflow is None:
621
+ identifier = f"UnrolledCycle '{self.name}' with date {self.unrolled_date}"
622
+ else:
623
+ identifier = f"UnrolledCycle '{self.name}' in workflow {self.workflow.name} with date {self.unrolled_date}"
624
+ return super().__repr__().replace("UnrolledCycle", identifier)
625
+
626
+ def unroll_tasks(self) -> Generator[tuple[str, UnrolledTask], None, None]:
627
+ for task in self._tasks:
628
+ yield from task.unroll(self)
629
+
630
+ @property
631
+ def unrolled_tasks(self) -> list[UnrolledTask]:
632
+ return list(self._unrolled_tasks_map.values())
633
+
634
+ @property
635
+ def unrolled_tasks_map(self) -> dict[str, UnrolledTask]:
636
+ return self._unrolled_tasks_map
637
+
638
+ @property
639
+ def unrolled_date(self) -> datetime:
640
+ return self._unrolled_date
641
+
642
+ @property
643
+ def workflow(self) -> Workflow:
644
+ return self._workflow
645
+
646
+
647
+ class Workflow:
648
+ def __init__(self, name: str, cycles: list[Cycle]):
649
+ self._name = name
650
+ self._cycles = cycles
651
+ for cycle in self._cycles:
652
+ cycle.workflow = self
653
+ self._validate_cycles()
654
+ self._unrolled_cycles_map = {(name, date): cycle for name, date, cycle in self.unroll_cycles()}
655
+
656
+ unrolled_outputs = []
657
+ for unrolled_cycle in self.unrolled_cycles:
658
+ for unrolled_task in unrolled_cycle.unrolled_tasks:
659
+ unrolled_outputs.extend(unrolled_task.unrolled_outputs)
660
+ self._unrolled_outputs = unrolled_outputs
661
+
662
+ def _validate_cycles(self):
663
+ """Checks if the defined workflow is correctly referencing key names."""
664
+ cycle_names = set()
665
+ for cycle in self._cycles:
666
+ if cycle.name in cycle_names:
667
+ msg = f"List of cycles does contain cycles with duplicate names. The cycle name '{cycle.name}' has been found twice."
668
+ raise ValueError(msg)
669
+ cycle_names.add(cycle.name)
670
+
671
+ def unroll_cycles(self) -> Generator[tuple[str, datetime, UnrolledCycle], None, None]:
672
+ for cycle in self._cycles:
673
+ yield from cycle.unroll()
674
+
675
+ @property
676
+ def name(self) -> str:
677
+ return self._name
678
+
679
+ @property
680
+ def cycles(self) -> list[Cycle]:
681
+ return self._cycles
682
+
683
+ @property
684
+ def unrolled_cycles(self) -> list[UnrolledCycle]:
685
+ return list(self._unrolled_cycles_map.values())
686
+
687
+ @property
688
+ def unrolled_cycles_map(self) -> dict[tuple[str, datetime], UnrolledCycle]:
689
+ return self._unrolled_cycles_map
@@ -0,0 +1,5 @@
1
+ from ._yaml_data_models import load_workflow_config
2
+
3
+ __all__ = [
4
+ "load_workflow_config",
5
+ ]
@@ -0,0 +1,24 @@
1
+ from isoduration.types import Duration
2
+
3
+
4
+ class TimeUtils:
5
+ @staticmethod
6
+ def duration_is_less_equal_zero(duration: Duration) -> bool:
7
+ if (
8
+ duration.date.years == 0
9
+ and duration.date.months == 0
10
+ and duration.date.days == 0
11
+ and duration.time.hours == 0
12
+ and duration.time.minutes == 0
13
+ and duration.time.seconds == 0
14
+ or (
15
+ duration.date.years < 0
16
+ or duration.date.months < 0
17
+ or duration.date.days < 0
18
+ or duration.time.hours < 0
19
+ or duration.time.minutes < 0
20
+ or duration.time.seconds < 0
21
+ )
22
+ ):
23
+ return True
24
+ return False
@@ -0,0 +1,361 @@
1
+ from __future__ import annotations
2
+
3
+ import time
4
+ from datetime import datetime
5
+ from os.path import expandvars
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from isoduration import parse_duration
10
+ from isoduration.types import Duration # pydantic needs type # noqa: TCH002
11
+ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
12
+
13
+ from sirocco import core
14
+
15
+ from ._utils import TimeUtils
16
+
17
+
18
+ class _NamedBaseModel(BaseModel):
19
+ """Base class for all classes with a key that specifies their name.
20
+
21
+ For example:
22
+
23
+ .. yaml
24
+
25
+ - property_name:
26
+ property: true
27
+
28
+ When parsing with this as parent class it is converted to
29
+ `{"name": "propery_name", "property": True}`.
30
+ """
31
+
32
+ name: str
33
+
34
+ def __init__(self, /, **data):
35
+ name_and_spec = {}
36
+ # - my_name:
37
+ # ...
38
+ if len(data) != 1:
39
+ msg = f"Expected dict with one element of the form {{'name': specification}} but got {data}."
40
+ raise ValueError(msg)
41
+ name_and_spec["name"] = next(iter(data.keys()))
42
+ # if no specification specified e.g. "- my_name:"
43
+ if (spec := next(iter(data.values()))) is not None:
44
+ name_and_spec.update(spec)
45
+
46
+ super().__init__(**name_and_spec)
47
+
48
+
49
+ class _LagDateBaseModel(BaseModel):
50
+ """Base class for all classes containg a list of dates or time lags."""
51
+
52
+ model_config = ConfigDict(arbitrary_types_allowed=True)
53
+ date: list[datetime] = [] # this is safe in pydantic
54
+ lag: list[Duration] = [] # this is safe in pydantic
55
+
56
+ @model_validator(mode="before")
57
+ @classmethod
58
+ def check_lag_xor_date_is_set(cls, data: Any) -> Any:
59
+ if "lag" in data and "date" in data:
60
+ msg = "Only one key 'lag' or 'date' is allowed. Not both."
61
+ raise ValueError(msg)
62
+ return data
63
+
64
+ @field_validator("lag", mode="before")
65
+ @classmethod
66
+ def convert_durations(cls, value) -> list[Duration]:
67
+ if value is None:
68
+ return []
69
+ values = value if isinstance(value, list) else [value]
70
+ return [parse_duration(value) for value in values]
71
+
72
+ @field_validator("date", mode="before")
73
+ @classmethod
74
+ def convert_datetimes(cls, value) -> list[datetime]:
75
+ if value is None:
76
+ return []
77
+ values = value if isinstance(value, list) else [value]
78
+ return [datetime.fromisoformat(value) for value in values]
79
+
80
+
81
+ class ConfigTask(_NamedBaseModel):
82
+ """
83
+ To create an instance of a task defined in a workflow file
84
+ """
85
+
86
+ command: str
87
+ command_option: str | None = None
88
+ host: str | None = None
89
+ account: str | None = None
90
+ plugin: str | None = None
91
+ config: str | None = None
92
+ uenv: dict | None = None
93
+ nodes: int | None = None
94
+ walltime: str | None = None
95
+ src: str | None = None
96
+ conda_env: str | None = None
97
+
98
+ def __init__(self, /, **data):
99
+ # We have to treat root special as it does not typically define a command
100
+ if "ROOT" in data and "command" not in data["ROOT"]:
101
+ data["ROOT"]["command"] = "ROOT_PLACEHOLDER"
102
+ super().__init__(**data)
103
+
104
+ @field_validator("command")
105
+ @classmethod
106
+ def expand_env_vars(cls, value: str) -> str:
107
+ """Expands any environment variables in the value"""
108
+ return expandvars(value)
109
+
110
+ @field_validator("walltime")
111
+ @classmethod
112
+ def convert_to_struct_time(cls, value: str | None) -> time.struct_time | None:
113
+ """Converts a string of form "%H:%M:%S" to a time.time_struct"""
114
+ return None if value is None else time.strptime(value, "%H:%M:%S")
115
+
116
+
117
+ class _DataBaseModel(_NamedBaseModel):
118
+ """
119
+ To create an instance of a data defined in a workflow file.
120
+ """
121
+
122
+ type: str
123
+ src: str
124
+ format: str | None = None
125
+
126
+ @field_validator("type")
127
+ @classmethod
128
+ def is_file_or_dir(cls, value: str) -> str:
129
+ """."""
130
+ if value not in ["file", "dir"]:
131
+ msg = "Must be one of 'file' or 'dir'."
132
+ raise ValueError(msg)
133
+ return value
134
+
135
+ @property
136
+ def available(self) -> bool:
137
+ return isinstance(self, ConfigAvailableData)
138
+
139
+
140
+ class ConfigAvailableData(_DataBaseModel):
141
+ pass
142
+
143
+
144
+ class ConfigGeneratedData(_DataBaseModel):
145
+ pass
146
+
147
+
148
+ class ConfigData(BaseModel):
149
+ available: list[ConfigAvailableData]
150
+ generated: list[ConfigGeneratedData]
151
+
152
+
153
+ class ConfigCycleTaskDepend(_NamedBaseModel, _LagDateBaseModel):
154
+ """
155
+ To create an instance of a input or output in a task in a cycle defined in a workflow file.
156
+ """
157
+
158
+ name: str # name of the task it depends on
159
+ cycle_name: str | None = None
160
+
161
+
162
+ class ConfigCycleTaskInput(_NamedBaseModel, _LagDateBaseModel):
163
+ """
164
+ To create an instance of an input in a task in a cycle defined in a workflow file.
165
+
166
+ For example:
167
+
168
+ .. yaml
169
+
170
+ - my_input:
171
+ date: ...
172
+ lag: ...
173
+ """
174
+
175
+ arg_option: str | None = None
176
+
177
+
178
+ class ConfigCycleTaskOutput(_NamedBaseModel):
179
+ """
180
+ To create an instance of an output in a task in a cycle defined in a workflow file.
181
+ """
182
+
183
+
184
+ class ConfigCycleTask(_NamedBaseModel):
185
+ """
186
+ To create an instance of a task in a cycle defined in a workflow file.
187
+ """
188
+
189
+ inputs: list[ConfigCycleTaskInput | str] | None = Field(default_factory=list)
190
+ outputs: list[ConfigCycleTaskOutput | str] | None = Field(default_factory=list)
191
+ depends: list[ConfigCycleTaskDepend | str] | None = Field(default_factory=list)
192
+
193
+ @field_validator("inputs", mode="before")
194
+ @classmethod
195
+ def convert_cycle_task_inputs(cls, values) -> list[ConfigCycleTaskInput]:
196
+ inputs = []
197
+ if values is None:
198
+ return inputs
199
+ for value in values:
200
+ if isinstance(value, str):
201
+ inputs.append({value: None})
202
+ elif isinstance(value, dict):
203
+ inputs.append(value)
204
+ return inputs
205
+
206
+ @field_validator("outputs", mode="before")
207
+ @classmethod
208
+ def convert_cycle_task_outputs(cls, values) -> list[ConfigCycleTaskOutput]:
209
+ outputs = []
210
+ if values is None:
211
+ return outputs
212
+ for value in values:
213
+ if isinstance(value, str):
214
+ outputs.append({value: None})
215
+ elif isinstance(value, dict):
216
+ outputs.append(value)
217
+ return outputs
218
+
219
+ @field_validator("depends", mode="before")
220
+ @classmethod
221
+ def convert_cycle_task_depends(cls, values) -> list[ConfigCycleTaskDepend]:
222
+ depends = []
223
+ if values is None:
224
+ return depends
225
+ for value in values:
226
+ if isinstance(value, str):
227
+ depends.append({value: None})
228
+ elif isinstance(value, dict):
229
+ depends.append(value)
230
+
231
+ return depends
232
+
233
+
234
+ class ConfigCycle(_NamedBaseModel):
235
+ """
236
+ To create an instance of a cycle defined in a workflow file.
237
+ """
238
+
239
+ model_config = ConfigDict(arbitrary_types_allowed=True)
240
+ name: str
241
+ tasks: list[ConfigCycleTask]
242
+ start_date: datetime | None = None
243
+ end_date: datetime | None = None
244
+ period: Duration | None = None
245
+
246
+ @field_validator("start_date", "end_date", mode="before")
247
+ @classmethod
248
+ def convert_datetime(cls, value) -> None | datetime:
249
+ return None if value is None else datetime.fromisoformat(value)
250
+
251
+ @field_validator("period", mode="before")
252
+ @classmethod
253
+ def convert_duration(cls, value):
254
+ return None if value is None else parse_duration(value)
255
+
256
+ @model_validator(mode="after")
257
+ def check_start_date_before_end_date(self) -> ConfigCycle:
258
+ if self.start_date is not None and self.end_date is not None and self.start_date > self.end_date:
259
+ msg = "For cycle {self._name!r} the start_date {start_date!r} lies after given end_date {end_date!r}."
260
+ raise ValueError(msg)
261
+ return self
262
+
263
+ @model_validator(mode="after")
264
+ def check_period_is_not_negative_or_zero(self) -> ConfigCycle:
265
+ if self.period is not None and TimeUtils.duration_is_less_equal_zero(self.period):
266
+ msg = f"For cycle {self.name!r} the period {self.period!r} is negative or zero."
267
+ raise ValueError(msg)
268
+ return self
269
+
270
+
271
+ class ConfigWorkflow(BaseModel):
272
+ name: str | None = None
273
+ start_date: datetime
274
+ end_date: datetime
275
+ cycles: list[ConfigCycle]
276
+ tasks: list[ConfigTask]
277
+ data: ConfigData
278
+ data_dict: dict = {}
279
+ task_dict: dict = {}
280
+
281
+ @field_validator("start_date", "end_date", mode="before")
282
+ @classmethod
283
+ def convert_datetime(cls, value) -> None | datetime:
284
+ return None if value is None else datetime.fromisoformat(value)
285
+
286
+ @model_validator(mode="after")
287
+ def check_start_date_before_end_date(self) -> ConfigWorkflow:
288
+ if self.start_date is not None and self.end_date is not None and self.start_date > self.end_date:
289
+ msg = "For workflow {self._name!r} the start_date {start_date!r} lies after given end_date {end_date!r}."
290
+ raise ValueError(msg)
291
+ return self
292
+
293
+ def to_core_workflow(self):
294
+ self.data_dict = {data.name: data for data in self.data.available} | {
295
+ data.name: data for data in self.data.generated
296
+ }
297
+ self.task_dict = {task.name: task for task in self.tasks}
298
+
299
+ core_cycles = [self._to_core_cycle(cycle) for cycle in self.cycles]
300
+ return core.Workflow(self.name, core_cycles)
301
+
302
+ def _to_core_cycle(self, cycle: ConfigCycle) -> core.Cycle:
303
+ core_tasks = [self._to_core_task(task) for task in cycle.tasks]
304
+ start_date = self.start_date if cycle.start_date is None else cycle.start_date
305
+ end_date = self.end_date if cycle.end_date is None else cycle.end_date
306
+ return core.Cycle(cycle.name, core_tasks, start_date, end_date, cycle.period)
307
+
308
+ def _to_core_task(self, cycle_task: ConfigCycleTask) -> core.Task:
309
+ inputs = []
310
+ outputs = []
311
+ dependencies = []
312
+
313
+ for input_ in cycle_task.inputs:
314
+ if (data := self.data_dict.get(input_.name)) is None:
315
+ msg = f"Task {cycle_task.name!r} has input {input_.name!r} that is not specied in the data section."
316
+ raise ValueError(msg)
317
+ core_data = core.Data(
318
+ input_.name, data.type, data.src, input_.lag, input_.date, input_.arg_option, available=data.available
319
+ )
320
+ inputs.append(core_data)
321
+
322
+ for output in cycle_task.outputs:
323
+ if (data := self.data_dict.get(output.name)) is None:
324
+ msg = f"Task {cycle_task.name!r} has output {output.name!r} that is not specied in the data section."
325
+ raise ValueError(msg)
326
+ core_data = core.Data(output.name, data.type, data.src, [], [], None, available=False)
327
+ outputs.append(core_data)
328
+
329
+ for depend in cycle_task.depends:
330
+ core_dependency = core.Dependency(depend.name, depend.lag, depend.date, depend.cycle_name)
331
+ dependencies.append(core_dependency)
332
+
333
+ return core.Task(
334
+ cycle_task.name,
335
+ self.task_dict[cycle_task.name].command,
336
+ inputs,
337
+ outputs,
338
+ dependencies,
339
+ self.task_dict[cycle_task.name].command_option,
340
+ )
341
+
342
+
343
+ def load_workflow_config(workflow_config: str) -> ConfigWorkflow:
344
+ """
345
+ Loads a python representation of a workflow config file.
346
+
347
+ :param workflow_config: the string to the config yaml file containing the workflow definition
348
+ """
349
+ from pydantic_yaml import parse_yaml_raw_as
350
+
351
+ config_path = Path(workflow_config)
352
+
353
+ content = config_path.read_text()
354
+
355
+ parsed_workflow = parse_yaml_raw_as(ConfigWorkflow, content)
356
+
357
+ # If name was not specified, then we use filename without file extension
358
+ if parsed_workflow.name is None:
359
+ parsed_workflow.name = config_path.stem
360
+
361
+ return parsed_workflow
@@ -0,0 +1,52 @@
1
+ Metadata-Version: 2.3
2
+ Name: sirocco
3
+ Version: 0.0.0.dev0
4
+ Summary: A library for creating weather and climate workflows.
5
+ Project-URL: Repository, https://github.com/C2SM/Sirocco.git
6
+ Project-URL: Changelog, https://github.com/C2SM/Sirocco/blob/main/CHANGELOG.md
7
+ Author-email: Matthieu Leclair <matthieu.leclair@env.ethz.ch>, Julian Geiger <julian.geiger@psi.ch>, Alexander Goscinski <alexander.goscinski@psi.ch>
8
+ Keywords: icon,aiida,aiida-workgraph,workflow
9
+ Classifier: Development Status :: 1 - Planning
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: Operating System :: POSIX :: Linux
12
+ Classifier: Programming Language :: Python
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Topic :: Scientific/Engineering
15
+ Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
16
+ Requires-Python: >=3.12
17
+ Requires-Dist: aiida-core>=2.5
18
+ Requires-Dist: isoduration
19
+ Requires-Dist: pydantic
20
+ Requires-Dist: pydantic-yaml
21
+ Description-Content-Type: text/markdown
22
+
23
+ # Sirocco
24
+
25
+ Sirocco is a library for creating weather and climate workflows from a yaml
26
+ format inspired by [cylc](https://cylc.github.io/) using
27
+ [AiiDA](https://www.aiida.net/) as workflow engine.
28
+
29
+ ## Install
30
+
31
+ To install it please use
32
+
33
+ ``` bash
34
+ pip install -e .
35
+ ```
36
+
37
+ ## Developer tools
38
+
39
+ To manage the repo we use [hatch]{.title-ref} please install it
40
+
41
+ ``` bash
42
+ pip install hatch
43
+ hatch test # run tests
44
+ hatch fmt # run formatting
45
+ hatch run docs:build # build docs
46
+ hatch run docs:serve # live preview of doc for development
47
+ ```
48
+
49
+ ## Resources
50
+
51
+ - <https://aiida-workgraph.readthedocs.io/en/latest/>
52
+ - <https://github.com/aiidateam/aiida-workgraph>
@@ -0,0 +1,8 @@
1
+ sirocco/__init__.py,sha256=ZEwAZxBF5K4YiYOhxbKPX-FowH0-NAhXQL8Hd5A64rg,73
2
+ sirocco/core.py,sha256=OI_pjW7knhAt4udLcIxki6H9MnbVace5AC-v2bhI8Q8,22835
3
+ sirocco/parsing/__init__.py,sha256=TWB2cUAKdMohdSB1fOm4jvUHsZeaKx2DBuGFI5SqMFk,95
4
+ sirocco/parsing/_utils.py,sha256=lx6HPi6V-19DBv2efU_KUYPqXChCST5X9Sx3CFcRPfc,746
5
+ sirocco/parsing/_yaml_data_models.py,sha256=uMD7we3mP-K1qlHOwGBb5fsAskzPZ-FvwTNYJ2y9TLo,11874
6
+ sirocco-0.0.0.dev0.dist-info/METADATA,sha256=e2eH0eyjPjHj8fAHyD6ojRiupRGQ30t886Kiqwaq-O0,1628
7
+ sirocco-0.0.0.dev0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
8
+ sirocco-0.0.0.dev0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.25.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any