remake 0.8.0a0__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {remake-0.8.0a0 → remake-0.8.2}/PKG-INFO +16 -6
- {remake-0.8.0a0 → remake-0.8.2}/README.md +8 -5
- {remake-0.8.0a0 → remake-0.8.2}/pyproject.toml +10 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/__init__.py +7 -2
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/__init__.py +3 -2
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/dag.py +54 -4
- remake-0.8.2/src/remake/core/exceptions.py +38 -0
- remake-0.8.2/src/remake/core/planner.py +482 -0
- remake-0.8.2/src/remake/core/remake.py +766 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/rule.py +90 -5
- remake-0.8.2/src/remake/core/scope.py +313 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/task.py +1 -1
- remake-0.8.2/src/remake/executors/slurm_executor.py +562 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/metadata/__init__.py +1 -0
- remake-0.8.2/src/remake/metadata/metadata_manager.py +126 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/metadata/sidecar.py +17 -1
- remake-0.8.2/src/remake/metadata/sqlite3_backend.py +570 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/remake_cmd.py +415 -326
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/__init__.py +1 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/code_compare.py +22 -4
- remake-0.8.2/src/remake/util/colour.py +64 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/version.py +1 -1
- {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/PKG-INFO +16 -6
- {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/SOURCES.txt +2 -1
- {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/requires.txt +8 -0
- remake-0.8.0a0/src/remake/core/exceptions.py +0 -31
- remake-0.8.0a0/src/remake/core/planner.py +0 -216
- remake-0.8.0a0/src/remake/core/remake.py +0 -227
- remake-0.8.0a0/src/remake/core/scope.py +0 -166
- remake-0.8.0a0/src/remake/executors/slurm_executor.py +0 -299
- remake-0.8.0a0/src/remake/metadata/metadata_manager.py +0 -49
- remake-0.8.0a0/src/remake/metadata/sqlite3_backend.py +0 -260
- {remake-0.8.0a0 → remake-0.8.2}/LICENCE +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/setup.cfg +0 -0
- {remake-0.8.0a0/src/remake → remake-0.8.2/src/remake/core}/tokens.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/__init__.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/dask_executor.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/executor.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/multiproc_executor.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/singleproc_executor.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/loader/__init__.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/command_line_args.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/config.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/util.py +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/dependency_links.txt +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/entry_points.txt +0 -0
- {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: remake
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.2
|
|
4
4
|
Summary: Smart remake tool
|
|
5
5
|
Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
|
|
6
6
|
Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
|
|
@@ -233,20 +233,30 @@ Provides-Extra: s3
|
|
|
233
233
|
Requires-Dist: boto3; extra == "s3"
|
|
234
234
|
Provides-Extra: dask
|
|
235
235
|
Requires-Dist: distributed>=2024.8.0; extra == "dask"
|
|
236
|
+
Provides-Extra: examples
|
|
237
|
+
Requires-Dist: xarray>=2024.7.0; extra == "examples"
|
|
238
|
+
Requires-Dist: netcdf4>=1.7.2; extra == "examples"
|
|
239
|
+
Requires-Dist: h5netcdf; extra == "examples"
|
|
240
|
+
Requires-Dist: zarr<3; extra == "examples"
|
|
241
|
+
Requires-Dist: dask; extra == "examples"
|
|
242
|
+
Requires-Dist: pyyaml; extra == "examples"
|
|
236
243
|
Dynamic: license-file
|
|
237
244
|
|
|
238
|
-
# remake
|
|
245
|
+
# remake
|
|
239
246
|
|
|
240
247
|
Remake is a smart Python build tool, similar to `make`. It makes it easy to build pipelines with complex dependencies, and deploy these to high-performance computing systems. It is particularly suited to use for scientific workflows, due to its ability to reliably recreate any set of output files, based on running only those tasks that are necessary.
|
|
241
248
|
|
|
249
|
+
**Documentation: <https://markmuetz.github.io/remake>**
|
|
250
|
+
|
|
242
251
|
## Status
|
|
243
252
|
|
|
244
|
-
|
|
245
|
-
[
|
|
246
|
-
|
|
253
|
+
remake 0.8.0 is a ground-up rewrite. See the
|
|
254
|
+
[changelog](https://markmuetz.github.io/remake/changelog/) for what's new and
|
|
255
|
+
[getting started](https://markmuetz.github.io/remake/getting-started/) for a
|
|
256
|
+
walkthrough.
|
|
247
257
|
|
|
248
258
|
## Installation
|
|
249
259
|
|
|
250
260
|
```bash
|
|
251
|
-
pip install
|
|
261
|
+
pip install remake
|
|
252
262
|
```
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
# remake
|
|
1
|
+
# remake
|
|
2
2
|
|
|
3
3
|
Remake is a smart Python build tool, similar to `make`. It makes it easy to build pipelines with complex dependencies, and deploy these to high-performance computing systems. It is particularly suited to use for scientific workflows, due to its ability to reliably recreate any set of output files, based on running only those tasks that are necessary.
|
|
4
4
|
|
|
5
|
+
**Documentation: <https://markmuetz.github.io/remake>**
|
|
6
|
+
|
|
5
7
|
## Status
|
|
6
8
|
|
|
7
|
-
|
|
8
|
-
[
|
|
9
|
-
|
|
9
|
+
remake 0.8.0 is a ground-up rewrite. See the
|
|
10
|
+
[changelog](https://markmuetz.github.io/remake/changelog/) for what's new and
|
|
11
|
+
[getting started](https://markmuetz.github.io/remake/getting-started/) for a
|
|
12
|
+
walkthrough.
|
|
10
13
|
|
|
11
14
|
## Installation
|
|
12
15
|
|
|
13
16
|
```bash
|
|
14
|
-
pip install
|
|
17
|
+
pip install remake
|
|
15
18
|
```
|
|
@@ -47,6 +47,16 @@ s3 = ["boto3"]
|
|
|
47
47
|
dask = [
|
|
48
48
|
"distributed>=2024.8.0",
|
|
49
49
|
]
|
|
50
|
+
# Everything examples/ needs beyond the (deliberately minimal) core deps:
|
|
51
|
+
# `pip install "remake[examples]"` / `uv add "remake[examples]"`.
|
|
52
|
+
examples = [
|
|
53
|
+
"xarray>=2024.7.0",
|
|
54
|
+
"netcdf4>=1.7.2",
|
|
55
|
+
"h5netcdf",
|
|
56
|
+
"zarr<3",
|
|
57
|
+
"dask",
|
|
58
|
+
"pyyaml",
|
|
59
|
+
]
|
|
50
60
|
|
|
51
61
|
[dependency-groups]
|
|
52
62
|
dev = [
|
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
from .version import __version__
|
|
2
2
|
|
|
3
3
|
from .core import (
|
|
4
|
-
|
|
4
|
+
Defer,
|
|
5
|
+
FileToken,
|
|
6
|
+
OutputToken,
|
|
7
|
+
PathToken,
|
|
5
8
|
Remake,
|
|
6
9
|
RemakeError,
|
|
7
10
|
Rule,
|
|
11
|
+
S3Object,
|
|
8
12
|
ScopeError,
|
|
9
13
|
ScopeWarning,
|
|
10
14
|
SignatureError,
|
|
11
15
|
Task,
|
|
16
|
+
ZarrStore,
|
|
17
|
+
deferrable,
|
|
12
18
|
rule,
|
|
13
19
|
)
|
|
14
20
|
from .executors import (
|
|
@@ -20,4 +26,3 @@ from .executors import (
|
|
|
20
26
|
)
|
|
21
27
|
from .loader import load_remake
|
|
22
28
|
from .metadata import MetadataManager, Sqlite3Backend, TaskRecord
|
|
23
|
-
from .tokens import FileToken, OutputToken, PathToken, S3Object, ZarrStore
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from .dag import build_rule_dag, expand_rule, resolve_matrix
|
|
2
2
|
from .exceptions import (
|
|
3
|
-
|
|
3
|
+
Defer,
|
|
4
4
|
RemakeError,
|
|
5
5
|
RemakeLoadError,
|
|
6
6
|
ScopeError,
|
|
@@ -8,6 +8,7 @@ from .exceptions import (
|
|
|
8
8
|
)
|
|
9
9
|
from .planner import plan
|
|
10
10
|
from .remake import Remake
|
|
11
|
-
from .rule import Rule, rule
|
|
11
|
+
from .rule import Rule, deferrable, rule
|
|
12
12
|
from .scope import ScopeWarning
|
|
13
13
|
from .task import Task
|
|
14
|
+
from .tokens import FileToken, OutputToken, PathToken, S3Object, ZarrStore
|
|
@@ -7,17 +7,28 @@ import itertools
|
|
|
7
7
|
|
|
8
8
|
import networkx as nx
|
|
9
9
|
|
|
10
|
-
from .exceptions import SignatureError
|
|
10
|
+
from .exceptions import Defer, SignatureError
|
|
11
|
+
from .rule import is_deferrable
|
|
11
12
|
from .task import Task
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
def build_rule_dag(rules):
|
|
15
16
|
"""Directed rule-level DAG from explicit depends_on declarations."""
|
|
17
|
+
rules_by_name = {rule.name: rule for rule in rules}
|
|
16
18
|
g = nx.DiGraph()
|
|
17
19
|
for rule in rules:
|
|
18
20
|
g.add_node(rule)
|
|
21
|
+
resolved = []
|
|
19
22
|
for dep in rule.depends_on:
|
|
23
|
+
if isinstance(dep, str):
|
|
24
|
+
if dep not in rules_by_name:
|
|
25
|
+
raise ValueError(
|
|
26
|
+
f'{rule.name}: depends_on references unknown rule {dep!r}'
|
|
27
|
+
)
|
|
28
|
+
dep = rules_by_name[dep]
|
|
29
|
+
resolved.append(dep)
|
|
20
30
|
g.add_edge(dep, rule)
|
|
31
|
+
rule.depends_on = resolved
|
|
21
32
|
if not nx.is_directed_acyclic_graph(g):
|
|
22
33
|
cycle = nx.find_cycle(g)
|
|
23
34
|
raise ValueError(f'Rule dependencies contain a cycle: {cycle}')
|
|
@@ -27,13 +38,26 @@ def build_rule_dag(rules):
|
|
|
27
38
|
def resolve_matrix(matrix):
|
|
28
39
|
"""Normalise all matrix forms to the canonical list[dict].
|
|
29
40
|
|
|
30
|
-
May raise
|
|
31
|
-
outputs do not yet exist.
|
|
41
|
+
May raise Defer if matrix is a `@deferrable` callable whose required
|
|
42
|
+
upstream outputs do not yet exist. A non-`@deferrable` matrix that raises
|
|
43
|
+
Defer is a programming error (SignatureError) — the contract must be
|
|
44
|
+
declared explicitly.
|
|
32
45
|
"""
|
|
33
46
|
if matrix is None:
|
|
34
47
|
return [{}]
|
|
35
48
|
if callable(matrix) and not isinstance(matrix, (dict, list)):
|
|
36
|
-
|
|
49
|
+
try:
|
|
50
|
+
return matrix()
|
|
51
|
+
except Defer:
|
|
52
|
+
if not is_deferrable(matrix):
|
|
53
|
+
name = getattr(matrix, '__name__', repr(matrix))
|
|
54
|
+
raise SignatureError(
|
|
55
|
+
f'matrix callable {name!r} raised Defer but is not marked '
|
|
56
|
+
f'@deferrable. Decorate it with @deferrable to declare that '
|
|
57
|
+
f'it derives its task list from upstream outputs and so may '
|
|
58
|
+
f'defer.'
|
|
59
|
+
) from None
|
|
60
|
+
raise
|
|
37
61
|
if isinstance(matrix, list):
|
|
38
62
|
return matrix
|
|
39
63
|
# dict: cartesian product of axes. A scalar key is one kwarg per value; a
|
|
@@ -68,6 +92,27 @@ def expand_rule(rule, predicate=None):
|
|
|
68
92
|
return list(iter_expand_rule(rule, predicate))
|
|
69
93
|
|
|
70
94
|
|
|
95
|
+
# Matrix kwarg values must survive the JSON round-trip used to ship SLURM
|
|
96
|
+
# job specs to compute nodes (a tuple/set comes back as a list, changing both
|
|
97
|
+
# Task.key and any kwarg-derived output path). Restrict them to JSON scalars,
|
|
98
|
+
# checked at plan time before anything is submitted. bool is an int subclass.
|
|
99
|
+
_SCALAR_TYPES = (str, int, float, bool, type(None))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _check_scalar_kwargs(rule, kwargs):
|
|
103
|
+
for key, value in kwargs.items():
|
|
104
|
+
if not isinstance(value, _SCALAR_TYPES):
|
|
105
|
+
raise SignatureError(
|
|
106
|
+
f'{rule.name}: matrix value {key}={value!r} is a '
|
|
107
|
+
f'{type(value).__name__}; matrix kwarg values must be scalars '
|
|
108
|
+
f'(str, int, float, bool, None). Non-scalars do not survive the '
|
|
109
|
+
f'JSON round-trip used for SLURM job specs (e.g. a tuple returns '
|
|
110
|
+
f'as a list), which silently changes the task key and any output '
|
|
111
|
+
f'path built from it. Encode the value as a string instead — e.g. '
|
|
112
|
+
f'a short label, or a JSON string parsed inside the rule.'
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
71
116
|
def iter_expand_rule(rule, predicate=None):
|
|
72
117
|
"""Generator form of expand_rule — yields Tasks one at a time."""
|
|
73
118
|
kwargs_list = resolve_matrix(rule.matrix)
|
|
@@ -76,6 +121,7 @@ def iter_expand_rule(rule, predicate=None):
|
|
|
76
121
|
# unknowable at decoration time for callable matrices.
|
|
77
122
|
_check_expanded_kwargs(rule, kwargs_list[0])
|
|
78
123
|
for kw in kwargs_list:
|
|
124
|
+
_check_scalar_kwargs(rule, kw)
|
|
79
125
|
if predicate is None or predicate({**kw, 'rule': rule.name}):
|
|
80
126
|
yield Task(rule=rule, kwargs=kw)
|
|
81
127
|
|
|
@@ -83,6 +129,8 @@ def iter_expand_rule(rule, predicate=None):
|
|
|
83
129
|
def _check_expanded_kwargs(rule, kwargs):
|
|
84
130
|
import inspect
|
|
85
131
|
|
|
132
|
+
from .rule import check_io_spec
|
|
133
|
+
|
|
86
134
|
names = list(inspect.signature(rule.fn).parameters)
|
|
87
135
|
expected = [n for n in names if n not in ('inputs', 'outputs')]
|
|
88
136
|
if set(expected) != set(kwargs):
|
|
@@ -90,3 +138,5 @@ def _check_expanded_kwargs(rule, kwargs):
|
|
|
90
138
|
f'{rule.name}: parameters {sorted(expected)} do not match resolved '
|
|
91
139
|
f'matrix keys {sorted(kwargs)}'
|
|
92
140
|
)
|
|
141
|
+
check_io_spec(rule.name, 'inputs', rule.inputs, set(kwargs))
|
|
142
|
+
check_io_spec(rule.name, 'outputs', rule.outputs, set(kwargs))
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
class RemakeError(Exception):
|
|
2
|
+
pass
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class RemakeLoadError(RemakeError):
|
|
6
|
+
pass
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SignatureError(RemakeError):
|
|
10
|
+
"""Rule function signature does not match its declarations."""
|
|
11
|
+
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ScopeError(RemakeError):
|
|
16
|
+
"""Rule function uses undeclared names from outer scope (strict mode)."""
|
|
17
|
+
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Defer(Exception):
|
|
22
|
+
"""Raised by a `@deferrable` matrix callable to signal that the rule
|
|
23
|
+
cannot be expanded this wave — its task list derives from an upstream
|
|
24
|
+
output that does not yet exist. A control-flow signal, not an error
|
|
25
|
+
(hence not a RemakeError): the planner defers the rule and the replan
|
|
26
|
+
loop / SLURM continuation job retries it once the upstream completes.
|
|
27
|
+
|
|
28
|
+
The planner additionally defers a `@deferrable` rule when an upstream is
|
|
29
|
+
*rerunning* this wave (its on-disk output is stale), so the matrix never
|
|
30
|
+
expands from an about-to-be-overwritten output.
|
|
31
|
+
|
|
32
|
+
Accepts path strings as context, surfaced to the user to show what is
|
|
33
|
+
blocking resolution.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(self, *paths):
|
|
37
|
+
self.paths = [str(p) for p in paths]
|
|
38
|
+
super().__init__(', '.join(self.paths) if self.paths else 'deferred')
|