remake 0.8.0a0__tar.gz → 0.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. {remake-0.8.0a0 → remake-0.8.2}/PKG-INFO +16 -6
  2. {remake-0.8.0a0 → remake-0.8.2}/README.md +8 -5
  3. {remake-0.8.0a0 → remake-0.8.2}/pyproject.toml +10 -0
  4. {remake-0.8.0a0 → remake-0.8.2}/src/remake/__init__.py +7 -2
  5. {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/__init__.py +3 -2
  6. {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/dag.py +54 -4
  7. remake-0.8.2/src/remake/core/exceptions.py +38 -0
  8. remake-0.8.2/src/remake/core/planner.py +482 -0
  9. remake-0.8.2/src/remake/core/remake.py +766 -0
  10. {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/rule.py +90 -5
  11. remake-0.8.2/src/remake/core/scope.py +313 -0
  12. {remake-0.8.0a0 → remake-0.8.2}/src/remake/core/task.py +1 -1
  13. remake-0.8.2/src/remake/executors/slurm_executor.py +562 -0
  14. {remake-0.8.0a0 → remake-0.8.2}/src/remake/metadata/__init__.py +1 -0
  15. remake-0.8.2/src/remake/metadata/metadata_manager.py +126 -0
  16. {remake-0.8.0a0 → remake-0.8.2}/src/remake/metadata/sidecar.py +17 -1
  17. remake-0.8.2/src/remake/metadata/sqlite3_backend.py +570 -0
  18. {remake-0.8.0a0 → remake-0.8.2}/src/remake/remake_cmd.py +415 -326
  19. {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/__init__.py +1 -0
  20. {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/code_compare.py +22 -4
  21. remake-0.8.2/src/remake/util/colour.py +64 -0
  22. {remake-0.8.0a0 → remake-0.8.2}/src/remake/version.py +1 -1
  23. {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/PKG-INFO +16 -6
  24. {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/SOURCES.txt +2 -1
  25. {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/requires.txt +8 -0
  26. remake-0.8.0a0/src/remake/core/exceptions.py +0 -31
  27. remake-0.8.0a0/src/remake/core/planner.py +0 -216
  28. remake-0.8.0a0/src/remake/core/remake.py +0 -227
  29. remake-0.8.0a0/src/remake/core/scope.py +0 -166
  30. remake-0.8.0a0/src/remake/executors/slurm_executor.py +0 -299
  31. remake-0.8.0a0/src/remake/metadata/metadata_manager.py +0 -49
  32. remake-0.8.0a0/src/remake/metadata/sqlite3_backend.py +0 -260
  33. {remake-0.8.0a0 → remake-0.8.2}/LICENCE +0 -0
  34. {remake-0.8.0a0 → remake-0.8.2}/setup.cfg +0 -0
  35. {remake-0.8.0a0/src/remake → remake-0.8.2/src/remake/core}/tokens.py +0 -0
  36. {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/__init__.py +0 -0
  37. {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/dask_executor.py +0 -0
  38. {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/executor.py +0 -0
  39. {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/multiproc_executor.py +0 -0
  40. {remake-0.8.0a0 → remake-0.8.2}/src/remake/executors/singleproc_executor.py +0 -0
  41. {remake-0.8.0a0 → remake-0.8.2}/src/remake/loader/__init__.py +0 -0
  42. {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/command_line_args.py +0 -0
  43. {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/config.py +0 -0
  44. {remake-0.8.0a0 → remake-0.8.2}/src/remake/util/util.py +0 -0
  45. {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/dependency_links.txt +0 -0
  46. {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/entry_points.txt +0 -0
  47. {remake-0.8.0a0 → remake-0.8.2}/src/remake.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: remake
3
- Version: 0.8.0a0
3
+ Version: 0.8.2
4
4
  Summary: Smart remake tool
5
5
  Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
6
6
  Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
@@ -233,20 +233,30 @@ Provides-Extra: s3
233
233
  Requires-Dist: boto3; extra == "s3"
234
234
  Provides-Extra: dask
235
235
  Requires-Dist: distributed>=2024.8.0; extra == "dask"
236
+ Provides-Extra: examples
237
+ Requires-Dist: xarray>=2024.7.0; extra == "examples"
238
+ Requires-Dist: netcdf4>=1.7.2; extra == "examples"
239
+ Requires-Dist: h5netcdf; extra == "examples"
240
+ Requires-Dist: zarr<3; extra == "examples"
241
+ Requires-Dist: dask; extra == "examples"
242
+ Requires-Dist: pyyaml; extra == "examples"
236
243
  Dynamic: license-file
237
244
 
238
- # remake
245
+ # remake
239
246
 
240
247
  Remake is a smart Python build tool, similar to `make`. It makes it easy to build pipelines with complex dependencies, and deploy these to high-performance computing systems. It is particularly suited to use for scientific workflows, due to its ability to reliably recreate any set of output files, based on running only those tasks that are necessary.
241
248
 
249
+ **Documentation: <https://markmuetz.github.io/remake>**
250
+
242
251
  ## Status
243
252
 
244
- remake3 is a clean-break redesign, currently being implemented. See
245
- [`design_docs/remake3_design.md`](design_docs/remake3_design.md) for the
246
- design and [`examples/`](examples) for the intended API.
253
+ remake 0.8.0 is a ground-up rewrite. See the
254
+ [changelog](https://markmuetz.github.io/remake/changelog/) for what's new and
255
+ [getting started](https://markmuetz.github.io/remake/getting-started/) for a
256
+ walkthrough.
247
257
 
248
258
  ## Installation
249
259
 
250
260
  ```bash
251
- pip install -e .
261
+ pip install remake
252
262
  ```
@@ -1,15 +1,18 @@
1
- # remake
1
+ # remake
2
2
 
3
3
  Remake is a smart Python build tool, similar to `make`. It makes it easy to build pipelines with complex dependencies, and deploy these to high-performance computing systems. It is particularly suited to use for scientific workflows, due to its ability to reliably recreate any set of output files, based on running only those tasks that are necessary.
4
4
 
5
+ **Documentation: <https://markmuetz.github.io/remake>**
6
+
5
7
  ## Status
6
8
 
7
- remake3 is a clean-break redesign, currently being implemented. See
8
- [`design_docs/remake3_design.md`](design_docs/remake3_design.md) for the
9
- design and [`examples/`](examples) for the intended API.
9
+ remake 0.8.0 is a ground-up rewrite. See the
10
+ [changelog](https://markmuetz.github.io/remake/changelog/) for what's new and
11
+ [getting started](https://markmuetz.github.io/remake/getting-started/) for a
12
+ walkthrough.
10
13
 
11
14
  ## Installation
12
15
 
13
16
  ```bash
14
- pip install -e .
17
+ pip install remake
15
18
  ```
@@ -47,6 +47,16 @@ s3 = ["boto3"]
47
47
  dask = [
48
48
  "distributed>=2024.8.0",
49
49
  ]
50
+ # Everything examples/ needs beyond the (deliberately minimal) core deps:
51
+ # `pip install "remake[examples]"` / `uv add "remake[examples]"`.
52
+ examples = [
53
+ "xarray>=2024.7.0",
54
+ "netcdf4>=1.7.2",
55
+ "h5netcdf",
56
+ "zarr<3",
57
+ "dask",
58
+ "pyyaml",
59
+ ]
50
60
 
51
61
  [dependency-groups]
52
62
  dev = [
@@ -1,14 +1,20 @@
1
1
  from .version import __version__
2
2
 
3
3
  from .core import (
4
- MatrixNotReady,
4
+ Defer,
5
+ FileToken,
6
+ OutputToken,
7
+ PathToken,
5
8
  Remake,
6
9
  RemakeError,
7
10
  Rule,
11
+ S3Object,
8
12
  ScopeError,
9
13
  ScopeWarning,
10
14
  SignatureError,
11
15
  Task,
16
+ ZarrStore,
17
+ deferrable,
12
18
  rule,
13
19
  )
14
20
  from .executors import (
@@ -20,4 +26,3 @@ from .executors import (
20
26
  )
21
27
  from .loader import load_remake
22
28
  from .metadata import MetadataManager, Sqlite3Backend, TaskRecord
23
- from .tokens import FileToken, OutputToken, PathToken, S3Object, ZarrStore
@@ -1,6 +1,6 @@
1
1
  from .dag import build_rule_dag, expand_rule, resolve_matrix
2
2
  from .exceptions import (
3
- MatrixNotReady,
3
+ Defer,
4
4
  RemakeError,
5
5
  RemakeLoadError,
6
6
  ScopeError,
@@ -8,6 +8,7 @@ from .exceptions import (
8
8
  )
9
9
  from .planner import plan
10
10
  from .remake import Remake
11
- from .rule import Rule, rule
11
+ from .rule import Rule, deferrable, rule
12
12
  from .scope import ScopeWarning
13
13
  from .task import Task
14
+ from .tokens import FileToken, OutputToken, PathToken, S3Object, ZarrStore
@@ -7,17 +7,28 @@ import itertools
7
7
 
8
8
  import networkx as nx
9
9
 
10
- from .exceptions import SignatureError
10
+ from .exceptions import Defer, SignatureError
11
+ from .rule import is_deferrable
11
12
  from .task import Task
12
13
 
13
14
 
14
15
  def build_rule_dag(rules):
15
16
  """Directed rule-level DAG from explicit depends_on declarations."""
17
+ rules_by_name = {rule.name: rule for rule in rules}
16
18
  g = nx.DiGraph()
17
19
  for rule in rules:
18
20
  g.add_node(rule)
21
+ resolved = []
19
22
  for dep in rule.depends_on:
23
+ if isinstance(dep, str):
24
+ if dep not in rules_by_name:
25
+ raise ValueError(
26
+ f'{rule.name}: depends_on references unknown rule {dep!r}'
27
+ )
28
+ dep = rules_by_name[dep]
29
+ resolved.append(dep)
20
30
  g.add_edge(dep, rule)
31
+ rule.depends_on = resolved
21
32
  if not nx.is_directed_acyclic_graph(g):
22
33
  cycle = nx.find_cycle(g)
23
34
  raise ValueError(f'Rule dependencies contain a cycle: {cycle}')
@@ -27,13 +38,26 @@ def build_rule_dag(rules):
27
38
  def resolve_matrix(matrix):
28
39
  """Normalise all matrix forms to the canonical list[dict].
29
40
 
30
- May raise MatrixNotReady if matrix is a callable whose required upstream
31
- outputs do not yet exist.
41
+ May raise Defer if matrix is a `@deferrable` callable whose required
42
+ upstream outputs do not yet exist. A non-`@deferrable` matrix that raises
43
+ Defer is a programming error (SignatureError) — the contract must be
44
+ declared explicitly.
32
45
  """
33
46
  if matrix is None:
34
47
  return [{}]
35
48
  if callable(matrix) and not isinstance(matrix, (dict, list)):
36
- return matrix()
49
+ try:
50
+ return matrix()
51
+ except Defer:
52
+ if not is_deferrable(matrix):
53
+ name = getattr(matrix, '__name__', repr(matrix))
54
+ raise SignatureError(
55
+ f'matrix callable {name!r} raised Defer but is not marked '
56
+ f'@deferrable. Decorate it with @deferrable to declare that '
57
+ f'it derives its task list from upstream outputs and so may '
58
+ f'defer.'
59
+ ) from None
60
+ raise
37
61
  if isinstance(matrix, list):
38
62
  return matrix
39
63
  # dict: cartesian product of axes. A scalar key is one kwarg per value; a
@@ -68,6 +92,27 @@ def expand_rule(rule, predicate=None):
68
92
  return list(iter_expand_rule(rule, predicate))
69
93
 
70
94
 
95
+ # Matrix kwarg values must survive the JSON round-trip used to ship SLURM
96
+ # job specs to compute nodes (a tuple/set comes back as a list, changing both
97
+ # Task.key and any kwarg-derived output path). Restrict them to JSON scalars,
98
+ # checked at plan time before anything is submitted. bool is an int subclass.
99
+ _SCALAR_TYPES = (str, int, float, bool, type(None))
100
+
101
+
102
+ def _check_scalar_kwargs(rule, kwargs):
103
+ for key, value in kwargs.items():
104
+ if not isinstance(value, _SCALAR_TYPES):
105
+ raise SignatureError(
106
+ f'{rule.name}: matrix value {key}={value!r} is a '
107
+ f'{type(value).__name__}; matrix kwarg values must be scalars '
108
+ f'(str, int, float, bool, None). Non-scalars do not survive the '
109
+ f'JSON round-trip used for SLURM job specs (e.g. a tuple returns '
110
+ f'as a list), which silently changes the task key and any output '
111
+ f'path built from it. Encode the value as a string instead — e.g. '
112
+ f'a short label, or a JSON string parsed inside the rule.'
113
+ )
114
+
115
+
71
116
  def iter_expand_rule(rule, predicate=None):
72
117
  """Generator form of expand_rule — yields Tasks one at a time."""
73
118
  kwargs_list = resolve_matrix(rule.matrix)
@@ -76,6 +121,7 @@ def iter_expand_rule(rule, predicate=None):
76
121
  # unknowable at decoration time for callable matrices.
77
122
  _check_expanded_kwargs(rule, kwargs_list[0])
78
123
  for kw in kwargs_list:
124
+ _check_scalar_kwargs(rule, kw)
79
125
  if predicate is None or predicate({**kw, 'rule': rule.name}):
80
126
  yield Task(rule=rule, kwargs=kw)
81
127
 
@@ -83,6 +129,8 @@ def iter_expand_rule(rule, predicate=None):
83
129
  def _check_expanded_kwargs(rule, kwargs):
84
130
  import inspect
85
131
 
132
+ from .rule import check_io_spec
133
+
86
134
  names = list(inspect.signature(rule.fn).parameters)
87
135
  expected = [n for n in names if n not in ('inputs', 'outputs')]
88
136
  if set(expected) != set(kwargs):
@@ -90,3 +138,5 @@ def _check_expanded_kwargs(rule, kwargs):
90
138
  f'{rule.name}: parameters {sorted(expected)} do not match resolved '
91
139
  f'matrix keys {sorted(kwargs)}'
92
140
  )
141
+ check_io_spec(rule.name, 'inputs', rule.inputs, set(kwargs))
142
+ check_io_spec(rule.name, 'outputs', rule.outputs, set(kwargs))
@@ -0,0 +1,38 @@
1
+ class RemakeError(Exception):
2
+ pass
3
+
4
+
5
+ class RemakeLoadError(RemakeError):
6
+ pass
7
+
8
+
9
+ class SignatureError(RemakeError):
10
+ """Rule function signature does not match its declarations."""
11
+
12
+ pass
13
+
14
+
15
+ class ScopeError(RemakeError):
16
+ """Rule function uses undeclared names from outer scope (strict mode)."""
17
+
18
+ pass
19
+
20
+
21
+ class Defer(Exception):
22
+ """Raised by a `@deferrable` matrix callable to signal that the rule
23
+ cannot be expanded this wave — its task list derives from an upstream
24
+ output that does not yet exist. A control-flow signal, not an error
25
+ (hence not a RemakeError): the planner defers the rule and the replan
26
+ loop / SLURM continuation job retries it once the upstream completes.
27
+
28
+ The planner additionally defers a `@deferrable` rule when an upstream is
29
+ *rerunning* this wave (its on-disk output is stale), so the matrix never
30
+ expands from an about-to-be-overwritten output.
31
+
32
+ Accepts path strings as context, surfaced to the user to show what is
33
+ blocking resolution.
34
+ """
35
+
36
+ def __init__(self, *paths):
37
+ self.paths = [str(p) for p in paths]
38
+ super().__init__(', '.join(self.paths) if self.paths else 'deferred')