remake 0.8.2__tar.gz → 0.8.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {remake-0.8.2 → remake-0.8.4}/PKG-INFO +2 -1
- {remake-0.8.2 → remake-0.8.4}/pyproject.toml +1 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/dag.py +7 -3
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/exceptions.py +17 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/planner.py +110 -15
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/remake.py +144 -30
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/rule.py +6 -1
- {remake-0.8.2 → remake-0.8.4}/src/remake/executors/dask_executor.py +11 -6
- {remake-0.8.2 → remake-0.8.4}/src/remake/executors/multiproc_executor.py +11 -6
- {remake-0.8.2 → remake-0.8.4}/src/remake/executors/singleproc_executor.py +4 -4
- {remake-0.8.2 → remake-0.8.4}/src/remake/loader/__init__.py +6 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/sqlite3_backend.py +192 -32
- {remake-0.8.2 → remake-0.8.4}/src/remake/remake_cmd.py +38 -10
- remake-0.8.4/src/remake/util/run_lock.py +77 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/version.py +1 -1
- {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/PKG-INFO +2 -1
- {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/SOURCES.txt +1 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/requires.txt +1 -0
- {remake-0.8.2 → remake-0.8.4}/LICENCE +0 -0
- {remake-0.8.2 → remake-0.8.4}/README.md +0 -0
- {remake-0.8.2 → remake-0.8.4}/setup.cfg +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/__init__.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/__init__.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/scope.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/task.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/core/tokens.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/executors/__init__.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/executors/executor.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/executors/slurm_executor.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/__init__.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/metadata_manager.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/sidecar.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/util/__init__.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/util/code_compare.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/util/colour.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/util/command_line_args.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/util/config.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake/util/util.py +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/dependency_links.txt +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/entry_points.txt +0 -0
- {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: remake
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.4
|
|
4
4
|
Summary: Smart remake tool
|
|
5
5
|
Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
|
|
6
6
|
Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
|
|
@@ -237,6 +237,7 @@ Provides-Extra: examples
|
|
|
237
237
|
Requires-Dist: xarray>=2024.7.0; extra == "examples"
|
|
238
238
|
Requires-Dist: netcdf4>=1.7.2; extra == "examples"
|
|
239
239
|
Requires-Dist: h5netcdf; extra == "examples"
|
|
240
|
+
Requires-Dist: h5py; extra == "examples"
|
|
240
241
|
Requires-Dist: zarr<3; extra == "examples"
|
|
241
242
|
Requires-Dist: dask; extra == "examples"
|
|
242
243
|
Requires-Dist: pyyaml; extra == "examples"
|
|
@@ -7,7 +7,7 @@ import itertools
|
|
|
7
7
|
|
|
8
8
|
import networkx as nx
|
|
9
9
|
|
|
10
|
-
from .exceptions import Defer, SignatureError
|
|
10
|
+
from .exceptions import Defer, RuleGraphError, SignatureError
|
|
11
11
|
from .rule import is_deferrable
|
|
12
12
|
from .task import Task
|
|
13
13
|
|
|
@@ -20,9 +20,13 @@ def build_rule_dag(rules):
|
|
|
20
20
|
g.add_node(rule)
|
|
21
21
|
resolved = []
|
|
22
22
|
for dep in rule.depends_on:
|
|
23
|
+
if not isinstance(dep, str) and getattr(dep, 'name', None) in rules_by_name:
|
|
24
|
+
# A Rule object resolved earlier may since have been replaced
|
|
25
|
+
# by a redefinition (notebook cell re-run): use the current one.
|
|
26
|
+
dep = rules_by_name[dep.name]
|
|
23
27
|
if isinstance(dep, str):
|
|
24
28
|
if dep not in rules_by_name:
|
|
25
|
-
raise
|
|
29
|
+
raise RuleGraphError(
|
|
26
30
|
f'{rule.name}: depends_on references unknown rule {dep!r}'
|
|
27
31
|
)
|
|
28
32
|
dep = rules_by_name[dep]
|
|
@@ -31,7 +35,7 @@ def build_rule_dag(rules):
|
|
|
31
35
|
rule.depends_on = resolved
|
|
32
36
|
if not nx.is_directed_acyclic_graph(g):
|
|
33
37
|
cycle = nx.find_cycle(g)
|
|
34
|
-
raise
|
|
38
|
+
raise RuleGraphError(f'Rule dependencies contain a cycle: {cycle}')
|
|
35
39
|
return g
|
|
36
40
|
|
|
37
41
|
|
|
@@ -18,6 +18,23 @@ class ScopeError(RemakeError):
|
|
|
18
18
|
pass
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
class RuleGraphError(RemakeError, ValueError):
|
|
22
|
+
"""Bad rule dependencies (unknown depends_on name, a cycle). Also a
|
|
23
|
+
ValueError, which is what these were raised as before 0.8.4."""
|
|
24
|
+
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TaskExit(Exception):
|
|
29
|
+
"""A task's code raised SystemExit (e.g. a CLI `main()` calling
|
|
30
|
+
`sys.exit`). run_task records the task as failed and raises this in its
|
|
31
|
+
place, so executors treat it as an ordinary task failure instead of the
|
|
32
|
+
whole run exiting silently. Deliberately not a RemakeError: it is a task
|
|
33
|
+
failure, not a usage error."""
|
|
34
|
+
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
21
38
|
class Defer(Exception):
|
|
22
39
|
"""Raised by a `@deferrable` matrix callable to signal that the rule
|
|
23
40
|
cannot be expanded this wave — its task list derives from an upstream
|
|
@@ -5,7 +5,11 @@ Rerun logic is DB-first: never run, failed, rule_run code changed (AST
|
|
|
5
5
|
compare), uses changed, or any relevant upstream task reruns. Filesystem
|
|
6
6
|
checks happen only via the opt-in check_outputs modes.
|
|
7
7
|
"""
|
|
8
|
+
import ast
|
|
9
|
+
import builtins
|
|
8
10
|
import difflib
|
|
11
|
+
import inspect
|
|
12
|
+
import os
|
|
9
13
|
from collections import namedtuple
|
|
10
14
|
from time import perf_counter
|
|
11
15
|
|
|
@@ -15,7 +19,7 @@ from loguru import logger
|
|
|
15
19
|
from ..metadata.metadata_manager import TASK_STATUS_FAILED, TASK_STATUS_SUCCESS
|
|
16
20
|
from ..util.code_compare import CodeComparer
|
|
17
21
|
from .dag import expand_rule
|
|
18
|
-
from .exceptions import Defer
|
|
22
|
+
from .exceptions import Defer, RemakeError
|
|
19
23
|
from .rule import is_deferrable
|
|
20
24
|
from .scope import (
|
|
21
25
|
io_hash,
|
|
@@ -27,20 +31,75 @@ from .scope import (
|
|
|
27
31
|
)
|
|
28
32
|
|
|
29
33
|
|
|
30
|
-
|
|
34
|
+
# Builtins a query may call (review 2026-09-24 M14): with none, reasonable
|
|
35
|
+
# queries like "year in range(2000, 2005)" silently matched nothing.
|
|
36
|
+
QUERY_BUILTINS = {
|
|
37
|
+
name: getattr(builtins, name)
|
|
38
|
+
for name in ('abs', 'all', 'any', 'bool', 'float', 'frozenset', 'int', 'len',
|
|
39
|
+
'list', 'max', 'min', 'range', 'round', 'set', 'sorted', 'str', 'tuple')
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def rule_kwarg_names(rules):
|
|
44
|
+
"""Every matrix kwarg name across `rules` — the rule functions'
|
|
45
|
+
parameters after inputs/outputs (the signature contract makes these the
|
|
46
|
+
matrix keys, for callable matrices too)."""
|
|
47
|
+
names = set()
|
|
48
|
+
for rule in rules:
|
|
49
|
+
names.update(p for p in inspect.signature(rule.fn).parameters
|
|
50
|
+
if p not in ('inputs', 'outputs'))
|
|
51
|
+
return names
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _free_names(tree):
|
|
55
|
+
"""Names a query expression reads, minus those it binds itself
|
|
56
|
+
(comprehension targets, lambda arguments)."""
|
|
57
|
+
loaded, bound = set(), set()
|
|
58
|
+
for node in ast.walk(tree):
|
|
59
|
+
if isinstance(node, ast.Name):
|
|
60
|
+
(loaded if isinstance(node.ctx, ast.Load) else bound).add(node.id)
|
|
61
|
+
elif isinstance(node, ast.arg):
|
|
62
|
+
bound.add(node.arg)
|
|
63
|
+
return loaded - bound
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def make_predicate(query, known_names=None):
|
|
31
67
|
"""Compile a task-filter expression evaluated against task kwargs plus
|
|
32
68
|
'rule' (the rule name), e.g. "year > 1985 and model == 'era5'",
|
|
33
|
-
"rule in ['extract', 'clean']".
|
|
69
|
+
"rule in ['extract', 'clean']". A few safe builtins are available
|
|
70
|
+
(QUERY_BUILTINS).
|
|
71
|
+
|
|
72
|
+
`known_names` (every kwarg name in the pipeline, see rule_kwarg_names)
|
|
73
|
+
turns a name no rule has — almost always a typo — into an error instead
|
|
74
|
+
of a silent no-match. Bad syntax and evaluation errors are RemakeErrors
|
|
75
|
+
(a clean CLI error) rather than raw tracebacks."""
|
|
34
76
|
# MM: this looks like a risk - using compile to compile the code?
|
|
35
77
|
# See how pyquerylist does this - it only allows certain Python ops.
|
|
36
|
-
|
|
37
|
-
|
|
78
|
+
try:
|
|
79
|
+
tree = ast.parse(query, mode='eval')
|
|
80
|
+
code = compile(tree, '<query>', 'eval')
|
|
81
|
+
except SyntaxError as e:
|
|
82
|
+
raise RemakeError(f'invalid query {query!r}: {e.msg}') from None
|
|
83
|
+
if known_names is not None:
|
|
84
|
+
unknown = _free_names(tree) - set(known_names) - {'rule'} - QUERY_BUILTINS.keys()
|
|
85
|
+
if unknown:
|
|
86
|
+
raise RemakeError(
|
|
87
|
+
f'query {query!r} uses unknown name(s) {sorted(unknown)} — not a '
|
|
88
|
+
f'matrix key of any rule (keys: {sorted(known_names)}), "rule", or an '
|
|
89
|
+
f'allowed builtin ({", ".join(sorted(QUERY_BUILTINS))})'
|
|
90
|
+
)
|
|
38
91
|
def predicate(kwargs):
|
|
39
92
|
try:
|
|
40
|
-
|
|
93
|
+
# kwargs go in the *globals*: a generator expression (e.g. inside
|
|
94
|
+
# any()) has its own scope and can't see eval's locals.
|
|
95
|
+
return bool(eval(code, {**kwargs, '__builtins__': QUERY_BUILTINS}))
|
|
41
96
|
except NameError:
|
|
42
97
|
# Query references a kwarg this rule doesn't have: no match.
|
|
43
98
|
return False
|
|
99
|
+
except Exception as e:
|
|
100
|
+
raise RemakeError(
|
|
101
|
+
f'query {query!r} failed on {dict(kwargs)}: {type(e).__name__}: {e}'
|
|
102
|
+
) from None
|
|
44
103
|
|
|
45
104
|
return predicate
|
|
46
105
|
|
|
@@ -124,22 +183,53 @@ def cascade_settled(rule_set, dag, selected, run_seq, status):
|
|
|
124
183
|
return settled
|
|
125
184
|
|
|
126
185
|
|
|
186
|
+
# Key in an executor's `failures` dict holding the normalised output paths of
|
|
187
|
+
# every task that failed (or was skipped) this run.
|
|
188
|
+
_FAILED_OUTPUTS = '_failed_outputs'
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _norm_paths(values):
|
|
192
|
+
return {os.path.normpath(str(v)) for v in values}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _io_paths(task, part):
|
|
196
|
+
"""A task's normalised input or output paths, or an empty set if its
|
|
197
|
+
callable spec raises — run_task has already recorded that failure, and
|
|
198
|
+
the executor's bookkeeping must not re-raise it (and crash the run)."""
|
|
199
|
+
try:
|
|
200
|
+
return _norm_paths(getattr(task, part).values())
|
|
201
|
+
except Exception:
|
|
202
|
+
return set()
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def record_failure(failures, task):
|
|
206
|
+
"""Record a failed (or skipped-for-upstream-failure) task in an
|
|
207
|
+
executor's `failures` dict, for upstream_failed."""
|
|
208
|
+
failures.setdefault(task.rule, set()).add(frozenset(task.kwargs.items()))
|
|
209
|
+
failures.setdefault(_FAILED_OUTPUTS, set()).update(_io_paths(task, 'outputs'))
|
|
210
|
+
|
|
211
|
+
|
|
127
212
|
def upstream_failed(task, failures):
|
|
128
213
|
"""Should task be skipped because upstream tasks failed this run?
|
|
129
214
|
|
|
130
|
-
failures:
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
215
|
+
failures: built by record_failure. Mirrors the planner's rerun
|
|
216
|
+
propagation — element-wise when the matrices are shared, conservative
|
|
217
|
+
(any failure taints all downstream tasks) otherwise — and additionally
|
|
218
|
+
skips any task whose declared inputs include an output of a failed task.
|
|
219
|
+
A shared matrix doesn't mean each task reads only its same-kwargs
|
|
220
|
+
upstream (e.g. inputs of year-1): without the path check such a task ran
|
|
221
|
+
on its failed upstream's stale or missing output (review 2026-09-24 H3,
|
|
222
|
+
failure-skip part; the full fix is 0.9).
|
|
134
223
|
"""
|
|
135
224
|
for dep in task.rule.depends_on:
|
|
136
225
|
failed = failures.get(dep)
|
|
137
226
|
if not failed:
|
|
138
227
|
continue
|
|
139
|
-
if _same_matrix(task.rule, dep):
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
228
|
+
if not _same_matrix(task.rule, dep):
|
|
229
|
+
return True
|
|
230
|
+
if frozenset(task.kwargs.items()) in failed:
|
|
231
|
+
return True
|
|
232
|
+
if _io_paths(task, 'inputs') & failures.get(_FAILED_OUTPUTS, set()):
|
|
143
233
|
return True
|
|
144
234
|
return False
|
|
145
235
|
|
|
@@ -323,9 +413,10 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
|
|
|
323
413
|
# MM: this is a core piece of logic, but I find it hard to understand end-to-end.
|
|
324
414
|
# MM: also quite a long func.
|
|
325
415
|
start = perf_counter()
|
|
326
|
-
predicate = make_predicate(query) if query else None
|
|
416
|
+
predicate = make_predicate(query, rule_kwarg_names(rules)) if query else None
|
|
327
417
|
code_comparer = CodeComparer()
|
|
328
418
|
rules = set(rules)
|
|
419
|
+
nmatched = 0
|
|
329
420
|
|
|
330
421
|
runnable = []
|
|
331
422
|
deferred = []
|
|
@@ -469,8 +560,12 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
|
|
|
469
560
|
(records[task.key].run_seq if task.key in records else None)
|
|
470
561
|
for task in tasks
|
|
471
562
|
}
|
|
563
|
+
nmatched += len(tasks)
|
|
472
564
|
logger.debug('{}: {} task(s), {} to rerun', rule.name, len(tasks), len(rule_rerun))
|
|
473
565
|
|
|
566
|
+
if predicate is not None and not nmatched and not deferred:
|
|
567
|
+
# Distinguish "your query selected nothing" from "all up to date".
|
|
568
|
+
logger.warning(f'query {query!r} matched no tasks')
|
|
474
569
|
elapsed = perf_counter() - start
|
|
475
570
|
logger.bind(
|
|
476
571
|
event='plan', nrunnable=len(runnable), ndeferred=len(deferred),
|
|
@@ -5,6 +5,7 @@ from collections import Counter
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from time import perf_counter
|
|
7
7
|
|
|
8
|
+
import networkx as nx
|
|
8
9
|
from loguru import logger
|
|
9
10
|
|
|
10
11
|
from ..metadata.metadata_manager import (
|
|
@@ -14,9 +15,16 @@ from ..metadata.metadata_manager import (
|
|
|
14
15
|
RecordCache,
|
|
15
16
|
)
|
|
16
17
|
from ..util import task_log_path
|
|
18
|
+
from ..util.run_lock import run_lock
|
|
17
19
|
from .dag import build_rule_dag, expand_rule, iter_expand_rule
|
|
18
|
-
from .exceptions import Defer, RemakeError
|
|
19
|
-
from .planner import
|
|
20
|
+
from .exceptions import Defer, RemakeError, TaskExit
|
|
21
|
+
from .planner import (
|
|
22
|
+
cascade_settled,
|
|
23
|
+
explain_task,
|
|
24
|
+
make_predicate,
|
|
25
|
+
plan,
|
|
26
|
+
rule_kwarg_names,
|
|
27
|
+
)
|
|
20
28
|
from .rule import Rule
|
|
21
29
|
from .scope import check_scope, exec_function
|
|
22
30
|
from .task import Task
|
|
@@ -45,6 +53,39 @@ class _TemplatePlaceholder:
|
|
|
45
53
|
return str(self)
|
|
46
54
|
|
|
47
55
|
|
|
56
|
+
def _blocked_reason(rule):
|
|
57
|
+
"""Why a rule is still deferred when a local run has nothing left to
|
|
58
|
+
try: its matrix's Defer paths if it still can't expand, else (it would
|
|
59
|
+
expand, so the planner is holding it back) an upstream that didn't
|
|
60
|
+
complete this run."""
|
|
61
|
+
from .rule import is_deferrable
|
|
62
|
+
|
|
63
|
+
if is_deferrable(rule.matrix):
|
|
64
|
+
try:
|
|
65
|
+
rule.matrix()
|
|
66
|
+
except Defer as e:
|
|
67
|
+
waiting = ', '.join(e.paths) if e.paths else 'an unspecified input'
|
|
68
|
+
return f'matrix not ready — waiting on {waiting}'
|
|
69
|
+
except Exception as e: # the matrix itself is broken: say so
|
|
70
|
+
return f'matrix raised {type(e).__name__}: {e}'
|
|
71
|
+
return 'an upstream rule did not complete (see failures above)'
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _where(rule):
|
|
75
|
+
fn = rule.fn
|
|
76
|
+
return f'{getattr(fn, "__module__", "?")}.{getattr(fn, "__qualname__", rule.name)}'
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _same_definition(a, b):
|
|
80
|
+
"""Two Rule objects for the same function re-defined (e.g. a notebook
|
|
81
|
+
cell re-run): same module and qualified name, but freshly compiled code.
|
|
82
|
+
Distinct functions that merely share a rule name (two modules each
|
|
83
|
+
defining `process`, a clashing name=) are not — nor are rules built by
|
|
84
|
+
one factory function, which share a single code object and differ only
|
|
85
|
+
in closure values."""
|
|
86
|
+
return _where(a) == _where(b) and a.fn.__code__ is not b.fn.__code__
|
|
87
|
+
|
|
88
|
+
|
|
48
89
|
class Remake:
|
|
49
90
|
def __init__(
|
|
50
91
|
self,
|
|
@@ -58,6 +99,10 @@ class Remake:
|
|
|
58
99
|
self.config = config or {}
|
|
59
100
|
self.metadata = metadata
|
|
60
101
|
self.check_outputs = check_outputs
|
|
102
|
+
# Rules the last run() left unresolved (deferred matrix never ready,
|
|
103
|
+
# or waiting on an upstream that failed). Non-empty means that run
|
|
104
|
+
# did not complete everything; the CLI exits non-zero on it.
|
|
105
|
+
self.blocked_rules = []
|
|
61
106
|
self.strict_scope = strict_scope
|
|
62
107
|
self.rules = []
|
|
63
108
|
self.dag = None
|
|
@@ -74,11 +119,26 @@ class Remake:
|
|
|
74
119
|
raise RemakeError(f'Not a Rule (use the @rule decorator): {rule!r}')
|
|
75
120
|
if rule in self.rules:
|
|
76
121
|
continue
|
|
122
|
+
# Task keys are sha1('<rule name>:<kwargs>'), so two rules sharing
|
|
123
|
+
# a name share task keys and DB records (review 2026-09-24 H8).
|
|
124
|
+
existing = next((r for r in self.rules if r.name == rule.name), None)
|
|
125
|
+
if existing is not None and not _same_definition(existing, rule):
|
|
126
|
+
raise RemakeError(
|
|
127
|
+
f'Duplicate rule name {rule.name!r}: defined by '
|
|
128
|
+
f'{_where(existing)} and {_where(rule)}. Rule names must be '
|
|
129
|
+
f'unique within a pipeline (rename one, or pass name=...)'
|
|
130
|
+
)
|
|
77
131
|
# Resolve tri-state strict_scope against the Remake default.
|
|
78
132
|
if rule.strict_scope is None and self.strict_scope:
|
|
79
133
|
check_scope(rule.fn, rule.uses, strict=True)
|
|
80
134
|
rule.remake = self
|
|
81
|
-
|
|
135
|
+
if existing is not None:
|
|
136
|
+
# The same function re-defined — a notebook cell or script
|
|
137
|
+
# section executed again: the new definition replaces the old.
|
|
138
|
+
logger.warning(f'Rule {rule.name!r} redefined: replacing the earlier definition')
|
|
139
|
+
self.rules[self.rules.index(existing)] = rule
|
|
140
|
+
else:
|
|
141
|
+
self.rules.append(rule)
|
|
82
142
|
self._finalized = False
|
|
83
143
|
|
|
84
144
|
def rules_from_current_module(self):
|
|
@@ -186,7 +246,7 @@ class Remake:
|
|
|
186
246
|
dynamic matrix is deferred."""
|
|
187
247
|
if not self._finalized:
|
|
188
248
|
self.finalize()
|
|
189
|
-
predicate = make_predicate(query) if query else None
|
|
249
|
+
predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
|
|
190
250
|
for rule in self.rules:
|
|
191
251
|
try:
|
|
192
252
|
yield from iter_expand_rule(rule, predicate)
|
|
@@ -285,7 +345,7 @@ class Remake:
|
|
|
285
345
|
remaining = Counter(task.rule.name for task in runnable)
|
|
286
346
|
runnable_keys = {task.key for task in runnable}
|
|
287
347
|
deferred_names = {rule.name for rule in deferred}
|
|
288
|
-
predicate = make_predicate(query) if query else None
|
|
348
|
+
predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
|
|
289
349
|
|
|
290
350
|
# Per-rule tally of why the to-run tasks would rerun. One plan() is
|
|
291
351
|
# already done (`runnable`); reuse it per task so this is plan-cost,
|
|
@@ -669,9 +729,21 @@ class Remake:
|
|
|
669
729
|
"""Run all tasks that need running, replanning after each wave so
|
|
670
730
|
dynamic (deferred) matrices resolve as their upstreams complete.
|
|
671
731
|
Returns the number of failed tasks (0 for asynchronous executors,
|
|
672
|
-
which don't know at submission time).
|
|
732
|
+
which don't know at submission time). Holds `.remake/run.lock` for
|
|
733
|
+
the duration: a second concurrent run in the same directory is a
|
|
734
|
+
RemakeError."""
|
|
673
735
|
if not self._finalized:
|
|
674
736
|
self.finalize()
|
|
737
|
+
self.blocked_rules = []
|
|
738
|
+
if executor is not None and executor.handles_deferred:
|
|
739
|
+
# SLURM: submission only; continuation jobs re-enter run() on
|
|
740
|
+
# compute nodes, and the executor has its own duplicate-submission
|
|
741
|
+
# guard — a directory lock would strand or block them.
|
|
742
|
+
return self._run(executor, query, force, ignore_code_changes)
|
|
743
|
+
with run_lock(self.metadata):
|
|
744
|
+
return self._run(executor, query, force, ignore_code_changes)
|
|
745
|
+
|
|
746
|
+
def _run(self, executor, query, force, ignore_code_changes):
|
|
675
747
|
# One run_seq for this whole invocation (shared across replanning
|
|
676
748
|
# waves); committed onto every task so downstream propagation survives
|
|
677
749
|
# to later invocations. See bugs/01_durable_rerun_propagation.md.
|
|
@@ -705,8 +777,19 @@ class Remake:
|
|
|
705
777
|
runnable = [t for t in runnable if t.key not in attempted]
|
|
706
778
|
if not runnable:
|
|
707
779
|
if deferred:
|
|
708
|
-
|
|
709
|
-
|
|
780
|
+
# Not everything completed: say which rules and why, and
|
|
781
|
+
# make the run's outcome reflect it (review 2026-09-24 M9;
|
|
782
|
+
# it used to be a bare warning and exit 0). A filtered run
|
|
783
|
+
# may deliberately leave out the upstream a deferred rule
|
|
784
|
+
# waits on: that one is reported but doesn't fail the run.
|
|
785
|
+
excused = self._excluded_by_query(deferred) if query else set()
|
|
786
|
+
for rule in deferred:
|
|
787
|
+
if rule in excused:
|
|
788
|
+
logger.warning(f'Blocked rule {rule.name} (upstream not '
|
|
789
|
+
f'selected by the query): {_blocked_reason(rule)}')
|
|
790
|
+
else:
|
|
791
|
+
self.blocked_rules.append(rule)
|
|
792
|
+
logger.error(f'Blocked rule {rule.name}: {_blocked_reason(rule)}')
|
|
710
793
|
break
|
|
711
794
|
wave += 1
|
|
712
795
|
logger.bind(event='wave', wave=wave, ntasks=len(runnable)).debug(
|
|
@@ -724,40 +807,71 @@ class Remake:
|
|
|
724
807
|
logger.info('Nothing to do')
|
|
725
808
|
return nfailed
|
|
726
809
|
|
|
810
|
+
def _excluded_by_query(self, deferred):
|
|
811
|
+
"""Deferred rules a filtered run can be excused for: those with an
|
|
812
|
+
upstream (any ancestor) that still has work outside the filter. If
|
|
813
|
+
every ancestor is complete and the rule is still deferred, the query
|
|
814
|
+
didn't cause it — it's genuinely blocked."""
|
|
815
|
+
runnable_all, deferred_all = self.plan()
|
|
816
|
+
unfinished = {t.rule for t in runnable_all} | set(deferred_all)
|
|
817
|
+
return {rule for rule in deferred
|
|
818
|
+
if nx.ancestors(self.dag, rule) & (unfinished - {rule})}
|
|
819
|
+
|
|
727
820
|
def run_task(self, task):
|
|
728
821
|
"""Execute one task and record the result. The single execution
|
|
729
822
|
entry point — used by all executors and `remake run-task`. Timing and
|
|
730
823
|
completion are logged here so every executor gets them uniformly
|
|
731
824
|
(per-element detail at TRACE, per-task duration at DEBUG — the
|
|
732
825
|
summarise-loops convention, per_task_logging.md)."""
|
|
733
|
-
# opt(lazy=True): the path lists are only built when a TRACE sink is
|
|
734
|
-
# attached (they'd cost real time at 1e6 tasks otherwise).
|
|
735
|
-
logger.opt(lazy=True).trace(
|
|
736
|
-
'running {}: inputs {} -> outputs {}', lambda: task,
|
|
737
|
-
lambda: [str(p) for p in task.inputs.values()],
|
|
738
|
-
lambda: [str(p) for p in task.outputs.values()],
|
|
739
|
-
)
|
|
740
|
-
for token in task.outputs.values():
|
|
741
|
-
if hasattr(token, '__fspath__'):
|
|
742
|
-
Path(token).parent.mkdir(parents=True, exist_ok=True)
|
|
743
|
-
|
|
744
|
-
fn = exec_function(task.rule.fn, task.rule.uses)
|
|
745
|
-
args = []
|
|
746
|
-
if task.rule.inputs is not None:
|
|
747
|
-
args.append(task.inputs)
|
|
748
|
-
if task.rule.outputs is not None:
|
|
749
|
-
args.append(task.outputs)
|
|
750
826
|
start = perf_counter()
|
|
827
|
+
# Everything that can fail — output dirs, io resolution (callable
|
|
828
|
+
# specs run here), the function itself — is inside the try, so every
|
|
829
|
+
# failure is recorded with its traceback (review 2026-09-24 M11).
|
|
751
830
|
try:
|
|
752
|
-
|
|
753
|
-
|
|
831
|
+
# opt(lazy=True): the path lists are only built when a TRACE sink
|
|
832
|
+
# is attached (they'd cost real time at 1e6 tasks otherwise).
|
|
833
|
+
logger.opt(lazy=True).trace(
|
|
834
|
+
'running {}: inputs {} -> outputs {}', lambda: task,
|
|
835
|
+
lambda: [str(p) for p in task.inputs.values()],
|
|
836
|
+
lambda: [str(p) for p in task.outputs.values()],
|
|
837
|
+
)
|
|
838
|
+
for token in task.outputs.values():
|
|
839
|
+
if hasattr(token, '__fspath__'):
|
|
840
|
+
Path(token).parent.mkdir(parents=True, exist_ok=True)
|
|
841
|
+
|
|
842
|
+
fn = exec_function(task.rule.fn, task.rule.uses)
|
|
843
|
+
args = []
|
|
844
|
+
if task.rule.inputs is not None:
|
|
845
|
+
args.append(task.inputs)
|
|
846
|
+
if task.rule.outputs is not None:
|
|
847
|
+
args.append(task.outputs)
|
|
848
|
+
try:
|
|
849
|
+
fn(*args, **task.kwargs)
|
|
850
|
+
except SystemExit as exc:
|
|
851
|
+
# sys.exit(0) / sys.exit() from a wrapped CLI main() means
|
|
852
|
+
# success; only a non-zero code is a failure (handled below).
|
|
853
|
+
# Mirror CPython: only None or an int 0 (incl. False) exits 0;
|
|
854
|
+
# sys.exit(0.0) or sys.exit('msg') exit 1.
|
|
855
|
+
if not (exc.code is None or (isinstance(exc.code, int) and exc.code == 0)):
|
|
856
|
+
raise
|
|
857
|
+
except (Exception, SystemExit) as exc:
|
|
858
|
+
# SystemExit from task code (a CLI main() calling sys.exit) is a
|
|
859
|
+
# task failure, not a request to end the run (review H6).
|
|
860
|
+
# KeyboardInterrupt still propagates: Ctrl-C stops the run.
|
|
754
861
|
elapsed = perf_counter() - start
|
|
862
|
+
tb = traceback.format_exc()
|
|
755
863
|
logger.bind(event='task_failed', task=str(task), rule=task.rule.name,
|
|
756
864
|
key=task.key, seconds=round(elapsed, 6),
|
|
757
865
|
).error(f'failed: {task} after {elapsed:.2f}s')
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
)
|
|
866
|
+
# The traceback at DEBUG: lands in per-task logs (DEBUG sinks)
|
|
867
|
+
# without flooding the INFO console on runs with many failures
|
|
868
|
+
# (review M12).
|
|
869
|
+
logger.debug('traceback for {}:\n{}', task, tb)
|
|
870
|
+
self.metadata.update_task(task, TASK_STATUS_FAILED, exception=tb)
|
|
871
|
+
if isinstance(exc, SystemExit):
|
|
872
|
+
raise TaskExit(
|
|
873
|
+
f'{task} called sys.exit({exc.code!r})'
|
|
874
|
+
) from exc
|
|
761
875
|
raise
|
|
762
876
|
elapsed = perf_counter() - start
|
|
763
877
|
logger.bind(event='task_complete', task=str(task), rule=task.rule.name,
|
|
@@ -216,7 +216,12 @@ def rule(
|
|
|
216
216
|
inputs=inputs,
|
|
217
217
|
outputs=outputs,
|
|
218
218
|
matrix=matrix,
|
|
219
|
-
|
|
219
|
+
# A bare name or Rule is one dependency — list('extract') would
|
|
220
|
+
# split it into characters (review 2026-09-24 L4).
|
|
221
|
+
depends_on=(
|
|
222
|
+
[depends_on] if isinstance(depends_on, (str, Rule))
|
|
223
|
+
else list(depends_on) if depends_on else []
|
|
224
|
+
),
|
|
220
225
|
uses=uses_,
|
|
221
226
|
strict_scope=strict_scope,
|
|
222
227
|
config=dict(config) if config else {},
|
|
@@ -23,13 +23,13 @@ import os
|
|
|
23
23
|
from loguru import logger
|
|
24
24
|
|
|
25
25
|
from ..core.exceptions import RemakeError
|
|
26
|
-
from ..core.planner import upstream_failed
|
|
26
|
+
from ..core.planner import record_failure, upstream_failed
|
|
27
27
|
from .executor import Executor
|
|
28
28
|
|
|
29
29
|
_worker_rmk_cache = {}
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def _run_spec(remakefile, rule_name, kwargs):
|
|
32
|
+
def _run_spec(remakefile, rule_name, kwargs, run_seq=None):
|
|
33
33
|
"""Runs on a dask worker. Returns True on success."""
|
|
34
34
|
from ..loader import load_remake
|
|
35
35
|
from ..metadata.sidecar import SidecarWriter
|
|
@@ -40,6 +40,9 @@ def _run_spec(remakefile, rule_name, kwargs):
|
|
|
40
40
|
rmk = load_remake(remakefile, finalize=False)
|
|
41
41
|
rmk.metadata = SidecarWriter()
|
|
42
42
|
_worker_rmk_cache[remakefile] = rmk
|
|
43
|
+
# The parent invocation's run_seq (durable propagation, bugs/01); set per
|
|
44
|
+
# call because long-lived workers outlive a single invocation.
|
|
45
|
+
rmk.metadata.run_seq = run_seq
|
|
43
46
|
task = rmk.task_from_spec(rule_name, kwargs)
|
|
44
47
|
logfile = task_log_path(task)
|
|
45
48
|
logfile.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -95,14 +98,15 @@ class DaskExecutor(Executor):
|
|
|
95
98
|
nfailed = 0
|
|
96
99
|
nskipped = 0
|
|
97
100
|
done = 0
|
|
98
|
-
failures = {} #
|
|
101
|
+
failures = {} # see planner.record_failure
|
|
102
|
+
run_seq = self.rmk.metadata.current_run_seq()
|
|
99
103
|
client, cluster = self._client()
|
|
100
104
|
try:
|
|
101
105
|
for rule, rule_tasks in groups:
|
|
102
106
|
to_run = []
|
|
103
107
|
for task in rule_tasks:
|
|
104
108
|
if upstream_failed(task, failures):
|
|
105
|
-
failures
|
|
109
|
+
record_failure(failures, task)
|
|
106
110
|
nskipped += 1
|
|
107
111
|
done += 1
|
|
108
112
|
logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
|
|
@@ -113,7 +117,8 @@ class DaskExecutor(Executor):
|
|
|
113
117
|
logger.info(f'{rule.name}: {len(to_run)} task(s) on dask ({self.nproc} workers)')
|
|
114
118
|
futures = {
|
|
115
119
|
client.submit(
|
|
116
|
-
_run_spec, self.remakefile, rule.name, task.kwargs,
|
|
120
|
+
_run_spec, self.remakefile, rule.name, task.kwargs, run_seq,
|
|
121
|
+
pure=False,
|
|
117
122
|
): task
|
|
118
123
|
for task in to_run
|
|
119
124
|
}
|
|
@@ -124,7 +129,7 @@ class DaskExecutor(Executor):
|
|
|
124
129
|
if future.result():
|
|
125
130
|
logger.info(f'{done}/{ntasks}: {task}')
|
|
126
131
|
else:
|
|
127
|
-
failures
|
|
132
|
+
record_failure(failures, task)
|
|
128
133
|
nfailed += 1
|
|
129
134
|
logger.error(f'{done}/{ntasks} failed: {task}')
|
|
130
135
|
finally:
|
|
@@ -22,7 +22,7 @@ from multiprocessing import get_context
|
|
|
22
22
|
from loguru import logger
|
|
23
23
|
|
|
24
24
|
from ..core.exceptions import RemakeError
|
|
25
|
-
from ..core.planner import upstream_failed
|
|
25
|
+
from ..core.planner import record_failure, upstream_failed
|
|
26
26
|
from .executor import Executor
|
|
27
27
|
|
|
28
28
|
_worker_rmk = None
|
|
@@ -53,7 +53,10 @@ def _worker_init(remakefile):
|
|
|
53
53
|
def _worker_run(spec):
|
|
54
54
|
from ..util import task_log_path
|
|
55
55
|
|
|
56
|
-
rule_name, kwargs = spec
|
|
56
|
+
rule_name, kwargs, run_seq = spec
|
|
57
|
+
# Stamp results with the parent invocation's run_seq so durable rerun
|
|
58
|
+
# propagation (bugs/01) works for multiproc runs as it does for SLURM.
|
|
59
|
+
_worker_rmk.metadata.run_seq = run_seq
|
|
57
60
|
task = _worker_rmk.task_from_spec(rule_name, kwargs)
|
|
58
61
|
logfile = task_log_path(task)
|
|
59
62
|
logfile.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -93,7 +96,8 @@ class MultiprocExecutor(Executor):
|
|
|
93
96
|
nfailed = 0
|
|
94
97
|
nskipped = 0
|
|
95
98
|
done = 0
|
|
96
|
-
failures = {} #
|
|
99
|
+
failures = {} # see planner.record_failure
|
|
100
|
+
run_seq = self.rmk.metadata.current_run_seq()
|
|
97
101
|
with ProcessPoolExecutor(
|
|
98
102
|
max_workers=self.nproc,
|
|
99
103
|
mp_context=get_context('spawn'),
|
|
@@ -107,7 +111,7 @@ class MultiprocExecutor(Executor):
|
|
|
107
111
|
to_run = []
|
|
108
112
|
for task in rule_tasks:
|
|
109
113
|
if upstream_failed(task, failures):
|
|
110
|
-
failures
|
|
114
|
+
record_failure(failures, task)
|
|
111
115
|
nskipped += 1
|
|
112
116
|
done += 1
|
|
113
117
|
logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
|
|
@@ -117,7 +121,8 @@ class MultiprocExecutor(Executor):
|
|
|
117
121
|
continue
|
|
118
122
|
logger.info(f'{rule.name}: {len(to_run)} task(s) on {self.nproc} proc(s)')
|
|
119
123
|
futures = {
|
|
120
|
-
pool.submit(_worker_run, (rule.name, t.kwargs)): t
|
|
124
|
+
pool.submit(_worker_run, (rule.name, t.kwargs, run_seq)): t
|
|
125
|
+
for t in to_run
|
|
121
126
|
}
|
|
122
127
|
# Barrier: drain this rule before starting the next.
|
|
123
128
|
for future in as_completed(futures):
|
|
@@ -126,7 +131,7 @@ class MultiprocExecutor(Executor):
|
|
|
126
131
|
if future.result():
|
|
127
132
|
logger.info(f'{done}/{ntasks}: {task}')
|
|
128
133
|
else:
|
|
129
|
-
failures
|
|
134
|
+
record_failure(failures, task)
|
|
130
135
|
nfailed += 1
|
|
131
136
|
logger.error(f'{done}/{ntasks} failed: {task}')
|
|
132
137
|
if nfailed:
|