remake 0.8.2__tar.gz → 0.8.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {remake-0.8.2 → remake-0.8.4}/PKG-INFO +2 -1
  2. {remake-0.8.2 → remake-0.8.4}/pyproject.toml +1 -0
  3. {remake-0.8.2 → remake-0.8.4}/src/remake/core/dag.py +7 -3
  4. {remake-0.8.2 → remake-0.8.4}/src/remake/core/exceptions.py +17 -0
  5. {remake-0.8.2 → remake-0.8.4}/src/remake/core/planner.py +110 -15
  6. {remake-0.8.2 → remake-0.8.4}/src/remake/core/remake.py +144 -30
  7. {remake-0.8.2 → remake-0.8.4}/src/remake/core/rule.py +6 -1
  8. {remake-0.8.2 → remake-0.8.4}/src/remake/executors/dask_executor.py +11 -6
  9. {remake-0.8.2 → remake-0.8.4}/src/remake/executors/multiproc_executor.py +11 -6
  10. {remake-0.8.2 → remake-0.8.4}/src/remake/executors/singleproc_executor.py +4 -4
  11. {remake-0.8.2 → remake-0.8.4}/src/remake/loader/__init__.py +6 -0
  12. {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/sqlite3_backend.py +192 -32
  13. {remake-0.8.2 → remake-0.8.4}/src/remake/remake_cmd.py +38 -10
  14. remake-0.8.4/src/remake/util/run_lock.py +77 -0
  15. {remake-0.8.2 → remake-0.8.4}/src/remake/version.py +1 -1
  16. {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/PKG-INFO +2 -1
  17. {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/SOURCES.txt +1 -0
  18. {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/requires.txt +1 -0
  19. {remake-0.8.2 → remake-0.8.4}/LICENCE +0 -0
  20. {remake-0.8.2 → remake-0.8.4}/README.md +0 -0
  21. {remake-0.8.2 → remake-0.8.4}/setup.cfg +0 -0
  22. {remake-0.8.2 → remake-0.8.4}/src/remake/__init__.py +0 -0
  23. {remake-0.8.2 → remake-0.8.4}/src/remake/core/__init__.py +0 -0
  24. {remake-0.8.2 → remake-0.8.4}/src/remake/core/scope.py +0 -0
  25. {remake-0.8.2 → remake-0.8.4}/src/remake/core/task.py +0 -0
  26. {remake-0.8.2 → remake-0.8.4}/src/remake/core/tokens.py +0 -0
  27. {remake-0.8.2 → remake-0.8.4}/src/remake/executors/__init__.py +0 -0
  28. {remake-0.8.2 → remake-0.8.4}/src/remake/executors/executor.py +0 -0
  29. {remake-0.8.2 → remake-0.8.4}/src/remake/executors/slurm_executor.py +0 -0
  30. {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/__init__.py +0 -0
  31. {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/metadata_manager.py +0 -0
  32. {remake-0.8.2 → remake-0.8.4}/src/remake/metadata/sidecar.py +0 -0
  33. {remake-0.8.2 → remake-0.8.4}/src/remake/util/__init__.py +0 -0
  34. {remake-0.8.2 → remake-0.8.4}/src/remake/util/code_compare.py +0 -0
  35. {remake-0.8.2 → remake-0.8.4}/src/remake/util/colour.py +0 -0
  36. {remake-0.8.2 → remake-0.8.4}/src/remake/util/command_line_args.py +0 -0
  37. {remake-0.8.2 → remake-0.8.4}/src/remake/util/config.py +0 -0
  38. {remake-0.8.2 → remake-0.8.4}/src/remake/util/util.py +0 -0
  39. {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/dependency_links.txt +0 -0
  40. {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/entry_points.txt +0 -0
  41. {remake-0.8.2 → remake-0.8.4}/src/remake.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: remake
3
- Version: 0.8.2
3
+ Version: 0.8.4
4
4
  Summary: Smart remake tool
5
5
  Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
6
6
  Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
@@ -237,6 +237,7 @@ Provides-Extra: examples
237
237
  Requires-Dist: xarray>=2024.7.0; extra == "examples"
238
238
  Requires-Dist: netcdf4>=1.7.2; extra == "examples"
239
239
  Requires-Dist: h5netcdf; extra == "examples"
240
+ Requires-Dist: h5py; extra == "examples"
240
241
  Requires-Dist: zarr<3; extra == "examples"
241
242
  Requires-Dist: dask; extra == "examples"
242
243
  Requires-Dist: pyyaml; extra == "examples"
@@ -53,6 +53,7 @@ examples = [
53
53
  "xarray>=2024.7.0",
54
54
  "netcdf4>=1.7.2",
55
55
  "h5netcdf",
56
+ "h5py", # h5netcdf >= 1.8 makes its h5py backend optional; we need it
56
57
  "zarr<3",
57
58
  "dask",
58
59
  "pyyaml",
@@ -7,7 +7,7 @@ import itertools
7
7
 
8
8
  import networkx as nx
9
9
 
10
- from .exceptions import Defer, SignatureError
10
+ from .exceptions import Defer, RuleGraphError, SignatureError
11
11
  from .rule import is_deferrable
12
12
  from .task import Task
13
13
 
@@ -20,9 +20,13 @@ def build_rule_dag(rules):
20
20
  g.add_node(rule)
21
21
  resolved = []
22
22
  for dep in rule.depends_on:
23
+ if not isinstance(dep, str) and getattr(dep, 'name', None) in rules_by_name:
24
+ # A Rule object resolved earlier may since have been replaced
25
+ # by a redefinition (notebook cell re-run): use the current one.
26
+ dep = rules_by_name[dep.name]
23
27
  if isinstance(dep, str):
24
28
  if dep not in rules_by_name:
25
- raise ValueError(
29
+ raise RuleGraphError(
26
30
  f'{rule.name}: depends_on references unknown rule {dep!r}'
27
31
  )
28
32
  dep = rules_by_name[dep]
@@ -31,7 +35,7 @@ def build_rule_dag(rules):
31
35
  rule.depends_on = resolved
32
36
  if not nx.is_directed_acyclic_graph(g):
33
37
  cycle = nx.find_cycle(g)
34
- raise ValueError(f'Rule dependencies contain a cycle: {cycle}')
38
+ raise RuleGraphError(f'Rule dependencies contain a cycle: {cycle}')
35
39
  return g
36
40
 
37
41
 
@@ -18,6 +18,23 @@ class ScopeError(RemakeError):
18
18
  pass
19
19
 
20
20
 
21
+ class RuleGraphError(RemakeError, ValueError):
22
+ """Bad rule dependencies (unknown depends_on name, a cycle). Also a
23
+ ValueError, which is what these were raised as before 0.8.4."""
24
+
25
+ pass
26
+
27
+
28
+ class TaskExit(Exception):
29
+ """A task's code raised SystemExit (e.g. a CLI `main()` calling
30
+ `sys.exit`). run_task records the task as failed and raises this in its
31
+ place, so executors treat it as an ordinary task failure instead of the
32
+ whole run exiting silently. Deliberately not a RemakeError: it is a task
33
+ failure, not a usage error."""
34
+
35
+ pass
36
+
37
+
21
38
  class Defer(Exception):
22
39
  """Raised by a `@deferrable` matrix callable to signal that the rule
23
40
  cannot be expanded this wave — its task list derives from an upstream
@@ -5,7 +5,11 @@ Rerun logic is DB-first: never run, failed, rule_run code changed (AST
5
5
  compare), uses changed, or any relevant upstream task reruns. Filesystem
6
6
  checks happen only via the opt-in check_outputs modes.
7
7
  """
8
+ import ast
9
+ import builtins
8
10
  import difflib
11
+ import inspect
12
+ import os
9
13
  from collections import namedtuple
10
14
  from time import perf_counter
11
15
 
@@ -15,7 +19,7 @@ from loguru import logger
15
19
  from ..metadata.metadata_manager import TASK_STATUS_FAILED, TASK_STATUS_SUCCESS
16
20
  from ..util.code_compare import CodeComparer
17
21
  from .dag import expand_rule
18
- from .exceptions import Defer
22
+ from .exceptions import Defer, RemakeError
19
23
  from .rule import is_deferrable
20
24
  from .scope import (
21
25
  io_hash,
@@ -27,20 +31,75 @@ from .scope import (
27
31
  )
28
32
 
29
33
 
30
- def make_predicate(query):
34
+ # Builtins a query may call (review 2026-09-24 M14): with none, reasonable
35
+ # queries like "year in range(2000, 2005)" silently matched nothing.
36
+ QUERY_BUILTINS = {
37
+ name: getattr(builtins, name)
38
+ for name in ('abs', 'all', 'any', 'bool', 'float', 'frozenset', 'int', 'len',
39
+ 'list', 'max', 'min', 'range', 'round', 'set', 'sorted', 'str', 'tuple')
40
+ }
41
+
42
+
43
+ def rule_kwarg_names(rules):
44
+ """Every matrix kwarg name across `rules` — the rule functions'
45
+ parameters after inputs/outputs (the signature contract makes these the
46
+ matrix keys, for callable matrices too)."""
47
+ names = set()
48
+ for rule in rules:
49
+ names.update(p for p in inspect.signature(rule.fn).parameters
50
+ if p not in ('inputs', 'outputs'))
51
+ return names
52
+
53
+
54
+ def _free_names(tree):
55
+ """Names a query expression reads, minus those it binds itself
56
+ (comprehension targets, lambda arguments)."""
57
+ loaded, bound = set(), set()
58
+ for node in ast.walk(tree):
59
+ if isinstance(node, ast.Name):
60
+ (loaded if isinstance(node.ctx, ast.Load) else bound).add(node.id)
61
+ elif isinstance(node, ast.arg):
62
+ bound.add(node.arg)
63
+ return loaded - bound
64
+
65
+
66
+ def make_predicate(query, known_names=None):
31
67
  """Compile a task-filter expression evaluated against task kwargs plus
32
68
  'rule' (the rule name), e.g. "year > 1985 and model == 'era5'",
33
- "rule in ['extract', 'clean']"."""
69
+ "rule in ['extract', 'clean']". A few safe builtins are available
70
+ (QUERY_BUILTINS).
71
+
72
+ `known_names` (every kwarg name in the pipeline, see rule_kwarg_names)
73
+ turns a name no rule has — almost always a typo — into an error instead
74
+ of a silent no-match. Bad syntax and evaluation errors are RemakeErrors
75
+ (a clean CLI error) rather than raw tracebacks."""
34
76
  # MM: this looks like a risk - using compile to compile the code?
35
77
  # See how pyquerylist does this - it only allows certain Python ops.
36
- code = compile(query, '<query>', 'eval')
37
-
78
+ try:
79
+ tree = ast.parse(query, mode='eval')
80
+ code = compile(tree, '<query>', 'eval')
81
+ except SyntaxError as e:
82
+ raise RemakeError(f'invalid query {query!r}: {e.msg}') from None
83
+ if known_names is not None:
84
+ unknown = _free_names(tree) - set(known_names) - {'rule'} - QUERY_BUILTINS.keys()
85
+ if unknown:
86
+ raise RemakeError(
87
+ f'query {query!r} uses unknown name(s) {sorted(unknown)} — not a '
88
+ f'matrix key of any rule (keys: {sorted(known_names)}), "rule", or an '
89
+ f'allowed builtin ({", ".join(sorted(QUERY_BUILTINS))})'
90
+ )
38
91
  def predicate(kwargs):
39
92
  try:
40
- return bool(eval(code, {'__builtins__': {}}, dict(kwargs)))
93
+ # kwargs go in the *globals*: a generator expression (e.g. inside
94
+ # any()) has its own scope and can't see eval's locals.
95
+ return bool(eval(code, {**kwargs, '__builtins__': QUERY_BUILTINS}))
41
96
  except NameError:
42
97
  # Query references a kwarg this rule doesn't have: no match.
43
98
  return False
99
+ except Exception as e:
100
+ raise RemakeError(
101
+ f'query {query!r} failed on {dict(kwargs)}: {type(e).__name__}: {e}'
102
+ ) from None
44
103
 
45
104
  return predicate
46
105
 
@@ -124,22 +183,53 @@ def cascade_settled(rule_set, dag, selected, run_seq, status):
124
183
  return settled
125
184
 
126
185
 
186
+ # Key in an executor's `failures` dict holding the normalised output paths of
187
+ # every task that failed (or was skipped) this run.
188
+ _FAILED_OUTPUTS = '_failed_outputs'
189
+
190
+
191
+ def _norm_paths(values):
192
+ return {os.path.normpath(str(v)) for v in values}
193
+
194
+
195
+ def _io_paths(task, part):
196
+ """A task's normalised input or output paths, or an empty set if its
197
+ callable spec raises — run_task has already recorded that failure, and
198
+ the executor's bookkeeping must not re-raise it (and crash the run)."""
199
+ try:
200
+ return _norm_paths(getattr(task, part).values())
201
+ except Exception:
202
+ return set()
203
+
204
+
205
+ def record_failure(failures, task):
206
+ """Record a failed (or skipped-for-upstream-failure) task in an
207
+ executor's `failures` dict, for upstream_failed."""
208
+ failures.setdefault(task.rule, set()).add(frozenset(task.kwargs.items()))
209
+ failures.setdefault(_FAILED_OUTPUTS, set()).update(_io_paths(task, 'outputs'))
210
+
211
+
127
212
  def upstream_failed(task, failures):
128
213
  """Should task be skipped because upstream tasks failed this run?
129
214
 
130
- failures: {rule: set of frozenset(kwargs.items())} accumulated by an
131
- executor. Mirrors the planner's rerun propagation: element-wise when
132
- the matrices are shared, conservative (any failure taints all
133
- downstream tasks) otherwise.
215
+ failures: built by record_failure. Mirrors the planner's rerun
216
+ propagation — element-wise when the matrices are shared, conservative
217
+ (any failure taints all downstream tasks) otherwise — and additionally
218
+ skips any task whose declared inputs include an output of a failed task.
219
+ A shared matrix doesn't mean each task reads only its same-kwargs
220
+ upstream (e.g. inputs of year-1): without the path check such a task ran
221
+ on its failed upstream's stale or missing output (review 2026-09-24 H3,
222
+ failure-skip part; the full fix is 0.9).
134
223
  """
135
224
  for dep in task.rule.depends_on:
136
225
  failed = failures.get(dep)
137
226
  if not failed:
138
227
  continue
139
- if _same_matrix(task.rule, dep):
140
- if frozenset(task.kwargs.items()) in failed:
141
- return True
142
- else:
228
+ if not _same_matrix(task.rule, dep):
229
+ return True
230
+ if frozenset(task.kwargs.items()) in failed:
231
+ return True
232
+ if _io_paths(task, 'inputs') & failures.get(_FAILED_OUTPUTS, set()):
143
233
  return True
144
234
  return False
145
235
 
@@ -323,9 +413,10 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
323
413
  # MM: this is a core piece of logic, but I find it hard to understand end-to-end.
324
414
  # MM: also quite a long func.
325
415
  start = perf_counter()
326
- predicate = make_predicate(query) if query else None
416
+ predicate = make_predicate(query, rule_kwarg_names(rules)) if query else None
327
417
  code_comparer = CodeComparer()
328
418
  rules = set(rules)
419
+ nmatched = 0
329
420
 
330
421
  runnable = []
331
422
  deferred = []
@@ -469,8 +560,12 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
469
560
  (records[task.key].run_seq if task.key in records else None)
470
561
  for task in tasks
471
562
  }
563
+ nmatched += len(tasks)
472
564
  logger.debug('{}: {} task(s), {} to rerun', rule.name, len(tasks), len(rule_rerun))
473
565
 
566
+ if predicate is not None and not nmatched and not deferred:
567
+ # Distinguish "your query selected nothing" from "all up to date".
568
+ logger.warning(f'query {query!r} matched no tasks')
474
569
  elapsed = perf_counter() - start
475
570
  logger.bind(
476
571
  event='plan', nrunnable=len(runnable), ndeferred=len(deferred),
@@ -5,6 +5,7 @@ from collections import Counter
5
5
  from pathlib import Path
6
6
  from time import perf_counter
7
7
 
8
+ import networkx as nx
8
9
  from loguru import logger
9
10
 
10
11
  from ..metadata.metadata_manager import (
@@ -14,9 +15,16 @@ from ..metadata.metadata_manager import (
14
15
  RecordCache,
15
16
  )
16
17
  from ..util import task_log_path
18
+ from ..util.run_lock import run_lock
17
19
  from .dag import build_rule_dag, expand_rule, iter_expand_rule
18
- from .exceptions import Defer, RemakeError
19
- from .planner import cascade_settled, explain_task, make_predicate, plan
20
+ from .exceptions import Defer, RemakeError, TaskExit
21
+ from .planner import (
22
+ cascade_settled,
23
+ explain_task,
24
+ make_predicate,
25
+ plan,
26
+ rule_kwarg_names,
27
+ )
20
28
  from .rule import Rule
21
29
  from .scope import check_scope, exec_function
22
30
  from .task import Task
@@ -45,6 +53,39 @@ class _TemplatePlaceholder:
45
53
  return str(self)
46
54
 
47
55
 
56
+ def _blocked_reason(rule):
57
+ """Why a rule is still deferred when a local run has nothing left to
58
+ try: its matrix's Defer paths if it still can't expand, else (it would
59
+ expand, so the planner is holding it back) an upstream that didn't
60
+ complete this run."""
61
+ from .rule import is_deferrable
62
+
63
+ if is_deferrable(rule.matrix):
64
+ try:
65
+ rule.matrix()
66
+ except Defer as e:
67
+ waiting = ', '.join(e.paths) if e.paths else 'an unspecified input'
68
+ return f'matrix not ready — waiting on {waiting}'
69
+ except Exception as e: # the matrix itself is broken: say so
70
+ return f'matrix raised {type(e).__name__}: {e}'
71
+ return 'an upstream rule did not complete (see failures above)'
72
+
73
+
74
+ def _where(rule):
75
+ fn = rule.fn
76
+ return f'{getattr(fn, "__module__", "?")}.{getattr(fn, "__qualname__", rule.name)}'
77
+
78
+
79
+ def _same_definition(a, b):
80
+ """Two Rule objects for the same function re-defined (e.g. a notebook
81
+ cell re-run): same module and qualified name, but freshly compiled code.
82
+ Distinct functions that merely share a rule name (two modules each
83
+ defining `process`, a clashing name=) are not — nor are rules built by
84
+ one factory function, which share a single code object and differ only
85
+ in closure values."""
86
+ return _where(a) == _where(b) and a.fn.__code__ is not b.fn.__code__
87
+
88
+
48
89
  class Remake:
49
90
  def __init__(
50
91
  self,
@@ -58,6 +99,10 @@ class Remake:
58
99
  self.config = config or {}
59
100
  self.metadata = metadata
60
101
  self.check_outputs = check_outputs
102
+ # Rules the last run() left unresolved (deferred matrix never ready,
103
+ # or waiting on an upstream that failed). Non-empty means that run
104
+ # did not complete everything; the CLI exits non-zero on it.
105
+ self.blocked_rules = []
61
106
  self.strict_scope = strict_scope
62
107
  self.rules = []
63
108
  self.dag = None
@@ -74,11 +119,26 @@ class Remake:
74
119
  raise RemakeError(f'Not a Rule (use the @rule decorator): {rule!r}')
75
120
  if rule in self.rules:
76
121
  continue
122
+ # Task keys are sha1('<rule name>:<kwargs>'), so two rules sharing
123
+ # a name share task keys and DB records (review 2026-09-24 H8).
124
+ existing = next((r for r in self.rules if r.name == rule.name), None)
125
+ if existing is not None and not _same_definition(existing, rule):
126
+ raise RemakeError(
127
+ f'Duplicate rule name {rule.name!r}: defined by '
128
+ f'{_where(existing)} and {_where(rule)}. Rule names must be '
129
+ f'unique within a pipeline (rename one, or pass name=...)'
130
+ )
77
131
  # Resolve tri-state strict_scope against the Remake default.
78
132
  if rule.strict_scope is None and self.strict_scope:
79
133
  check_scope(rule.fn, rule.uses, strict=True)
80
134
  rule.remake = self
81
- self.rules.append(rule)
135
+ if existing is not None:
136
+ # The same function re-defined — a notebook cell or script
137
+ # section executed again: the new definition replaces the old.
138
+ logger.warning(f'Rule {rule.name!r} redefined: replacing the earlier definition')
139
+ self.rules[self.rules.index(existing)] = rule
140
+ else:
141
+ self.rules.append(rule)
82
142
  self._finalized = False
83
143
 
84
144
  def rules_from_current_module(self):
@@ -186,7 +246,7 @@ class Remake:
186
246
  dynamic matrix is deferred."""
187
247
  if not self._finalized:
188
248
  self.finalize()
189
- predicate = make_predicate(query) if query else None
249
+ predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
190
250
  for rule in self.rules:
191
251
  try:
192
252
  yield from iter_expand_rule(rule, predicate)
@@ -285,7 +345,7 @@ class Remake:
285
345
  remaining = Counter(task.rule.name for task in runnable)
286
346
  runnable_keys = {task.key for task in runnable}
287
347
  deferred_names = {rule.name for rule in deferred}
288
- predicate = make_predicate(query) if query else None
348
+ predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
289
349
 
290
350
  # Per-rule tally of why the to-run tasks would rerun. One plan() is
291
351
  # already done (`runnable`); reuse it per task so this is plan-cost,
@@ -669,9 +729,21 @@ class Remake:
669
729
  """Run all tasks that need running, replanning after each wave so
670
730
  dynamic (deferred) matrices resolve as their upstreams complete.
671
731
  Returns the number of failed tasks (0 for asynchronous executors,
672
- which don't know at submission time)."""
732
+ which don't know at submission time). Holds `.remake/run.lock` for
733
+ the duration: a second concurrent run in the same directory is a
734
+ RemakeError."""
673
735
  if not self._finalized:
674
736
  self.finalize()
737
+ self.blocked_rules = []
738
+ if executor is not None and executor.handles_deferred:
739
+ # SLURM: submission only; continuation jobs re-enter run() on
740
+ # compute nodes, and the executor has its own duplicate-submission
741
+ # guard — a directory lock would strand or block them.
742
+ return self._run(executor, query, force, ignore_code_changes)
743
+ with run_lock(self.metadata):
744
+ return self._run(executor, query, force, ignore_code_changes)
745
+
746
+ def _run(self, executor, query, force, ignore_code_changes):
675
747
  # One run_seq for this whole invocation (shared across replanning
676
748
  # waves); committed onto every task so downstream propagation survives
677
749
  # to later invocations. See bugs/01_durable_rerun_propagation.md.
@@ -705,8 +777,19 @@ class Remake:
705
777
  runnable = [t for t in runnable if t.key not in attempted]
706
778
  if not runnable:
707
779
  if deferred:
708
- names = ', '.join(rule.name for rule in deferred)
709
- logger.warning(f'Blocked rules (matrix not ready): {names}')
780
+ # Not everything completed: say which rules and why, and
781
+ # make the run's outcome reflect it (review 2026-09-24 M9;
782
+ # it used to be a bare warning and exit 0). A filtered run
783
+ # may deliberately leave out the upstream a deferred rule
784
+ # waits on: that one is reported but doesn't fail the run.
785
+ excused = self._excluded_by_query(deferred) if query else set()
786
+ for rule in deferred:
787
+ if rule in excused:
788
+ logger.warning(f'Blocked rule {rule.name} (upstream not '
789
+ f'selected by the query): {_blocked_reason(rule)}')
790
+ else:
791
+ self.blocked_rules.append(rule)
792
+ logger.error(f'Blocked rule {rule.name}: {_blocked_reason(rule)}')
710
793
  break
711
794
  wave += 1
712
795
  logger.bind(event='wave', wave=wave, ntasks=len(runnable)).debug(
@@ -724,40 +807,71 @@ class Remake:
724
807
  logger.info('Nothing to do')
725
808
  return nfailed
726
809
 
810
+ def _excluded_by_query(self, deferred):
811
+ """Deferred rules a filtered run can be excused for: those with an
812
+ upstream (any ancestor) that still has work outside the filter. If
813
+ every ancestor is complete and the rule is still deferred, the query
814
+ didn't cause it — it's genuinely blocked."""
815
+ runnable_all, deferred_all = self.plan()
816
+ unfinished = {t.rule for t in runnable_all} | set(deferred_all)
817
+ return {rule for rule in deferred
818
+ if nx.ancestors(self.dag, rule) & (unfinished - {rule})}
819
+
727
820
  def run_task(self, task):
728
821
  """Execute one task and record the result. The single execution
729
822
  entry point — used by all executors and `remake run-task`. Timing and
730
823
  completion are logged here so every executor gets them uniformly
731
824
  (per-element detail at TRACE, per-task duration at DEBUG — the
732
825
  summarise-loops convention, per_task_logging.md)."""
733
- # opt(lazy=True): the path lists are only built when a TRACE sink is
734
- # attached (they'd cost real time at 1e6 tasks otherwise).
735
- logger.opt(lazy=True).trace(
736
- 'running {}: inputs {} -> outputs {}', lambda: task,
737
- lambda: [str(p) for p in task.inputs.values()],
738
- lambda: [str(p) for p in task.outputs.values()],
739
- )
740
- for token in task.outputs.values():
741
- if hasattr(token, '__fspath__'):
742
- Path(token).parent.mkdir(parents=True, exist_ok=True)
743
-
744
- fn = exec_function(task.rule.fn, task.rule.uses)
745
- args = []
746
- if task.rule.inputs is not None:
747
- args.append(task.inputs)
748
- if task.rule.outputs is not None:
749
- args.append(task.outputs)
750
826
  start = perf_counter()
827
+ # Everything that can fail — output dirs, io resolution (callable
828
+ # specs run here), the function itself — is inside the try, so every
829
+ # failure is recorded with its traceback (review 2026-09-24 M11).
751
830
  try:
752
- fn(*args, **task.kwargs)
753
- except Exception:
831
+ # opt(lazy=True): the path lists are only built when a TRACE sink
832
+ # is attached (they'd cost real time at 1e6 tasks otherwise).
833
+ logger.opt(lazy=True).trace(
834
+ 'running {}: inputs {} -> outputs {}', lambda: task,
835
+ lambda: [str(p) for p in task.inputs.values()],
836
+ lambda: [str(p) for p in task.outputs.values()],
837
+ )
838
+ for token in task.outputs.values():
839
+ if hasattr(token, '__fspath__'):
840
+ Path(token).parent.mkdir(parents=True, exist_ok=True)
841
+
842
+ fn = exec_function(task.rule.fn, task.rule.uses)
843
+ args = []
844
+ if task.rule.inputs is not None:
845
+ args.append(task.inputs)
846
+ if task.rule.outputs is not None:
847
+ args.append(task.outputs)
848
+ try:
849
+ fn(*args, **task.kwargs)
850
+ except SystemExit as exc:
851
+ # sys.exit(0) / sys.exit() from a wrapped CLI main() means
852
+ # success; only a non-zero code is a failure (handled below).
853
+ # Mirror CPython: only None or an int 0 (incl. False) exits 0;
854
+ # sys.exit(0.0) or sys.exit('msg') exit 1.
855
+ if not (exc.code is None or (isinstance(exc.code, int) and exc.code == 0)):
856
+ raise
857
+ except (Exception, SystemExit) as exc:
858
+ # SystemExit from task code (a CLI main() calling sys.exit) is a
859
+ # task failure, not a request to end the run (review H6).
860
+ # KeyboardInterrupt still propagates: Ctrl-C stops the run.
754
861
  elapsed = perf_counter() - start
862
+ tb = traceback.format_exc()
755
863
  logger.bind(event='task_failed', task=str(task), rule=task.rule.name,
756
864
  key=task.key, seconds=round(elapsed, 6),
757
865
  ).error(f'failed: {task} after {elapsed:.2f}s')
758
- self.metadata.update_task(
759
- task, TASK_STATUS_FAILED, exception=traceback.format_exc()
760
- )
866
+ # The traceback at DEBUG: lands in per-task logs (DEBUG sinks)
867
+ # without flooding the INFO console on runs with many failures
868
+ # (review M12).
869
+ logger.debug('traceback for {}:\n{}', task, tb)
870
+ self.metadata.update_task(task, TASK_STATUS_FAILED, exception=tb)
871
+ if isinstance(exc, SystemExit):
872
+ raise TaskExit(
873
+ f'{task} called sys.exit({exc.code!r})'
874
+ ) from exc
761
875
  raise
762
876
  elapsed = perf_counter() - start
763
877
  logger.bind(event='task_complete', task=str(task), rule=task.rule.name,
@@ -216,7 +216,12 @@ def rule(
216
216
  inputs=inputs,
217
217
  outputs=outputs,
218
218
  matrix=matrix,
219
- depends_on=list(depends_on) if depends_on else [],
219
+ # A bare name or Rule is one dependency — list('extract') would
220
+ # split it into characters (review 2026-09-24 L4).
221
+ depends_on=(
222
+ [depends_on] if isinstance(depends_on, (str, Rule))
223
+ else list(depends_on) if depends_on else []
224
+ ),
220
225
  uses=uses_,
221
226
  strict_scope=strict_scope,
222
227
  config=dict(config) if config else {},
@@ -23,13 +23,13 @@ import os
23
23
  from loguru import logger
24
24
 
25
25
  from ..core.exceptions import RemakeError
26
- from ..core.planner import upstream_failed
26
+ from ..core.planner import record_failure, upstream_failed
27
27
  from .executor import Executor
28
28
 
29
29
  _worker_rmk_cache = {}
30
30
 
31
31
 
32
- def _run_spec(remakefile, rule_name, kwargs):
32
+ def _run_spec(remakefile, rule_name, kwargs, run_seq=None):
33
33
  """Runs on a dask worker. Returns True on success."""
34
34
  from ..loader import load_remake
35
35
  from ..metadata.sidecar import SidecarWriter
@@ -40,6 +40,9 @@ def _run_spec(remakefile, rule_name, kwargs):
40
40
  rmk = load_remake(remakefile, finalize=False)
41
41
  rmk.metadata = SidecarWriter()
42
42
  _worker_rmk_cache[remakefile] = rmk
43
+ # The parent invocation's run_seq (durable propagation, bugs/01); set per
44
+ # call because long-lived workers outlive a single invocation.
45
+ rmk.metadata.run_seq = run_seq
43
46
  task = rmk.task_from_spec(rule_name, kwargs)
44
47
  logfile = task_log_path(task)
45
48
  logfile.parent.mkdir(parents=True, exist_ok=True)
@@ -95,14 +98,15 @@ class DaskExecutor(Executor):
95
98
  nfailed = 0
96
99
  nskipped = 0
97
100
  done = 0
98
- failures = {} # rule -> set of frozenset(kwargs.items())
101
+ failures = {} # see planner.record_failure
102
+ run_seq = self.rmk.metadata.current_run_seq()
99
103
  client, cluster = self._client()
100
104
  try:
101
105
  for rule, rule_tasks in groups:
102
106
  to_run = []
103
107
  for task in rule_tasks:
104
108
  if upstream_failed(task, failures):
105
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
109
+ record_failure(failures, task)
106
110
  nskipped += 1
107
111
  done += 1
108
112
  logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
@@ -113,7 +117,8 @@ class DaskExecutor(Executor):
113
117
  logger.info(f'{rule.name}: {len(to_run)} task(s) on dask ({self.nproc} workers)')
114
118
  futures = {
115
119
  client.submit(
116
- _run_spec, self.remakefile, rule.name, task.kwargs, pure=False
120
+ _run_spec, self.remakefile, rule.name, task.kwargs, run_seq,
121
+ pure=False,
117
122
  ): task
118
123
  for task in to_run
119
124
  }
@@ -124,7 +129,7 @@ class DaskExecutor(Executor):
124
129
  if future.result():
125
130
  logger.info(f'{done}/{ntasks}: {task}')
126
131
  else:
127
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
132
+ record_failure(failures, task)
128
133
  nfailed += 1
129
134
  logger.error(f'{done}/{ntasks} failed: {task}')
130
135
  finally:
@@ -22,7 +22,7 @@ from multiprocessing import get_context
22
22
  from loguru import logger
23
23
 
24
24
  from ..core.exceptions import RemakeError
25
- from ..core.planner import upstream_failed
25
+ from ..core.planner import record_failure, upstream_failed
26
26
  from .executor import Executor
27
27
 
28
28
  _worker_rmk = None
@@ -53,7 +53,10 @@ def _worker_init(remakefile):
53
53
  def _worker_run(spec):
54
54
  from ..util import task_log_path
55
55
 
56
- rule_name, kwargs = spec
56
+ rule_name, kwargs, run_seq = spec
57
+ # Stamp results with the parent invocation's run_seq so durable rerun
58
+ # propagation (bugs/01) works for multiproc runs as it does for SLURM.
59
+ _worker_rmk.metadata.run_seq = run_seq
57
60
  task = _worker_rmk.task_from_spec(rule_name, kwargs)
58
61
  logfile = task_log_path(task)
59
62
  logfile.parent.mkdir(parents=True, exist_ok=True)
@@ -93,7 +96,8 @@ class MultiprocExecutor(Executor):
93
96
  nfailed = 0
94
97
  nskipped = 0
95
98
  done = 0
96
- failures = {} # rule -> set of frozenset(kwargs.items())
99
+ failures = {} # see planner.record_failure
100
+ run_seq = self.rmk.metadata.current_run_seq()
97
101
  with ProcessPoolExecutor(
98
102
  max_workers=self.nproc,
99
103
  mp_context=get_context('spawn'),
@@ -107,7 +111,7 @@ class MultiprocExecutor(Executor):
107
111
  to_run = []
108
112
  for task in rule_tasks:
109
113
  if upstream_failed(task, failures):
110
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
114
+ record_failure(failures, task)
111
115
  nskipped += 1
112
116
  done += 1
113
117
  logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
@@ -117,7 +121,8 @@ class MultiprocExecutor(Executor):
117
121
  continue
118
122
  logger.info(f'{rule.name}: {len(to_run)} task(s) on {self.nproc} proc(s)')
119
123
  futures = {
120
- pool.submit(_worker_run, (rule.name, t.kwargs)): t for t in to_run
124
+ pool.submit(_worker_run, (rule.name, t.kwargs, run_seq)): t
125
+ for t in to_run
121
126
  }
122
127
  # Barrier: drain this rule before starting the next.
123
128
  for future in as_completed(futures):
@@ -126,7 +131,7 @@ class MultiprocExecutor(Executor):
126
131
  if future.result():
127
132
  logger.info(f'{done}/{ntasks}: {task}')
128
133
  else:
129
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
134
+ record_failure(failures, task)
130
135
  nfailed += 1
131
136
  logger.error(f'{done}/{ntasks} failed: {task}')
132
137
  if nfailed: