remake 0.8.3__tar.gz → 0.8.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {remake-0.8.3 → remake-0.8.4}/PKG-INFO +1 -1
  2. {remake-0.8.3 → remake-0.8.4}/src/remake/core/dag.py +7 -3
  3. {remake-0.8.3 → remake-0.8.4}/src/remake/core/exceptions.py +17 -0
  4. {remake-0.8.3 → remake-0.8.4}/src/remake/core/planner.py +110 -15
  5. {remake-0.8.3 → remake-0.8.4}/src/remake/core/remake.py +144 -30
  6. {remake-0.8.3 → remake-0.8.4}/src/remake/core/rule.py +6 -1
  7. {remake-0.8.3 → remake-0.8.4}/src/remake/executors/dask_executor.py +11 -6
  8. {remake-0.8.3 → remake-0.8.4}/src/remake/executors/multiproc_executor.py +11 -6
  9. {remake-0.8.3 → remake-0.8.4}/src/remake/executors/singleproc_executor.py +4 -4
  10. {remake-0.8.3 → remake-0.8.4}/src/remake/loader/__init__.py +6 -0
  11. {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/sqlite3_backend.py +170 -32
  12. {remake-0.8.3 → remake-0.8.4}/src/remake/remake_cmd.py +38 -10
  13. remake-0.8.4/src/remake/util/run_lock.py +77 -0
  14. {remake-0.8.3 → remake-0.8.4}/src/remake/version.py +1 -1
  15. {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/PKG-INFO +1 -1
  16. {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/SOURCES.txt +1 -0
  17. {remake-0.8.3 → remake-0.8.4}/LICENCE +0 -0
  18. {remake-0.8.3 → remake-0.8.4}/README.md +0 -0
  19. {remake-0.8.3 → remake-0.8.4}/pyproject.toml +0 -0
  20. {remake-0.8.3 → remake-0.8.4}/setup.cfg +0 -0
  21. {remake-0.8.3 → remake-0.8.4}/src/remake/__init__.py +0 -0
  22. {remake-0.8.3 → remake-0.8.4}/src/remake/core/__init__.py +0 -0
  23. {remake-0.8.3 → remake-0.8.4}/src/remake/core/scope.py +0 -0
  24. {remake-0.8.3 → remake-0.8.4}/src/remake/core/task.py +0 -0
  25. {remake-0.8.3 → remake-0.8.4}/src/remake/core/tokens.py +0 -0
  26. {remake-0.8.3 → remake-0.8.4}/src/remake/executors/__init__.py +0 -0
  27. {remake-0.8.3 → remake-0.8.4}/src/remake/executors/executor.py +0 -0
  28. {remake-0.8.3 → remake-0.8.4}/src/remake/executors/slurm_executor.py +0 -0
  29. {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/__init__.py +0 -0
  30. {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/metadata_manager.py +0 -0
  31. {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/sidecar.py +0 -0
  32. {remake-0.8.3 → remake-0.8.4}/src/remake/util/__init__.py +0 -0
  33. {remake-0.8.3 → remake-0.8.4}/src/remake/util/code_compare.py +0 -0
  34. {remake-0.8.3 → remake-0.8.4}/src/remake/util/colour.py +0 -0
  35. {remake-0.8.3 → remake-0.8.4}/src/remake/util/command_line_args.py +0 -0
  36. {remake-0.8.3 → remake-0.8.4}/src/remake/util/config.py +0 -0
  37. {remake-0.8.3 → remake-0.8.4}/src/remake/util/util.py +0 -0
  38. {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/dependency_links.txt +0 -0
  39. {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/entry_points.txt +0 -0
  40. {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/requires.txt +0 -0
  41. {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: remake
3
- Version: 0.8.3
3
+ Version: 0.8.4
4
4
  Summary: Smart remake tool
5
5
  Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
6
6
  Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
@@ -7,7 +7,7 @@ import itertools
7
7
 
8
8
  import networkx as nx
9
9
 
10
- from .exceptions import Defer, SignatureError
10
+ from .exceptions import Defer, RuleGraphError, SignatureError
11
11
  from .rule import is_deferrable
12
12
  from .task import Task
13
13
 
@@ -20,9 +20,13 @@ def build_rule_dag(rules):
20
20
  g.add_node(rule)
21
21
  resolved = []
22
22
  for dep in rule.depends_on:
23
+ if not isinstance(dep, str) and getattr(dep, 'name', None) in rules_by_name:
24
+ # A Rule object resolved earlier may since have been replaced
25
+ # by a redefinition (notebook cell re-run): use the current one.
26
+ dep = rules_by_name[dep.name]
23
27
  if isinstance(dep, str):
24
28
  if dep not in rules_by_name:
25
- raise ValueError(
29
+ raise RuleGraphError(
26
30
  f'{rule.name}: depends_on references unknown rule {dep!r}'
27
31
  )
28
32
  dep = rules_by_name[dep]
@@ -31,7 +35,7 @@ def build_rule_dag(rules):
31
35
  rule.depends_on = resolved
32
36
  if not nx.is_directed_acyclic_graph(g):
33
37
  cycle = nx.find_cycle(g)
34
- raise ValueError(f'Rule dependencies contain a cycle: {cycle}')
38
+ raise RuleGraphError(f'Rule dependencies contain a cycle: {cycle}')
35
39
  return g
36
40
 
37
41
 
@@ -18,6 +18,23 @@ class ScopeError(RemakeError):
18
18
  pass
19
19
 
20
20
 
21
+ class RuleGraphError(RemakeError, ValueError):
22
+ """Bad rule dependencies (unknown depends_on name, a cycle). Also a
23
+ ValueError, which is what these were raised as before 0.8.4."""
24
+
25
+ pass
26
+
27
+
28
+ class TaskExit(Exception):
29
+ """A task's code raised SystemExit (e.g. a CLI `main()` calling
30
+ `sys.exit`). run_task records the task as failed and raises this in its
31
+ place, so executors treat it as an ordinary task failure instead of the
32
+ whole run exiting silently. Deliberately not a RemakeError: it is a task
33
+ failure, not a usage error."""
34
+
35
+ pass
36
+
37
+
21
38
  class Defer(Exception):
22
39
  """Raised by a `@deferrable` matrix callable to signal that the rule
23
40
  cannot be expanded this wave — its task list derives from an upstream
@@ -5,7 +5,11 @@ Rerun logic is DB-first: never run, failed, rule_run code changed (AST
5
5
  compare), uses changed, or any relevant upstream task reruns. Filesystem
6
6
  checks happen only via the opt-in check_outputs modes.
7
7
  """
8
+ import ast
9
+ import builtins
8
10
  import difflib
11
+ import inspect
12
+ import os
9
13
  from collections import namedtuple
10
14
  from time import perf_counter
11
15
 
@@ -15,7 +19,7 @@ from loguru import logger
15
19
  from ..metadata.metadata_manager import TASK_STATUS_FAILED, TASK_STATUS_SUCCESS
16
20
  from ..util.code_compare import CodeComparer
17
21
  from .dag import expand_rule
18
- from .exceptions import Defer
22
+ from .exceptions import Defer, RemakeError
19
23
  from .rule import is_deferrable
20
24
  from .scope import (
21
25
  io_hash,
@@ -27,20 +31,75 @@ from .scope import (
27
31
  )
28
32
 
29
33
 
30
- def make_predicate(query):
34
+ # Builtins a query may call (review 2026-09-24 M14): with none, reasonable
35
+ # queries like "year in range(2000, 2005)" silently matched nothing.
36
+ QUERY_BUILTINS = {
37
+ name: getattr(builtins, name)
38
+ for name in ('abs', 'all', 'any', 'bool', 'float', 'frozenset', 'int', 'len',
39
+ 'list', 'max', 'min', 'range', 'round', 'set', 'sorted', 'str', 'tuple')
40
+ }
41
+
42
+
43
+ def rule_kwarg_names(rules):
44
+ """Every matrix kwarg name across `rules` — the rule functions'
45
+ parameters after inputs/outputs (the signature contract makes these the
46
+ matrix keys, for callable matrices too)."""
47
+ names = set()
48
+ for rule in rules:
49
+ names.update(p for p in inspect.signature(rule.fn).parameters
50
+ if p not in ('inputs', 'outputs'))
51
+ return names
52
+
53
+
54
+ def _free_names(tree):
55
+ """Names a query expression reads, minus those it binds itself
56
+ (comprehension targets, lambda arguments)."""
57
+ loaded, bound = set(), set()
58
+ for node in ast.walk(tree):
59
+ if isinstance(node, ast.Name):
60
+ (loaded if isinstance(node.ctx, ast.Load) else bound).add(node.id)
61
+ elif isinstance(node, ast.arg):
62
+ bound.add(node.arg)
63
+ return loaded - bound
64
+
65
+
66
+ def make_predicate(query, known_names=None):
31
67
  """Compile a task-filter expression evaluated against task kwargs plus
32
68
  'rule' (the rule name), e.g. "year > 1985 and model == 'era5'",
33
- "rule in ['extract', 'clean']"."""
69
+ "rule in ['extract', 'clean']". A few safe builtins are available
70
+ (QUERY_BUILTINS).
71
+
72
+ `known_names` (every kwarg name in the pipeline, see rule_kwarg_names)
73
+ turns a name no rule has — almost always a typo — into an error instead
74
+ of a silent no-match. Bad syntax and evaluation errors are RemakeErrors
75
+ (a clean CLI error) rather than raw tracebacks."""
34
76
  # MM: this looks like a risk - using compile to compile the code?
35
77
  # See how pyquerylist does this - it only allows certain Python ops.
36
- code = compile(query, '<query>', 'eval')
37
-
78
+ try:
79
+ tree = ast.parse(query, mode='eval')
80
+ code = compile(tree, '<query>', 'eval')
81
+ except SyntaxError as e:
82
+ raise RemakeError(f'invalid query {query!r}: {e.msg}') from None
83
+ if known_names is not None:
84
+ unknown = _free_names(tree) - set(known_names) - {'rule'} - QUERY_BUILTINS.keys()
85
+ if unknown:
86
+ raise RemakeError(
87
+ f'query {query!r} uses unknown name(s) {sorted(unknown)} — not a '
88
+ f'matrix key of any rule (keys: {sorted(known_names)}), "rule", or an '
89
+ f'allowed builtin ({", ".join(sorted(QUERY_BUILTINS))})'
90
+ )
38
91
  def predicate(kwargs):
39
92
  try:
40
- return bool(eval(code, {'__builtins__': {}}, dict(kwargs)))
93
+ # kwargs go in the *globals*: a generator expression (e.g. inside
94
+ # any()) has its own scope and can't see eval's locals.
95
+ return bool(eval(code, {**kwargs, '__builtins__': QUERY_BUILTINS}))
41
96
  except NameError:
42
97
  # Query references a kwarg this rule doesn't have: no match.
43
98
  return False
99
+ except Exception as e:
100
+ raise RemakeError(
101
+ f'query {query!r} failed on {dict(kwargs)}: {type(e).__name__}: {e}'
102
+ ) from None
44
103
 
45
104
  return predicate
46
105
 
@@ -124,22 +183,53 @@ def cascade_settled(rule_set, dag, selected, run_seq, status):
124
183
  return settled
125
184
 
126
185
 
186
+ # Key in an executor's `failures` dict holding the normalised output paths of
187
+ # every task that failed (or was skipped) this run.
188
+ _FAILED_OUTPUTS = '_failed_outputs'
189
+
190
+
191
+ def _norm_paths(values):
192
+ return {os.path.normpath(str(v)) for v in values}
193
+
194
+
195
+ def _io_paths(task, part):
196
+ """A task's normalised input or output paths, or an empty set if its
197
+ callable spec raises — run_task has already recorded that failure, and
198
+ the executor's bookkeeping must not re-raise it (and crash the run)."""
199
+ try:
200
+ return _norm_paths(getattr(task, part).values())
201
+ except Exception:
202
+ return set()
203
+
204
+
205
+ def record_failure(failures, task):
206
+ """Record a failed (or skipped-for-upstream-failure) task in an
207
+ executor's `failures` dict, for upstream_failed."""
208
+ failures.setdefault(task.rule, set()).add(frozenset(task.kwargs.items()))
209
+ failures.setdefault(_FAILED_OUTPUTS, set()).update(_io_paths(task, 'outputs'))
210
+
211
+
127
212
  def upstream_failed(task, failures):
128
213
  """Should task be skipped because upstream tasks failed this run?
129
214
 
130
- failures: {rule: set of frozenset(kwargs.items())} accumulated by an
131
- executor. Mirrors the planner's rerun propagation: element-wise when
132
- the matrices are shared, conservative (any failure taints all
133
- downstream tasks) otherwise.
215
+ failures: built by record_failure. Mirrors the planner's rerun
216
+ propagation — element-wise when the matrices are shared, conservative
217
+ (any failure taints all downstream tasks) otherwise — and additionally
218
+ skips any task whose declared inputs include an output of a failed task.
219
+ A shared matrix doesn't mean each task reads only its same-kwargs
220
+ upstream (e.g. inputs of year-1): without the path check such a task ran
221
+ on its failed upstream's stale or missing output (review 2026-09-24 H3,
222
+ failure-skip part; the full fix is 0.9).
134
223
  """
135
224
  for dep in task.rule.depends_on:
136
225
  failed = failures.get(dep)
137
226
  if not failed:
138
227
  continue
139
- if _same_matrix(task.rule, dep):
140
- if frozenset(task.kwargs.items()) in failed:
141
- return True
142
- else:
228
+ if not _same_matrix(task.rule, dep):
229
+ return True
230
+ if frozenset(task.kwargs.items()) in failed:
231
+ return True
232
+ if _io_paths(task, 'inputs') & failures.get(_FAILED_OUTPUTS, set()):
143
233
  return True
144
234
  return False
145
235
 
@@ -323,9 +413,10 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
323
413
  # MM: this is a core piece of logic, but I find it hard to understand end-to-end.
324
414
  # MM: also quite a long func.
325
415
  start = perf_counter()
326
- predicate = make_predicate(query) if query else None
416
+ predicate = make_predicate(query, rule_kwarg_names(rules)) if query else None
327
417
  code_comparer = CodeComparer()
328
418
  rules = set(rules)
419
+ nmatched = 0
329
420
 
330
421
  runnable = []
331
422
  deferred = []
@@ -469,8 +560,12 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
469
560
  (records[task.key].run_seq if task.key in records else None)
470
561
  for task in tasks
471
562
  }
563
+ nmatched += len(tasks)
472
564
  logger.debug('{}: {} task(s), {} to rerun', rule.name, len(tasks), len(rule_rerun))
473
565
 
566
+ if predicate is not None and not nmatched and not deferred:
567
+ # Distinguish "your query selected nothing" from "all up to date".
568
+ logger.warning(f'query {query!r} matched no tasks')
474
569
  elapsed = perf_counter() - start
475
570
  logger.bind(
476
571
  event='plan', nrunnable=len(runnable), ndeferred=len(deferred),
@@ -5,6 +5,7 @@ from collections import Counter
5
5
  from pathlib import Path
6
6
  from time import perf_counter
7
7
 
8
+ import networkx as nx
8
9
  from loguru import logger
9
10
 
10
11
  from ..metadata.metadata_manager import (
@@ -14,9 +15,16 @@ from ..metadata.metadata_manager import (
14
15
  RecordCache,
15
16
  )
16
17
  from ..util import task_log_path
18
+ from ..util.run_lock import run_lock
17
19
  from .dag import build_rule_dag, expand_rule, iter_expand_rule
18
- from .exceptions import Defer, RemakeError
19
- from .planner import cascade_settled, explain_task, make_predicate, plan
20
+ from .exceptions import Defer, RemakeError, TaskExit
21
+ from .planner import (
22
+ cascade_settled,
23
+ explain_task,
24
+ make_predicate,
25
+ plan,
26
+ rule_kwarg_names,
27
+ )
20
28
  from .rule import Rule
21
29
  from .scope import check_scope, exec_function
22
30
  from .task import Task
@@ -45,6 +53,39 @@ class _TemplatePlaceholder:
45
53
  return str(self)
46
54
 
47
55
 
56
+ def _blocked_reason(rule):
57
+ """Why a rule is still deferred when a local run has nothing left to
58
+ try: its matrix's Defer paths if it still can't expand, else (it would
59
+ expand, so the planner is holding it back) an upstream that didn't
60
+ complete this run."""
61
+ from .rule import is_deferrable
62
+
63
+ if is_deferrable(rule.matrix):
64
+ try:
65
+ rule.matrix()
66
+ except Defer as e:
67
+ waiting = ', '.join(e.paths) if e.paths else 'an unspecified input'
68
+ return f'matrix not ready — waiting on {waiting}'
69
+ except Exception as e: # the matrix itself is broken: say so
70
+ return f'matrix raised {type(e).__name__}: {e}'
71
+ return 'an upstream rule did not complete (see failures above)'
72
+
73
+
74
+ def _where(rule):
75
+ fn = rule.fn
76
+ return f'{getattr(fn, "__module__", "?")}.{getattr(fn, "__qualname__", rule.name)}'
77
+
78
+
79
+ def _same_definition(a, b):
80
+ """Two Rule objects for the same function re-defined (e.g. a notebook
81
+ cell re-run): same module and qualified name, but freshly compiled code.
82
+ Distinct functions that merely share a rule name (two modules each
83
+ defining `process`, a clashing name=) are not — nor are rules built by
84
+ one factory function, which share a single code object and differ only
85
+ in closure values."""
86
+ return _where(a) == _where(b) and a.fn.__code__ is not b.fn.__code__
87
+
88
+
48
89
  class Remake:
49
90
  def __init__(
50
91
  self,
@@ -58,6 +99,10 @@ class Remake:
58
99
  self.config = config or {}
59
100
  self.metadata = metadata
60
101
  self.check_outputs = check_outputs
102
+ # Rules the last run() left unresolved (deferred matrix never ready,
103
+ # or waiting on an upstream that failed). Non-empty means that run
104
+ # did not complete everything; the CLI exits non-zero on it.
105
+ self.blocked_rules = []
61
106
  self.strict_scope = strict_scope
62
107
  self.rules = []
63
108
  self.dag = None
@@ -74,11 +119,26 @@ class Remake:
74
119
  raise RemakeError(f'Not a Rule (use the @rule decorator): {rule!r}')
75
120
  if rule in self.rules:
76
121
  continue
122
+ # Task keys are sha1('<rule name>:<kwargs>'), so two rules sharing
123
+ # a name share task keys and DB records (review 2026-09-24 H8).
124
+ existing = next((r for r in self.rules if r.name == rule.name), None)
125
+ if existing is not None and not _same_definition(existing, rule):
126
+ raise RemakeError(
127
+ f'Duplicate rule name {rule.name!r}: defined by '
128
+ f'{_where(existing)} and {_where(rule)}. Rule names must be '
129
+ f'unique within a pipeline (rename one, or pass name=...)'
130
+ )
77
131
  # Resolve tri-state strict_scope against the Remake default.
78
132
  if rule.strict_scope is None and self.strict_scope:
79
133
  check_scope(rule.fn, rule.uses, strict=True)
80
134
  rule.remake = self
81
- self.rules.append(rule)
135
+ if existing is not None:
136
+ # The same function re-defined — a notebook cell or script
137
+ # section executed again: the new definition replaces the old.
138
+ logger.warning(f'Rule {rule.name!r} redefined: replacing the earlier definition')
139
+ self.rules[self.rules.index(existing)] = rule
140
+ else:
141
+ self.rules.append(rule)
82
142
  self._finalized = False
83
143
 
84
144
  def rules_from_current_module(self):
@@ -186,7 +246,7 @@ class Remake:
186
246
  dynamic matrix is deferred."""
187
247
  if not self._finalized:
188
248
  self.finalize()
189
- predicate = make_predicate(query) if query else None
249
+ predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
190
250
  for rule in self.rules:
191
251
  try:
192
252
  yield from iter_expand_rule(rule, predicate)
@@ -285,7 +345,7 @@ class Remake:
285
345
  remaining = Counter(task.rule.name for task in runnable)
286
346
  runnable_keys = {task.key for task in runnable}
287
347
  deferred_names = {rule.name for rule in deferred}
288
- predicate = make_predicate(query) if query else None
348
+ predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
289
349
 
290
350
  # Per-rule tally of why the to-run tasks would rerun. One plan() is
291
351
  # already done (`runnable`); reuse it per task so this is plan-cost,
@@ -669,9 +729,21 @@ class Remake:
669
729
  """Run all tasks that need running, replanning after each wave so
670
730
  dynamic (deferred) matrices resolve as their upstreams complete.
671
731
  Returns the number of failed tasks (0 for asynchronous executors,
672
- which don't know at submission time)."""
732
+ which don't know at submission time). Holds `.remake/run.lock` for
733
+ the duration: a second concurrent run in the same directory is a
734
+ RemakeError."""
673
735
  if not self._finalized:
674
736
  self.finalize()
737
+ self.blocked_rules = []
738
+ if executor is not None and executor.handles_deferred:
739
+ # SLURM: submission only; continuation jobs re-enter run() on
740
+ # compute nodes, and the executor has its own duplicate-submission
741
+ # guard — a directory lock would strand or block them.
742
+ return self._run(executor, query, force, ignore_code_changes)
743
+ with run_lock(self.metadata):
744
+ return self._run(executor, query, force, ignore_code_changes)
745
+
746
+ def _run(self, executor, query, force, ignore_code_changes):
675
747
  # One run_seq for this whole invocation (shared across replanning
676
748
  # waves); committed onto every task so downstream propagation survives
677
749
  # to later invocations. See bugs/01_durable_rerun_propagation.md.
@@ -705,8 +777,19 @@ class Remake:
705
777
  runnable = [t for t in runnable if t.key not in attempted]
706
778
  if not runnable:
707
779
  if deferred:
708
- names = ', '.join(rule.name for rule in deferred)
709
- logger.warning(f'Blocked rules (matrix not ready): {names}')
780
+ # Not everything completed: say which rules and why, and
781
+ # make the run's outcome reflect it (review 2026-09-24 M9;
782
+ # it used to be a bare warning and exit 0). A filtered run
783
+ # may deliberately leave out the upstream a deferred rule
784
+ # waits on: that one is reported but doesn't fail the run.
785
+ excused = self._excluded_by_query(deferred) if query else set()
786
+ for rule in deferred:
787
+ if rule in excused:
788
+ logger.warning(f'Blocked rule {rule.name} (upstream not '
789
+ f'selected by the query): {_blocked_reason(rule)}')
790
+ else:
791
+ self.blocked_rules.append(rule)
792
+ logger.error(f'Blocked rule {rule.name}: {_blocked_reason(rule)}')
710
793
  break
711
794
  wave += 1
712
795
  logger.bind(event='wave', wave=wave, ntasks=len(runnable)).debug(
@@ -724,40 +807,71 @@ class Remake:
724
807
  logger.info('Nothing to do')
725
808
  return nfailed
726
809
 
810
+ def _excluded_by_query(self, deferred):
811
+ """Deferred rules a filtered run can be excused for: those with an
812
+ upstream (any ancestor) that still has work outside the filter. If
813
+ every ancestor is complete and the rule is still deferred, the query
814
+ didn't cause it — it's genuinely blocked."""
815
+ runnable_all, deferred_all = self.plan()
816
+ unfinished = {t.rule for t in runnable_all} | set(deferred_all)
817
+ return {rule for rule in deferred
818
+ if nx.ancestors(self.dag, rule) & (unfinished - {rule})}
819
+
727
820
  def run_task(self, task):
728
821
  """Execute one task and record the result. The single execution
729
822
  entry point — used by all executors and `remake run-task`. Timing and
730
823
  completion are logged here so every executor gets them uniformly
731
824
  (per-element detail at TRACE, per-task duration at DEBUG — the
732
825
  summarise-loops convention, per_task_logging.md)."""
733
- # opt(lazy=True): the path lists are only built when a TRACE sink is
734
- # attached (they'd cost real time at 1e6 tasks otherwise).
735
- logger.opt(lazy=True).trace(
736
- 'running {}: inputs {} -> outputs {}', lambda: task,
737
- lambda: [str(p) for p in task.inputs.values()],
738
- lambda: [str(p) for p in task.outputs.values()],
739
- )
740
- for token in task.outputs.values():
741
- if hasattr(token, '__fspath__'):
742
- Path(token).parent.mkdir(parents=True, exist_ok=True)
743
-
744
- fn = exec_function(task.rule.fn, task.rule.uses)
745
- args = []
746
- if task.rule.inputs is not None:
747
- args.append(task.inputs)
748
- if task.rule.outputs is not None:
749
- args.append(task.outputs)
750
826
  start = perf_counter()
827
+ # Everything that can fail — output dirs, io resolution (callable
828
+ # specs run here), the function itself — is inside the try, so every
829
+ # failure is recorded with its traceback (review 2026-09-24 M11).
751
830
  try:
752
- fn(*args, **task.kwargs)
753
- except Exception:
831
+ # opt(lazy=True): the path lists are only built when a TRACE sink
832
+ # is attached (they'd cost real time at 1e6 tasks otherwise).
833
+ logger.opt(lazy=True).trace(
834
+ 'running {}: inputs {} -> outputs {}', lambda: task,
835
+ lambda: [str(p) for p in task.inputs.values()],
836
+ lambda: [str(p) for p in task.outputs.values()],
837
+ )
838
+ for token in task.outputs.values():
839
+ if hasattr(token, '__fspath__'):
840
+ Path(token).parent.mkdir(parents=True, exist_ok=True)
841
+
842
+ fn = exec_function(task.rule.fn, task.rule.uses)
843
+ args = []
844
+ if task.rule.inputs is not None:
845
+ args.append(task.inputs)
846
+ if task.rule.outputs is not None:
847
+ args.append(task.outputs)
848
+ try:
849
+ fn(*args, **task.kwargs)
850
+ except SystemExit as exc:
851
+ # sys.exit(0) / sys.exit() from a wrapped CLI main() means
852
+ # success; only a non-zero code is a failure (handled below).
853
+ # Mirror CPython: only None or an int 0 (incl. False) exits 0;
854
+ # sys.exit(0.0) or sys.exit('msg') exit 1.
855
+ if not (exc.code is None or (isinstance(exc.code, int) and exc.code == 0)):
856
+ raise
857
+ except (Exception, SystemExit) as exc:
858
+ # SystemExit from task code (a CLI main() calling sys.exit) is a
859
+ # task failure, not a request to end the run (review H6).
860
+ # KeyboardInterrupt still propagates: Ctrl-C stops the run.
754
861
  elapsed = perf_counter() - start
862
+ tb = traceback.format_exc()
755
863
  logger.bind(event='task_failed', task=str(task), rule=task.rule.name,
756
864
  key=task.key, seconds=round(elapsed, 6),
757
865
  ).error(f'failed: {task} after {elapsed:.2f}s')
758
- self.metadata.update_task(
759
- task, TASK_STATUS_FAILED, exception=traceback.format_exc()
760
- )
866
+ # The traceback at DEBUG: lands in per-task logs (DEBUG sinks)
867
+ # without flooding the INFO console on runs with many failures
868
+ # (review M12).
869
+ logger.debug('traceback for {}:\n{}', task, tb)
870
+ self.metadata.update_task(task, TASK_STATUS_FAILED, exception=tb)
871
+ if isinstance(exc, SystemExit):
872
+ raise TaskExit(
873
+ f'{task} called sys.exit({exc.code!r})'
874
+ ) from exc
761
875
  raise
762
876
  elapsed = perf_counter() - start
763
877
  logger.bind(event='task_complete', task=str(task), rule=task.rule.name,
@@ -216,7 +216,12 @@ def rule(
216
216
  inputs=inputs,
217
217
  outputs=outputs,
218
218
  matrix=matrix,
219
- depends_on=list(depends_on) if depends_on else [],
219
+ # A bare name or Rule is one dependency — list('extract') would
220
+ # split it into characters (review 2026-09-24 L4).
221
+ depends_on=(
222
+ [depends_on] if isinstance(depends_on, (str, Rule))
223
+ else list(depends_on) if depends_on else []
224
+ ),
220
225
  uses=uses_,
221
226
  strict_scope=strict_scope,
222
227
  config=dict(config) if config else {},
@@ -23,13 +23,13 @@ import os
23
23
  from loguru import logger
24
24
 
25
25
  from ..core.exceptions import RemakeError
26
- from ..core.planner import upstream_failed
26
+ from ..core.planner import record_failure, upstream_failed
27
27
  from .executor import Executor
28
28
 
29
29
  _worker_rmk_cache = {}
30
30
 
31
31
 
32
- def _run_spec(remakefile, rule_name, kwargs):
32
+ def _run_spec(remakefile, rule_name, kwargs, run_seq=None):
33
33
  """Runs on a dask worker. Returns True on success."""
34
34
  from ..loader import load_remake
35
35
  from ..metadata.sidecar import SidecarWriter
@@ -40,6 +40,9 @@ def _run_spec(remakefile, rule_name, kwargs):
40
40
  rmk = load_remake(remakefile, finalize=False)
41
41
  rmk.metadata = SidecarWriter()
42
42
  _worker_rmk_cache[remakefile] = rmk
43
+ # The parent invocation's run_seq (durable propagation, bugs/01); set per
44
+ # call because long-lived workers outlive a single invocation.
45
+ rmk.metadata.run_seq = run_seq
43
46
  task = rmk.task_from_spec(rule_name, kwargs)
44
47
  logfile = task_log_path(task)
45
48
  logfile.parent.mkdir(parents=True, exist_ok=True)
@@ -95,14 +98,15 @@ class DaskExecutor(Executor):
95
98
  nfailed = 0
96
99
  nskipped = 0
97
100
  done = 0
98
- failures = {} # rule -> set of frozenset(kwargs.items())
101
+ failures = {} # see planner.record_failure
102
+ run_seq = self.rmk.metadata.current_run_seq()
99
103
  client, cluster = self._client()
100
104
  try:
101
105
  for rule, rule_tasks in groups:
102
106
  to_run = []
103
107
  for task in rule_tasks:
104
108
  if upstream_failed(task, failures):
105
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
109
+ record_failure(failures, task)
106
110
  nskipped += 1
107
111
  done += 1
108
112
  logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
@@ -113,7 +117,8 @@ class DaskExecutor(Executor):
113
117
  logger.info(f'{rule.name}: {len(to_run)} task(s) on dask ({self.nproc} workers)')
114
118
  futures = {
115
119
  client.submit(
116
- _run_spec, self.remakefile, rule.name, task.kwargs, pure=False
120
+ _run_spec, self.remakefile, rule.name, task.kwargs, run_seq,
121
+ pure=False,
117
122
  ): task
118
123
  for task in to_run
119
124
  }
@@ -124,7 +129,7 @@ class DaskExecutor(Executor):
124
129
  if future.result():
125
130
  logger.info(f'{done}/{ntasks}: {task}')
126
131
  else:
127
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
132
+ record_failure(failures, task)
128
133
  nfailed += 1
129
134
  logger.error(f'{done}/{ntasks} failed: {task}')
130
135
  finally:
@@ -22,7 +22,7 @@ from multiprocessing import get_context
22
22
  from loguru import logger
23
23
 
24
24
  from ..core.exceptions import RemakeError
25
- from ..core.planner import upstream_failed
25
+ from ..core.planner import record_failure, upstream_failed
26
26
  from .executor import Executor
27
27
 
28
28
  _worker_rmk = None
@@ -53,7 +53,10 @@ def _worker_init(remakefile):
53
53
  def _worker_run(spec):
54
54
  from ..util import task_log_path
55
55
 
56
- rule_name, kwargs = spec
56
+ rule_name, kwargs, run_seq = spec
57
+ # Stamp results with the parent invocation's run_seq so durable rerun
58
+ # propagation (bugs/01) works for multiproc runs as it does for SLURM.
59
+ _worker_rmk.metadata.run_seq = run_seq
57
60
  task = _worker_rmk.task_from_spec(rule_name, kwargs)
58
61
  logfile = task_log_path(task)
59
62
  logfile.parent.mkdir(parents=True, exist_ok=True)
@@ -93,7 +96,8 @@ class MultiprocExecutor(Executor):
93
96
  nfailed = 0
94
97
  nskipped = 0
95
98
  done = 0
96
- failures = {} # rule -> set of frozenset(kwargs.items())
99
+ failures = {} # see planner.record_failure
100
+ run_seq = self.rmk.metadata.current_run_seq()
97
101
  with ProcessPoolExecutor(
98
102
  max_workers=self.nproc,
99
103
  mp_context=get_context('spawn'),
@@ -107,7 +111,7 @@ class MultiprocExecutor(Executor):
107
111
  to_run = []
108
112
  for task in rule_tasks:
109
113
  if upstream_failed(task, failures):
110
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
114
+ record_failure(failures, task)
111
115
  nskipped += 1
112
116
  done += 1
113
117
  logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
@@ -117,7 +121,8 @@ class MultiprocExecutor(Executor):
117
121
  continue
118
122
  logger.info(f'{rule.name}: {len(to_run)} task(s) on {self.nproc} proc(s)')
119
123
  futures = {
120
- pool.submit(_worker_run, (rule.name, t.kwargs)): t for t in to_run
124
+ pool.submit(_worker_run, (rule.name, t.kwargs, run_seq)): t
125
+ for t in to_run
121
126
  }
122
127
  # Barrier: drain this rule before starting the next.
123
128
  for future in as_completed(futures):
@@ -126,7 +131,7 @@ class MultiprocExecutor(Executor):
126
131
  if future.result():
127
132
  logger.info(f'{done}/{ntasks}: {task}')
128
133
  else:
129
- failures.setdefault(rule, set()).add(frozenset(task.kwargs.items()))
134
+ record_failure(failures, task)
130
135
  nfailed += 1
131
136
  logger.error(f'{done}/{ntasks} failed: {task}')
132
137
  if nfailed:
@@ -2,7 +2,7 @@ import math
2
2
 
3
3
  from loguru import logger
4
4
 
5
- from ..core.planner import upstream_failed
5
+ from ..core.planner import record_failure, upstream_failed
6
6
  from .executor import Executor
7
7
 
8
8
 
@@ -12,7 +12,7 @@ class SingleprocExecutor(Executor):
12
12
  ndigits = math.floor(math.log10(ntasks)) + 1 if ntasks else 1
13
13
  nfailed = 0
14
14
  nskipped = 0
15
- failures = {} # rule -> set of frozenset(kwargs.items())
15
+ failures = {} # see planner.record_failure
16
16
  for i, task in enumerate(tasks):
17
17
  prefix = f'{i + 1:>{ndigits}}/{ntasks}'
18
18
  if upstream_failed(task, failures):
@@ -20,7 +20,7 @@ class SingleprocExecutor(Executor):
20
20
  # fail noisily on missing inputs. Left unrecorded (pending):
21
21
  # fixing the upstream makes the next run pick them up.
22
22
  # Counts as a failure for downstream propagation.
23
- failures.setdefault(task.rule, set()).add(frozenset(task.kwargs.items()))
23
+ record_failure(failures, task)
24
24
  nskipped += 1
25
25
  logger.warning(f'{prefix} skipped (upstream failed): {task}')
26
26
  continue
@@ -32,7 +32,7 @@ class SingleprocExecutor(Executor):
32
32
  raise # remake run -X: let the debugger see it
33
33
  # Failure is recorded by run_task; carry on so independent
34
34
  # tasks still run.
35
- failures.setdefault(task.rule, set()).add(frozenset(task.kwargs.items()))
35
+ record_failure(failures, task)
36
36
  nfailed += 1
37
37
  if nfailed:
38
38
  skipped = f' ({nskipped} downstream task(s) skipped)' if nskipped else ''
@@ -42,8 +42,14 @@ def load_remake(filename, finalize=True):
42
42
  from ..core.remake import Remake
43
43
 
44
44
  filename = Path(filename)
45
+ if not filename.name or filename.is_dir():
46
+ raise RemakeLoadError(f'Expected a remakefile (.py), got a directory: {filename}')
45
47
  if not filename.suffix:
46
48
  filename = filename.with_suffix('.py')
49
+ if filename.suffix != '.py':
50
+ raise RemakeLoadError(f'Not a Python remakefile (expected a .py file): {filename}')
51
+ if not (Path.cwd() / filename).is_file():
52
+ raise RemakeLoadError(f'Remakefile not found: {Path.cwd() / filename}')
47
53
  remake_module = load_module(filename)
48
54
  remakes = [o for o in vars(remake_module).values() if isinstance(o, Remake)]
49
55
  if len(remakes) > 1:
@@ -12,10 +12,11 @@ import json
12
12
  import random
13
13
  import sqlite3
14
14
  from pathlib import Path
15
- from time import perf_counter, sleep
15
+ from time import monotonic, perf_counter, sleep
16
16
 
17
17
  from loguru import logger
18
18
 
19
+ from ..core.exceptions import RemakeError
19
20
  from ..core.scope import io_hash as compute_io_hash
20
21
  from ..core.scope import raw_uses_parts
21
22
  from ..core.scope import uses_hash as compute_uses_hash
@@ -102,23 +103,92 @@ INSERT INTO meta(key, value) VALUES ('run_seq', 0);
102
103
  """
103
104
 
104
105
 
106
+ def _schema_statements():
107
+ """SQL_SCHEMA as single, idempotent statements, run one by one inside an
108
+ explicit transaction (executescript would COMMIT it first).
109
+
110
+ Idempotent (IF NOT EXISTS / OR IGNORE) so they can run on every open: a
111
+ DB left partially created by an older remake — whose executescript
112
+ committed statement by statement — gets its missing tables and index
113
+ instead of failing on the ones it has. sqlite3.complete_statement is
114
+ tokenizer-aware, so a ';' inside a comment doesn't split a statement."""
115
+ stmts, buf = [], ''
116
+ for line in SQL_SCHEMA.splitlines(keepends=True):
117
+ buf += line
118
+ if sqlite3.complete_statement(buf):
119
+ stmts.append(_idempotent(buf))
120
+ buf = ''
121
+ assert not buf.strip(), 'SQL_SCHEMA ends with an incomplete statement'
122
+ return stmts
123
+
124
+
125
+ def _idempotent(stmt):
126
+ for old, new in (('CREATE TABLE ', 'CREATE TABLE IF NOT EXISTS '),
127
+ ('CREATE UNIQUE INDEX ', 'CREATE UNIQUE INDEX IF NOT EXISTS '),
128
+ ('CREATE INDEX ', 'CREATE INDEX IF NOT EXISTS '),
129
+ ('INSERT INTO ', 'INSERT OR IGNORE INTO ')):
130
+ stmt = stmt.replace(old, new)
131
+ return stmt
132
+
133
+
134
+ # How long a writer keeps retrying a locked DB before giving up. Long enough
135
+ # to sit out another process's one-time migration or a big sidecar ingest;
136
+ # finite, so a DB that stays locked becomes an error rather than a hang.
137
+ LOCK_RETRY_SECONDS = 600
138
+
139
+
140
+ def _quarantine(path, why):
141
+ """Move a bad sidecar aside (`*.json.bad`, outside the ingest glob) with
142
+ one warning, instead of failing or re-warning on every command."""
143
+ bad = path.with_name(path.name + '.bad')
144
+ try:
145
+ path.replace(bad)
146
+ except FileNotFoundError:
147
+ return # another process got there first
148
+ logger.warning(f'Quarantined {why} sidecar result: {bad}')
149
+
150
+
151
+ def _is_lock_error(exc):
152
+ msg = str(exc).lower()
153
+ return 'locked' in msg or 'busy' in msg
154
+
155
+
156
+ def _retry_locked(fn, what):
157
+ """Call fn(), retrying with capped exponential backoff while SQLite
158
+ reports lock contention. Any other OperationalError ("no such table",
159
+ disk full, malformed schema) is re-raised at once instead of being
160
+ retried forever (todos: "Bound and message-match retry_lock_commit")."""
161
+ deadline = monotonic() + LOCK_RETRY_SECONDS
162
+ nattempts = 1
163
+ while True:
164
+ try:
165
+ return fn()
166
+ except sqlite3.OperationalError as oe:
167
+ if not _is_lock_error(oe):
168
+ raise
169
+ if monotonic() >= deadline:
170
+ raise RemakeError(
171
+ f'{what}: database still locked after {LOCK_RETRY_SECONDS}s '
172
+ f'— is another remake process stuck? ({oe})'
173
+ ) from oe
174
+ logger.debug(f'{what}: {oe} (attempt {nattempts}); retrying')
175
+ nattempts += 1
176
+ sleep(min(30, 2**nattempts * random.random()))
177
+
178
+
105
179
  def retry_lock_commit(fn):
106
- """Run fn in an EXCLUSIVE transaction, retrying with exponential backoff
107
- on lock contention (concurrent workers/SLURM jobs share the DB)."""
180
+ """Run fn in an EXCLUSIVE transaction, retrying with backoff on lock
181
+ contention (concurrent workers/SLURM jobs share the DB)."""
108
182
 
109
183
  def inner(self, *args, **kwargs):
110
- nattempts = 1
111
- while True:
112
- try:
113
- with self.conn:
114
- self.conn.execute('BEGIN EXCLUSIVE')
115
- ret = fn(self, *args, **kwargs)
116
- self.conn.commit()
117
- return ret
118
- except sqlite3.OperationalError as oe:
119
- logger.debug(f'OperationalError: {oe}')
120
- nattempts += 1
121
- sleep(2**nattempts * random.random())
184
+ def attempt():
185
+ with self.conn:
186
+ self.conn.execute('BEGIN EXCLUSIVE')
187
+ ret = fn(self, *args, **kwargs)
188
+ self.conn.commit()
189
+ return ret
190
+
191
+ return _retry_locked(attempt, fn.__name__)
122
192
 
123
193
  return inner
124
194
 
@@ -127,18 +197,25 @@ class Sqlite3Backend(MetadataManager):
127
197
  def __init__(self, dbloc='.remake/remake.db'):
128
198
  self.dbloc = str(dbloc)
129
199
  self.code_comparer = CodeComparer()
130
- in_memory = self.dbloc == ':memory:'
131
- create_db = in_memory or not Path(self.dbloc).exists()
132
- if create_db and not in_memory:
133
- logger.info(f'Creating sqlite3 database: {self.dbloc}')
200
+ if self.dbloc != ':memory:':
134
201
  Path(self.dbloc).parent.mkdir(parents=True, exist_ok=True)
135
202
  # No detect_types: timestamps are read as plain strings (TaskRecord
136
203
  # .timestamp), and the implicit converter is deprecated in 3.12.
137
204
  self.conn = sqlite3.connect(self.dbloc)
138
- if create_db:
139
- self.conn.executescript(SQL_SCHEMA)
140
- else:
141
- self._add_missing_columns()
205
+ try:
206
+ self._init_schema()
207
+ except sqlite3.DatabaseError as e:
208
+ self.conn.close()
209
+ if isinstance(e, sqlite3.OperationalError):
210
+ raise
211
+ # e.g. "file is not a database": a corrupt or foreign file.
212
+ raise RemakeError(
213
+ f'{self.dbloc} is not a usable remake database ({e}); move it '
214
+ f'aside to start a fresh one'
215
+ ) from None
216
+ except BaseException:
217
+ self.conn.close()
218
+ raise
142
219
  self.conn.isolation_level = 'EXCLUSIVE'
143
220
  # rule name -> (rule_id, run_code_id, uses_code_id, io_code_id):
144
221
  # the rule's row plus this invocation's current interned code ids.
@@ -169,18 +246,65 @@ class Sqlite3Backend(MetadataManager):
169
246
  except Exception:
170
247
  pass
171
248
 
249
+ def _init_schema(self):
250
+ """Create the schema, or migrate an existing DB, in ONE exclusive
251
+ transaction (review 2026-09-24 M5/M6).
252
+
253
+ Whether to create is decided by the presence of the `task` table,
254
+ checked inside the lock — not by the file existing: sqlite3.connect
255
+ creates the file before any schema is written, so a first run killed
256
+ mid-create (disk/quota full, Ctrl-C) used to leave a 0-byte file that
257
+ bricked every later command. Holding the lock across check + DDL also
258
+ serialises concurrent first openers (continuation jobs, `info` next
259
+ to `run`), which used to race to ALTER the same columns; and a crash
260
+ mid-migration now rolls back cleanly and is redone on the next open,
261
+ instead of leaving a half-migrated DB that is skipped forever."""
262
+ conn = self.conn
263
+ conn.isolation_level = None # explicit transaction control below
264
+ _retry_locked(lambda: conn.execute('BEGIN EXCLUSIVE'), 'opening the database')
265
+ try:
266
+ tables = {row[0] for row in conn.execute(
267
+ "SELECT name FROM sqlite_master WHERE type='table'")}
268
+ if 'task' not in tables and self.dbloc != ':memory:':
269
+ logger.info(f'Creating sqlite3 database: {self.dbloc}')
270
+ # Old-shape tables that exist are left alone here (IF NOT EXISTS)
271
+ # and brought up to date column by column below.
272
+ for stmt in _schema_statements():
273
+ conn.execute(stmt)
274
+ needs_vacuum = self._add_missing_columns()
275
+ conn.execute('COMMIT')
276
+ except BaseException:
277
+ # SQLite may already have rolled back by itself (disk full, I/O
278
+ # error); an unconditional ROLLBACK would then raise "no
279
+ # transaction is active" and hide the real error.
280
+ if conn.in_transaction:
281
+ conn.execute('ROLLBACK')
282
+ raise
283
+ if needs_vacuum:
284
+ # VACUUM can't run inside a transaction, so it follows the
285
+ # (committed) migration. Best effort: if another process holds
286
+ # the DB it is skipped — the space is reclaimed on a later VACUUM.
287
+ logger.info('Vacuuming (one-time; reclaims the inline-hash space)')
288
+ try:
289
+ conn.execute('VACUUM')
290
+ except sqlite3.OperationalError as e:
291
+ logger.warning(f'VACUUM skipped ({e}); the DB is correct, just larger')
292
+
172
293
  def _add_missing_columns(self):
173
294
  """Lightweight forward-compat for columns added after a DB was first
174
295
  created (still no general migration support — see the module docstring).
175
296
  A pre-existing record left with io_code_id/run_seq NULL is treated as
176
297
  'not yet tracked' by the planner, so upgrading does not force a mass
177
- rerun."""
298
+ rerun. Runs inside _init_schema's exclusive transaction: no commits
299
+ here. Returns True when a VACUUM should follow."""
300
+ needs_vacuum = False
178
301
  cols = {row[1] for row in self.conn.execute('PRAGMA table_info(task)')}
179
302
  if 'run_seq' not in cols:
180
303
  logger.info('Adding task.run_seq column to existing DB')
181
304
  self.conn.execute('ALTER TABLE task ADD COLUMN run_seq INTEGER')
182
305
  if 'uses_code_id' not in cols:
183
306
  self._migrate_inline_hashes_to_code_ids(cols)
307
+ needs_vacuum = True
184
308
  rule_cols = {row[1] for row in self.conn.execute('PRAGMA table_info(rule)')}
185
309
  if 'remakefile' not in rule_cols:
186
310
  logger.info('Adding rule.remakefile column to existing DB')
@@ -201,14 +325,15 @@ class Sqlite3Backend(MetadataManager):
201
325
  self.conn.execute(
202
326
  'CREATE TABLE meta (key TEXT NOT NULL PRIMARY KEY, value INTEGER NOT NULL)')
203
327
  self.conn.execute("INSERT INTO meta(key, value) VALUES ('run_seq', 0)")
204
- self.conn.commit()
328
+ return needs_vacuum
205
329
 
206
330
  def _migrate_inline_hashes_to_code_ids(self, cols):
207
331
  """One-time in-place migration: the old task.uses_hash/io_hash columns
208
332
  stored the full normalised uses/io strings inline per row — duplicated
209
333
  across every task of a rule (measured at 99.8% of a 272 MB field DB).
210
334
  Intern each distinct value into `code` once, point integer FKs at it,
211
- drop the text columns, and VACUUM to return the space."""
335
+ drop the text columns; the caller VACUUMs afterwards to return the
336
+ space."""
212
337
  logger.info('Migrating task.uses_hash/io_hash to code-table FKs')
213
338
  self.conn.execute('ALTER TABLE task ADD COLUMN uses_code_id INTEGER')
214
339
  self.conn.execute('ALTER TABLE task ADD COLUMN io_code_id INTEGER')
@@ -242,9 +367,6 @@ class Sqlite3Backend(MetadataManager):
242
367
  # frees the space (after the VACUUM below) and nothing reads
243
368
  # it any more.
244
369
  self.conn.execute(f'UPDATE task SET {col} = NULL')
245
- self.conn.commit()
246
- logger.info('Vacuuming (one-time; reclaims the inline-hash space)')
247
- self.conn.execute('VACUUM')
248
370
 
249
371
  @retry_lock_commit
250
372
  def _allocate_run_seq(self):
@@ -471,8 +593,15 @@ class Sqlite3Backend(MetadataManager):
471
593
  payload = json.loads(path.read_text())
472
594
  except FileNotFoundError:
473
595
  continue # another process ingested it first
474
- except json.JSONDecodeError:
475
- logger.warning(f'Skipping unreadable sidecar: {path}')
596
+ except (json.JSONDecodeError, UnicodeDecodeError):
597
+ _quarantine(path, 'unreadable')
598
+ continue
599
+ if not (isinstance(payload, dict)
600
+ and isinstance(payload.get('status'), int)):
601
+ # Valid JSON of the wrong shape (hand-edited, another
602
+ # remake version) used to crash every plan/info/run
603
+ # inside the ingest transaction (review 2026-09-24 L14).
604
+ _quarantine(path, 'malformed (no integer "status")')
476
605
  continue
477
606
  logger.trace('sidecar {} ({}): {}', key, rule.name, payload.get('status'))
478
607
  pending.append((rule, key, payload, path))
@@ -517,7 +646,16 @@ class Sqlite3Backend(MetadataManager):
517
646
  ' run_seq = excluded.run_seq, '
518
647
  ' last_run_timestamp = excluded.last_run_timestamp, '
519
648
  ' last_run_status = excluded.last_run_status, '
520
- ' exception = excluded.exception',
649
+ ' exception = excluded.exception '
650
+ # Never let an older result overwrite a newer one (review
651
+ # 2026-09-24 M17): a sidecar left un-ingested (SLURM element,
652
+ # dead multiproc parent) must not revert a later write such as
653
+ # `remake run-task`. Ordered by run_seq — allocated centrally
654
+ # per invocation — not wall clocks, which skew between compute
655
+ # and login nodes. Without a run_seq on either side (legacy
656
+ # records/sidecars) the sidecar applies, as before.
657
+ 'WHERE task.run_seq IS NULL OR excluded.run_seq IS NULL '
658
+ ' OR excluded.run_seq >= task.run_seq',
521
659
  (
522
660
  key,
523
661
  rule_id,
@@ -107,7 +107,10 @@ def _make_executor(name, rmk, nproc=None):
107
107
  f"{sorted([*builtin, 'multiproc', 'dask'])} or a "
108
108
  f'dotted path like mymodule:MyExecutor'
109
109
  )
110
- cls = getattr(importlib.import_module(module_name), cls_name)
110
+ try:
111
+ cls = getattr(importlib.import_module(module_name), cls_name)
112
+ except (ImportError, AttributeError) as e:
113
+ raise RemakeError(f'Cannot load executor {name!r}: {e}') from None
111
114
  if not (isinstance(cls, type) and issubclass(cls, Executor)):
112
115
  raise RemakeError(f'{name!r} is not an Executor subclass')
113
116
  return cls(rmk)
@@ -404,7 +407,9 @@ class RemakeCLI:
404
407
  force=args.force,
405
408
  ignore_code_changes=args.ignore_code_changes,
406
409
  )
407
- return 1 if nfailed else 0
410
+ # Exit 0 only when everything needed completed: failed tasks and
411
+ # rules left blocked (deferred matrix never ready) are both non-zero.
412
+ return 1 if nfailed or rmk.blocked_rules else 0
408
413
 
409
414
  def remake_run_task(self, args):
410
415
  rmk = self._load(args)
@@ -433,6 +438,11 @@ class RemakeCLI:
433
438
  f'remake run {args.remakefile} --executor slurm'
434
439
  )
435
440
  specs = json.loads(specs_path.read_text())
441
+ if not 0 <= args.index < len(specs):
442
+ raise RemakeError(
443
+ f'array index {args.index} out of range: {specs_path} has '
444
+ f'{len(specs)} task(s)'
445
+ )
436
446
  spec = specs[args.index]
437
447
  # run_seq was fixed at submission; carry it into the sidecar so its
438
448
  # stamp matches the rest of this submission's tasks (older job specs
@@ -601,11 +611,12 @@ class RemakeCLI:
601
611
  def remake_ls_tasks(self, args):
602
612
  from .core.dag import iter_expand_rule
603
613
  from .core.exceptions import Defer
604
- from .core.planner import make_predicate
614
+ from .core.planner import make_predicate, rule_kwarg_names
605
615
 
606
616
  rmk = self._load(args)
607
617
  rmk.finalize()
608
- predicate = make_predicate(args.query) if args.query else None
618
+ predicate = (make_predicate(args.query, rule_kwarg_names(rmk.rules))
619
+ if args.query else None)
609
620
  paint = Painter(args.colour)
610
621
 
611
622
  def input_files(task):
@@ -899,17 +910,21 @@ def remake_cmd(argv=None):
899
910
  # Logs go to stderr; stdout carries command output only (so --json and
900
911
  # piping stay clean).
901
912
  logger.remove()
913
+ # Honour --colour / NO_COLOR / a non-TTY stderr like the rest of the
914
+ # output (review 2026-09-24 L25: hardcoded colorize=True filled CI logs
915
+ # and SLURM .err files with escape codes).
916
+ colorize = Painter(getattr(args, 'colour', 'auto'), sys.stderr).enabled
902
917
  if args.trace:
903
- logger.add(sys.stderr, colorize=True, level='TRACE')
918
+ logger.add(sys.stderr, colorize=colorize, level='TRACE')
904
919
  elif args.debug:
905
- logger.add(sys.stderr, colorize=True, level='DEBUG')
920
+ logger.add(sys.stderr, colorize=colorize, level='DEBUG')
906
921
  elif args.warning:
907
922
  logger.add(
908
- sys.stderr, colorize=True, format='<bold><lvl>{message}</lvl></bold>', level='WARNING'
923
+ sys.stderr, colorize=colorize, format='<lvl>{message}</lvl>', level='WARNING'
909
924
  )
910
925
  else:
911
926
  logger.add(
912
- sys.stderr, colorize=True, format='<bold><lvl>{message}</lvl></bold>', level='INFO'
927
+ sys.stderr, colorize=colorize, format='<lvl>{message}</lvl>', level='INFO'
913
928
  )
914
929
 
915
930
  # Anchor execution to the remakefile's directory: cd there so .remake/ and
@@ -927,7 +942,7 @@ def remake_cmd(argv=None):
927
942
  except OSError as e:
928
943
  print(f'error: cannot enter remakefile directory {rf.parent}: {e}',
929
944
  file=sys.stderr)
930
- return 1
945
+ return 2
931
946
  args.remakefile = rf.name
932
947
 
933
948
  # Per-task-process subcommands (SLURM array elements) get a per-task log
@@ -941,7 +956,12 @@ def remake_cmd(argv=None):
941
956
  # structured sink), so a miner can group an invocation's lines and
942
957
  # correlate e.g. a plan total with its constituent status queries.
943
958
  logger.configure(extra={'run_id': uuid.uuid4().hex[:12]})
944
- Path('.remake').mkdir(parents=True, exist_ok=True)
959
+ try:
960
+ Path('.remake').mkdir(parents=True, exist_ok=True)
961
+ except OSError as e:
962
+ print(f'error: cannot create .remake/ metadata dir: {e}', file=sys.stderr)
963
+ os.chdir(orig_cwd)
964
+ return 2
945
965
  debug_level = 'TRACE' if args.trace else 'DEBUG'
946
966
  # Three always-on file sinks next to the metadata DB, split so the
947
967
  # streams don't compete for one rotation window (logs_analysis §3.2):
@@ -970,6 +990,14 @@ def remake_cmd(argv=None):
970
990
 
971
991
  try:
972
992
  return cli.dispatch()
993
+ except BrokenPipeError:
994
+ # Output piped into something that stopped reading (`| head`): stop
995
+ # quietly like other CLI tools, not with a traceback (review L26).
996
+ # Point stdout at devnull so the interpreter's final flush can't
997
+ # raise again.
998
+ devnull = os.open(os.devnull, os.O_WRONLY)
999
+ os.dup2(devnull, sys.stdout.fileno())
1000
+ return 141 # 128 + SIGPIPE, what a shell reports for a killed writer
973
1001
  except RemakeError as e:
974
1002
  # User-facing errors (bad query, >1-task match, unknown rule, ...)
975
1003
  # print cleanly and exit 2; keep the traceback only under -X.
@@ -0,0 +1,77 @@
1
+ """A per-directory lock so two local `remake run`s can't execute the same
2
+ tasks at once (review 2026-09-24 M15: both would run every task and race on
3
+ the same output files).
4
+
5
+ The lock file sits next to the metadata DB (`.remake/run.lock`) and records
6
+ the holder's host and PID. It is created atomically (O_EXCL) and removed when
7
+ the run ends, including on Ctrl-C. A lock left by a process that died on this
8
+ host is detected (its PID is gone) and taken over; one from another host (a
9
+ shared filesystem) can't be checked, so the error says how to clear it.
10
+ """
11
+ import contextlib
12
+ import json
13
+ import os
14
+ import socket
15
+ from pathlib import Path
16
+
17
+ from loguru import logger
18
+
19
+ from ..core.exceptions import RemakeError
20
+
21
+
22
+ def _pid_alive(pid):
23
+ try:
24
+ os.kill(pid, 0)
25
+ except ProcessLookupError:
26
+ return False
27
+ except PermissionError:
28
+ return True # exists, owned by someone else
29
+ return True
30
+
31
+
32
+ def _read_holder(path):
33
+ try:
34
+ return json.loads(path.read_text())
35
+ except (OSError, ValueError):
36
+ return None
37
+
38
+
39
+ @contextlib.contextmanager
40
+ def run_lock(metadata):
41
+ dbloc = getattr(metadata, 'dbloc', None)
42
+ if not dbloc or dbloc == ':memory:':
43
+ yield # nothing on disk to protect (tests, custom backends)
44
+ return
45
+ path = Path(dbloc).parent / 'run.lock'
46
+ me = {'host': socket.gethostname(), 'pid': os.getpid()}
47
+ for attempt in (1, 2):
48
+ try:
49
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
50
+ except FileExistsError:
51
+ holder = _read_holder(path) or {}
52
+ same_host = holder.get('host') == me['host']
53
+ pid = holder.get('pid')
54
+ if attempt == 1 and same_host and isinstance(pid, int) and not _pid_alive(pid):
55
+ # Re-check right before removing: if another run has just
56
+ # replaced the stale lock with its own, leave that one alone
57
+ # (narrows the takeover race to a re-read-then-unlink window).
58
+ if _read_holder(path) == holder:
59
+ logger.warning(f'Removing stale run lock left by dead process {pid}: {path}')
60
+ path.unlink(missing_ok=True)
61
+ continue
62
+ who = f'{holder.get("host", "?")} pid {holder.get("pid", "?")}'
63
+ raise RemakeError(
64
+ f'another remake run is active in this directory ({who}; lock '
65
+ f'{path}). Wait for it to finish; if it is not running, delete '
66
+ f'{path}'
67
+ ) from None
68
+ with os.fdopen(fd, 'w') as f:
69
+ json.dump(me, f)
70
+ break
71
+ try:
72
+ yield
73
+ finally:
74
+ # Only remove our own lock: if it was deleted by hand and another
75
+ # run took the directory, that run's lock must survive our exit.
76
+ if _read_holder(path) == me:
77
+ path.unlink(missing_ok=True)
@@ -1,5 +1,5 @@
1
1
  # Must remain a simple literal: pyproject reads it statically at build time.
2
- __version__ = '0.8.3'
2
+ __version__ = '0.8.4'
3
3
 
4
4
  if __name__ == '__main__':
5
5
  print(__version__)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: remake
3
- Version: 0.8.3
3
+ Version: 0.8.4
4
4
  Summary: Smart remake tool
5
5
  Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
6
6
  Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
@@ -35,4 +35,5 @@ src/remake/util/code_compare.py
35
35
  src/remake/util/colour.py
36
36
  src/remake/util/command_line_args.py
37
37
  src/remake/util/config.py
38
+ src/remake/util/run_lock.py
38
39
  src/remake/util/util.py
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes