remake 0.8.3__tar.gz → 0.8.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {remake-0.8.3 → remake-0.8.4}/PKG-INFO +1 -1
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/dag.py +7 -3
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/exceptions.py +17 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/planner.py +110 -15
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/remake.py +144 -30
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/rule.py +6 -1
- {remake-0.8.3 → remake-0.8.4}/src/remake/executors/dask_executor.py +11 -6
- {remake-0.8.3 → remake-0.8.4}/src/remake/executors/multiproc_executor.py +11 -6
- {remake-0.8.3 → remake-0.8.4}/src/remake/executors/singleproc_executor.py +4 -4
- {remake-0.8.3 → remake-0.8.4}/src/remake/loader/__init__.py +6 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/sqlite3_backend.py +170 -32
- {remake-0.8.3 → remake-0.8.4}/src/remake/remake_cmd.py +38 -10
- remake-0.8.4/src/remake/util/run_lock.py +77 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/version.py +1 -1
- {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/PKG-INFO +1 -1
- {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/SOURCES.txt +1 -0
- {remake-0.8.3 → remake-0.8.4}/LICENCE +0 -0
- {remake-0.8.3 → remake-0.8.4}/README.md +0 -0
- {remake-0.8.3 → remake-0.8.4}/pyproject.toml +0 -0
- {remake-0.8.3 → remake-0.8.4}/setup.cfg +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/__init__.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/__init__.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/scope.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/task.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/core/tokens.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/executors/__init__.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/executors/executor.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/executors/slurm_executor.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/__init__.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/metadata_manager.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/metadata/sidecar.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/util/__init__.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/util/code_compare.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/util/colour.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/util/command_line_args.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/util/config.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake/util/util.py +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/dependency_links.txt +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/entry_points.txt +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/requires.txt +0 -0
- {remake-0.8.3 → remake-0.8.4}/src/remake.egg-info/top_level.txt +0 -0
|
@@ -7,7 +7,7 @@ import itertools
|
|
|
7
7
|
|
|
8
8
|
import networkx as nx
|
|
9
9
|
|
|
10
|
-
from .exceptions import Defer, SignatureError
|
|
10
|
+
from .exceptions import Defer, RuleGraphError, SignatureError
|
|
11
11
|
from .rule import is_deferrable
|
|
12
12
|
from .task import Task
|
|
13
13
|
|
|
@@ -20,9 +20,13 @@ def build_rule_dag(rules):
|
|
|
20
20
|
g.add_node(rule)
|
|
21
21
|
resolved = []
|
|
22
22
|
for dep in rule.depends_on:
|
|
23
|
+
if not isinstance(dep, str) and getattr(dep, 'name', None) in rules_by_name:
|
|
24
|
+
# A Rule object resolved earlier may since have been replaced
|
|
25
|
+
# by a redefinition (notebook cell re-run): use the current one.
|
|
26
|
+
dep = rules_by_name[dep.name]
|
|
23
27
|
if isinstance(dep, str):
|
|
24
28
|
if dep not in rules_by_name:
|
|
25
|
-
raise
|
|
29
|
+
raise RuleGraphError(
|
|
26
30
|
f'{rule.name}: depends_on references unknown rule {dep!r}'
|
|
27
31
|
)
|
|
28
32
|
dep = rules_by_name[dep]
|
|
@@ -31,7 +35,7 @@ def build_rule_dag(rules):
|
|
|
31
35
|
rule.depends_on = resolved
|
|
32
36
|
if not nx.is_directed_acyclic_graph(g):
|
|
33
37
|
cycle = nx.find_cycle(g)
|
|
34
|
-
raise
|
|
38
|
+
raise RuleGraphError(f'Rule dependencies contain a cycle: {cycle}')
|
|
35
39
|
return g
|
|
36
40
|
|
|
37
41
|
|
|
@@ -18,6 +18,23 @@ class ScopeError(RemakeError):
|
|
|
18
18
|
pass
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
class RuleGraphError(RemakeError, ValueError):
|
|
22
|
+
"""Bad rule dependencies (unknown depends_on name, a cycle). Also a
|
|
23
|
+
ValueError, which is what these were raised as before 0.8.4."""
|
|
24
|
+
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TaskExit(Exception):
|
|
29
|
+
"""A task's code raised SystemExit (e.g. a CLI `main()` calling
|
|
30
|
+
`sys.exit`). run_task records the task as failed and raises this in its
|
|
31
|
+
place, so executors treat it as an ordinary task failure instead of the
|
|
32
|
+
whole run exiting silently. Deliberately not a RemakeError: it is a task
|
|
33
|
+
failure, not a usage error."""
|
|
34
|
+
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
21
38
|
class Defer(Exception):
|
|
22
39
|
"""Raised by a `@deferrable` matrix callable to signal that the rule
|
|
23
40
|
cannot be expanded this wave — its task list derives from an upstream
|
|
@@ -5,7 +5,11 @@ Rerun logic is DB-first: never run, failed, rule_run code changed (AST
|
|
|
5
5
|
compare), uses changed, or any relevant upstream task reruns. Filesystem
|
|
6
6
|
checks happen only via the opt-in check_outputs modes.
|
|
7
7
|
"""
|
|
8
|
+
import ast
|
|
9
|
+
import builtins
|
|
8
10
|
import difflib
|
|
11
|
+
import inspect
|
|
12
|
+
import os
|
|
9
13
|
from collections import namedtuple
|
|
10
14
|
from time import perf_counter
|
|
11
15
|
|
|
@@ -15,7 +19,7 @@ from loguru import logger
|
|
|
15
19
|
from ..metadata.metadata_manager import TASK_STATUS_FAILED, TASK_STATUS_SUCCESS
|
|
16
20
|
from ..util.code_compare import CodeComparer
|
|
17
21
|
from .dag import expand_rule
|
|
18
|
-
from .exceptions import Defer
|
|
22
|
+
from .exceptions import Defer, RemakeError
|
|
19
23
|
from .rule import is_deferrable
|
|
20
24
|
from .scope import (
|
|
21
25
|
io_hash,
|
|
@@ -27,20 +31,75 @@ from .scope import (
|
|
|
27
31
|
)
|
|
28
32
|
|
|
29
33
|
|
|
30
|
-
|
|
34
|
+
# Builtins a query may call (review 2026-09-24 M14): with none, reasonable
|
|
35
|
+
# queries like "year in range(2000, 2005)" silently matched nothing.
|
|
36
|
+
QUERY_BUILTINS = {
|
|
37
|
+
name: getattr(builtins, name)
|
|
38
|
+
for name in ('abs', 'all', 'any', 'bool', 'float', 'frozenset', 'int', 'len',
|
|
39
|
+
'list', 'max', 'min', 'range', 'round', 'set', 'sorted', 'str', 'tuple')
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def rule_kwarg_names(rules):
|
|
44
|
+
"""Every matrix kwarg name across `rules` — the rule functions'
|
|
45
|
+
parameters after inputs/outputs (the signature contract makes these the
|
|
46
|
+
matrix keys, for callable matrices too)."""
|
|
47
|
+
names = set()
|
|
48
|
+
for rule in rules:
|
|
49
|
+
names.update(p for p in inspect.signature(rule.fn).parameters
|
|
50
|
+
if p not in ('inputs', 'outputs'))
|
|
51
|
+
return names
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _free_names(tree):
|
|
55
|
+
"""Names a query expression reads, minus those it binds itself
|
|
56
|
+
(comprehension targets, lambda arguments)."""
|
|
57
|
+
loaded, bound = set(), set()
|
|
58
|
+
for node in ast.walk(tree):
|
|
59
|
+
if isinstance(node, ast.Name):
|
|
60
|
+
(loaded if isinstance(node.ctx, ast.Load) else bound).add(node.id)
|
|
61
|
+
elif isinstance(node, ast.arg):
|
|
62
|
+
bound.add(node.arg)
|
|
63
|
+
return loaded - bound
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def make_predicate(query, known_names=None):
|
|
31
67
|
"""Compile a task-filter expression evaluated against task kwargs plus
|
|
32
68
|
'rule' (the rule name), e.g. "year > 1985 and model == 'era5'",
|
|
33
|
-
"rule in ['extract', 'clean']".
|
|
69
|
+
"rule in ['extract', 'clean']". A few safe builtins are available
|
|
70
|
+
(QUERY_BUILTINS).
|
|
71
|
+
|
|
72
|
+
`known_names` (every kwarg name in the pipeline, see rule_kwarg_names)
|
|
73
|
+
turns a name no rule has — almost always a typo — into an error instead
|
|
74
|
+
of a silent no-match. Bad syntax and evaluation errors are RemakeErrors
|
|
75
|
+
(a clean CLI error) rather than raw tracebacks."""
|
|
34
76
|
# MM: this looks like a risk - using compile to compile the code?
|
|
35
77
|
# See how pyquerylist does this - it only allows certain Python ops.
|
|
36
|
-
|
|
37
|
-
|
|
78
|
+
try:
|
|
79
|
+
tree = ast.parse(query, mode='eval')
|
|
80
|
+
code = compile(tree, '<query>', 'eval')
|
|
81
|
+
except SyntaxError as e:
|
|
82
|
+
raise RemakeError(f'invalid query {query!r}: {e.msg}') from None
|
|
83
|
+
if known_names is not None:
|
|
84
|
+
unknown = _free_names(tree) - set(known_names) - {'rule'} - QUERY_BUILTINS.keys()
|
|
85
|
+
if unknown:
|
|
86
|
+
raise RemakeError(
|
|
87
|
+
f'query {query!r} uses unknown name(s) {sorted(unknown)} — not a '
|
|
88
|
+
f'matrix key of any rule (keys: {sorted(known_names)}), "rule", or an '
|
|
89
|
+
f'allowed builtin ({", ".join(sorted(QUERY_BUILTINS))})'
|
|
90
|
+
)
|
|
38
91
|
def predicate(kwargs):
|
|
39
92
|
try:
|
|
40
|
-
|
|
93
|
+
# kwargs go in the *globals*: a generator expression (e.g. inside
|
|
94
|
+
# any()) has its own scope and can't see eval's locals.
|
|
95
|
+
return bool(eval(code, {**kwargs, '__builtins__': QUERY_BUILTINS}))
|
|
41
96
|
except NameError:
|
|
42
97
|
# Query references a kwarg this rule doesn't have: no match.
|
|
43
98
|
return False
|
|
99
|
+
except Exception as e:
|
|
100
|
+
raise RemakeError(
|
|
101
|
+
f'query {query!r} failed on {dict(kwargs)}: {type(e).__name__}: {e}'
|
|
102
|
+
) from None
|
|
44
103
|
|
|
45
104
|
return predicate
|
|
46
105
|
|
|
@@ -124,22 +183,53 @@ def cascade_settled(rule_set, dag, selected, run_seq, status):
|
|
|
124
183
|
return settled
|
|
125
184
|
|
|
126
185
|
|
|
186
|
+
# Key in an executor's `failures` dict holding the normalised output paths of
|
|
187
|
+
# every task that failed (or was skipped) this run.
|
|
188
|
+
_FAILED_OUTPUTS = '_failed_outputs'
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _norm_paths(values):
|
|
192
|
+
return {os.path.normpath(str(v)) for v in values}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _io_paths(task, part):
|
|
196
|
+
"""A task's normalised input or output paths, or an empty set if its
|
|
197
|
+
callable spec raises — run_task has already recorded that failure, and
|
|
198
|
+
the executor's bookkeeping must not re-raise it (and crash the run)."""
|
|
199
|
+
try:
|
|
200
|
+
return _norm_paths(getattr(task, part).values())
|
|
201
|
+
except Exception:
|
|
202
|
+
return set()
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def record_failure(failures, task):
|
|
206
|
+
"""Record a failed (or skipped-for-upstream-failure) task in an
|
|
207
|
+
executor's `failures` dict, for upstream_failed."""
|
|
208
|
+
failures.setdefault(task.rule, set()).add(frozenset(task.kwargs.items()))
|
|
209
|
+
failures.setdefault(_FAILED_OUTPUTS, set()).update(_io_paths(task, 'outputs'))
|
|
210
|
+
|
|
211
|
+
|
|
127
212
|
def upstream_failed(task, failures):
|
|
128
213
|
"""Should task be skipped because upstream tasks failed this run?
|
|
129
214
|
|
|
130
|
-
failures:
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
215
|
+
failures: built by record_failure. Mirrors the planner's rerun
|
|
216
|
+
propagation — element-wise when the matrices are shared, conservative
|
|
217
|
+
(any failure taints all downstream tasks) otherwise — and additionally
|
|
218
|
+
skips any task whose declared inputs include an output of a failed task.
|
|
219
|
+
A shared matrix doesn't mean each task reads only its same-kwargs
|
|
220
|
+
upstream (e.g. inputs of year-1): without the path check such a task ran
|
|
221
|
+
on its failed upstream's stale or missing output (review 2026-09-24 H3,
|
|
222
|
+
failure-skip part; the full fix is 0.9).
|
|
134
223
|
"""
|
|
135
224
|
for dep in task.rule.depends_on:
|
|
136
225
|
failed = failures.get(dep)
|
|
137
226
|
if not failed:
|
|
138
227
|
continue
|
|
139
|
-
if _same_matrix(task.rule, dep):
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
228
|
+
if not _same_matrix(task.rule, dep):
|
|
229
|
+
return True
|
|
230
|
+
if frozenset(task.kwargs.items()) in failed:
|
|
231
|
+
return True
|
|
232
|
+
if _io_paths(task, 'inputs') & failures.get(_FAILED_OUTPUTS, set()):
|
|
143
233
|
return True
|
|
144
234
|
return False
|
|
145
235
|
|
|
@@ -323,9 +413,10 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
|
|
|
323
413
|
# MM: this is a core piece of logic, but I find it hard to understand end-to-end.
|
|
324
414
|
# MM: also quite a long func.
|
|
325
415
|
start = perf_counter()
|
|
326
|
-
predicate = make_predicate(query) if query else None
|
|
416
|
+
predicate = make_predicate(query, rule_kwarg_names(rules)) if query else None
|
|
327
417
|
code_comparer = CodeComparer()
|
|
328
418
|
rules = set(rules)
|
|
419
|
+
nmatched = 0
|
|
329
420
|
|
|
330
421
|
runnable = []
|
|
331
422
|
deferred = []
|
|
@@ -469,8 +560,12 @@ def plan(rules, dag, metadata, *, query=None, force=False, check_outputs='never'
|
|
|
469
560
|
(records[task.key].run_seq if task.key in records else None)
|
|
470
561
|
for task in tasks
|
|
471
562
|
}
|
|
563
|
+
nmatched += len(tasks)
|
|
472
564
|
logger.debug('{}: {} task(s), {} to rerun', rule.name, len(tasks), len(rule_rerun))
|
|
473
565
|
|
|
566
|
+
if predicate is not None and not nmatched and not deferred:
|
|
567
|
+
# Distinguish "your query selected nothing" from "all up to date".
|
|
568
|
+
logger.warning(f'query {query!r} matched no tasks')
|
|
474
569
|
elapsed = perf_counter() - start
|
|
475
570
|
logger.bind(
|
|
476
571
|
event='plan', nrunnable=len(runnable), ndeferred=len(deferred),
|
|
@@ -5,6 +5,7 @@ from collections import Counter
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from time import perf_counter
|
|
7
7
|
|
|
8
|
+
import networkx as nx
|
|
8
9
|
from loguru import logger
|
|
9
10
|
|
|
10
11
|
from ..metadata.metadata_manager import (
|
|
@@ -14,9 +15,16 @@ from ..metadata.metadata_manager import (
|
|
|
14
15
|
RecordCache,
|
|
15
16
|
)
|
|
16
17
|
from ..util import task_log_path
|
|
18
|
+
from ..util.run_lock import run_lock
|
|
17
19
|
from .dag import build_rule_dag, expand_rule, iter_expand_rule
|
|
18
|
-
from .exceptions import Defer, RemakeError
|
|
19
|
-
from .planner import
|
|
20
|
+
from .exceptions import Defer, RemakeError, TaskExit
|
|
21
|
+
from .planner import (
|
|
22
|
+
cascade_settled,
|
|
23
|
+
explain_task,
|
|
24
|
+
make_predicate,
|
|
25
|
+
plan,
|
|
26
|
+
rule_kwarg_names,
|
|
27
|
+
)
|
|
20
28
|
from .rule import Rule
|
|
21
29
|
from .scope import check_scope, exec_function
|
|
22
30
|
from .task import Task
|
|
@@ -45,6 +53,39 @@ class _TemplatePlaceholder:
|
|
|
45
53
|
return str(self)
|
|
46
54
|
|
|
47
55
|
|
|
56
|
+
def _blocked_reason(rule):
|
|
57
|
+
"""Why a rule is still deferred when a local run has nothing left to
|
|
58
|
+
try: its matrix's Defer paths if it still can't expand, else (it would
|
|
59
|
+
expand, so the planner is holding it back) an upstream that didn't
|
|
60
|
+
complete this run."""
|
|
61
|
+
from .rule import is_deferrable
|
|
62
|
+
|
|
63
|
+
if is_deferrable(rule.matrix):
|
|
64
|
+
try:
|
|
65
|
+
rule.matrix()
|
|
66
|
+
except Defer as e:
|
|
67
|
+
waiting = ', '.join(e.paths) if e.paths else 'an unspecified input'
|
|
68
|
+
return f'matrix not ready — waiting on {waiting}'
|
|
69
|
+
except Exception as e: # the matrix itself is broken: say so
|
|
70
|
+
return f'matrix raised {type(e).__name__}: {e}'
|
|
71
|
+
return 'an upstream rule did not complete (see failures above)'
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _where(rule):
|
|
75
|
+
fn = rule.fn
|
|
76
|
+
return f'{getattr(fn, "__module__", "?")}.{getattr(fn, "__qualname__", rule.name)}'
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _same_definition(a, b):
|
|
80
|
+
"""Two Rule objects for the same function re-defined (e.g. a notebook
|
|
81
|
+
cell re-run): same module and qualified name, but freshly compiled code.
|
|
82
|
+
Distinct functions that merely share a rule name (two modules each
|
|
83
|
+
defining `process`, a clashing name=) are not — nor are rules built by
|
|
84
|
+
one factory function, which share a single code object and differ only
|
|
85
|
+
in closure values."""
|
|
86
|
+
return _where(a) == _where(b) and a.fn.__code__ is not b.fn.__code__
|
|
87
|
+
|
|
88
|
+
|
|
48
89
|
class Remake:
|
|
49
90
|
def __init__(
|
|
50
91
|
self,
|
|
@@ -58,6 +99,10 @@ class Remake:
|
|
|
58
99
|
self.config = config or {}
|
|
59
100
|
self.metadata = metadata
|
|
60
101
|
self.check_outputs = check_outputs
|
|
102
|
+
# Rules the last run() left unresolved (deferred matrix never ready,
|
|
103
|
+
# or waiting on an upstream that failed). Non-empty means that run
|
|
104
|
+
# did not complete everything; the CLI exits non-zero on it.
|
|
105
|
+
self.blocked_rules = []
|
|
61
106
|
self.strict_scope = strict_scope
|
|
62
107
|
self.rules = []
|
|
63
108
|
self.dag = None
|
|
@@ -74,11 +119,26 @@ class Remake:
|
|
|
74
119
|
raise RemakeError(f'Not a Rule (use the @rule decorator): {rule!r}')
|
|
75
120
|
if rule in self.rules:
|
|
76
121
|
continue
|
|
122
|
+
# Task keys are sha1('<rule name>:<kwargs>'), so two rules sharing
|
|
123
|
+
# a name share task keys and DB records (review 2026-09-24 H8).
|
|
124
|
+
existing = next((r for r in self.rules if r.name == rule.name), None)
|
|
125
|
+
if existing is not None and not _same_definition(existing, rule):
|
|
126
|
+
raise RemakeError(
|
|
127
|
+
f'Duplicate rule name {rule.name!r}: defined by '
|
|
128
|
+
f'{_where(existing)} and {_where(rule)}. Rule names must be '
|
|
129
|
+
f'unique within a pipeline (rename one, or pass name=...)'
|
|
130
|
+
)
|
|
77
131
|
# Resolve tri-state strict_scope against the Remake default.
|
|
78
132
|
if rule.strict_scope is None and self.strict_scope:
|
|
79
133
|
check_scope(rule.fn, rule.uses, strict=True)
|
|
80
134
|
rule.remake = self
|
|
81
|
-
|
|
135
|
+
if existing is not None:
|
|
136
|
+
# The same function re-defined — a notebook cell or script
|
|
137
|
+
# section executed again: the new definition replaces the old.
|
|
138
|
+
logger.warning(f'Rule {rule.name!r} redefined: replacing the earlier definition')
|
|
139
|
+
self.rules[self.rules.index(existing)] = rule
|
|
140
|
+
else:
|
|
141
|
+
self.rules.append(rule)
|
|
82
142
|
self._finalized = False
|
|
83
143
|
|
|
84
144
|
def rules_from_current_module(self):
|
|
@@ -186,7 +246,7 @@ class Remake:
|
|
|
186
246
|
dynamic matrix is deferred."""
|
|
187
247
|
if not self._finalized:
|
|
188
248
|
self.finalize()
|
|
189
|
-
predicate = make_predicate(query) if query else None
|
|
249
|
+
predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
|
|
190
250
|
for rule in self.rules:
|
|
191
251
|
try:
|
|
192
252
|
yield from iter_expand_rule(rule, predicate)
|
|
@@ -285,7 +345,7 @@ class Remake:
|
|
|
285
345
|
remaining = Counter(task.rule.name for task in runnable)
|
|
286
346
|
runnable_keys = {task.key for task in runnable}
|
|
287
347
|
deferred_names = {rule.name for rule in deferred}
|
|
288
|
-
predicate = make_predicate(query) if query else None
|
|
348
|
+
predicate = make_predicate(query, rule_kwarg_names(self.rules)) if query else None
|
|
289
349
|
|
|
290
350
|
# Per-rule tally of why the to-run tasks would rerun. One plan() is
|
|
291
351
|
# already done (`runnable`); reuse it per task so this is plan-cost,
|
|
@@ -669,9 +729,21 @@ class Remake:
|
|
|
669
729
|
"""Run all tasks that need running, replanning after each wave so
|
|
670
730
|
dynamic (deferred) matrices resolve as their upstreams complete.
|
|
671
731
|
Returns the number of failed tasks (0 for asynchronous executors,
|
|
672
|
-
which don't know at submission time).
|
|
732
|
+
which don't know at submission time). Holds `.remake/run.lock` for
|
|
733
|
+
the duration: a second concurrent run in the same directory is a
|
|
734
|
+
RemakeError."""
|
|
673
735
|
if not self._finalized:
|
|
674
736
|
self.finalize()
|
|
737
|
+
self.blocked_rules = []
|
|
738
|
+
if executor is not None and executor.handles_deferred:
|
|
739
|
+
# SLURM: submission only; continuation jobs re-enter run() on
|
|
740
|
+
# compute nodes, and the executor has its own duplicate-submission
|
|
741
|
+
# guard — a directory lock would strand or block them.
|
|
742
|
+
return self._run(executor, query, force, ignore_code_changes)
|
|
743
|
+
with run_lock(self.metadata):
|
|
744
|
+
return self._run(executor, query, force, ignore_code_changes)
|
|
745
|
+
|
|
746
|
+
def _run(self, executor, query, force, ignore_code_changes):
|
|
675
747
|
# One run_seq for this whole invocation (shared across replanning
|
|
676
748
|
# waves); committed onto every task so downstream propagation survives
|
|
677
749
|
# to later invocations. See bugs/01_durable_rerun_propagation.md.
|
|
@@ -705,8 +777,19 @@ class Remake:
|
|
|
705
777
|
runnable = [t for t in runnable if t.key not in attempted]
|
|
706
778
|
if not runnable:
|
|
707
779
|
if deferred:
|
|
708
|
-
|
|
709
|
-
|
|
780
|
+
# Not everything completed: say which rules and why, and
|
|
781
|
+
# make the run's outcome reflect it (review 2026-09-24 M9;
|
|
782
|
+
# it used to be a bare warning and exit 0). A filtered run
|
|
783
|
+
# may deliberately leave out the upstream a deferred rule
|
|
784
|
+
# waits on: that one is reported but doesn't fail the run.
|
|
785
|
+
excused = self._excluded_by_query(deferred) if query else set()
|
|
786
|
+
for rule in deferred:
|
|
787
|
+
if rule in excused:
|
|
788
|
+
logger.warning(f'Blocked rule {rule.name} (upstream not '
|
|
789
|
+
f'selected by the query): {_blocked_reason(rule)}')
|
|
790
|
+
else:
|
|
791
|
+
self.blocked_rules.append(rule)
|
|
792
|
+
logger.error(f'Blocked rule {rule.name}: {_blocked_reason(rule)}')
|
|
710
793
|
break
|
|
711
794
|
wave += 1
|
|
712
795
|
logger.bind(event='wave', wave=wave, ntasks=len(runnable)).debug(
|
|
@@ -724,40 +807,71 @@ class Remake:
|
|
|
724
807
|
logger.info('Nothing to do')
|
|
725
808
|
return nfailed
|
|
726
809
|
|
|
810
|
+
def _excluded_by_query(self, deferred):
|
|
811
|
+
"""Deferred rules a filtered run can be excused for: those with an
|
|
812
|
+
upstream (any ancestor) that still has work outside the filter. If
|
|
813
|
+
every ancestor is complete and the rule is still deferred, the query
|
|
814
|
+
didn't cause it — it's genuinely blocked."""
|
|
815
|
+
runnable_all, deferred_all = self.plan()
|
|
816
|
+
unfinished = {t.rule for t in runnable_all} | set(deferred_all)
|
|
817
|
+
return {rule for rule in deferred
|
|
818
|
+
if nx.ancestors(self.dag, rule) & (unfinished - {rule})}
|
|
819
|
+
|
|
727
820
|
def run_task(self, task):
|
|
728
821
|
"""Execute one task and record the result. The single execution
|
|
729
822
|
entry point — used by all executors and `remake run-task`. Timing and
|
|
730
823
|
completion are logged here so every executor gets them uniformly
|
|
731
824
|
(per-element detail at TRACE, per-task duration at DEBUG — the
|
|
732
825
|
summarise-loops convention, per_task_logging.md)."""
|
|
733
|
-
# opt(lazy=True): the path lists are only built when a TRACE sink is
|
|
734
|
-
# attached (they'd cost real time at 1e6 tasks otherwise).
|
|
735
|
-
logger.opt(lazy=True).trace(
|
|
736
|
-
'running {}: inputs {} -> outputs {}', lambda: task,
|
|
737
|
-
lambda: [str(p) for p in task.inputs.values()],
|
|
738
|
-
lambda: [str(p) for p in task.outputs.values()],
|
|
739
|
-
)
|
|
740
|
-
for token in task.outputs.values():
|
|
741
|
-
if hasattr(token, '__fspath__'):
|
|
742
|
-
Path(token).parent.mkdir(parents=True, exist_ok=True)
|
|
743
|
-
|
|
744
|
-
fn = exec_function(task.rule.fn, task.rule.uses)
|
|
745
|
-
args = []
|
|
746
|
-
if task.rule.inputs is not None:
|
|
747
|
-
args.append(task.inputs)
|
|
748
|
-
if task.rule.outputs is not None:
|
|
749
|
-
args.append(task.outputs)
|
|
750
826
|
start = perf_counter()
|
|
827
|
+
# Everything that can fail — output dirs, io resolution (callable
|
|
828
|
+
# specs run here), the function itself — is inside the try, so every
|
|
829
|
+
# failure is recorded with its traceback (review 2026-09-24 M11).
|
|
751
830
|
try:
|
|
752
|
-
|
|
753
|
-
|
|
831
|
+
# opt(lazy=True): the path lists are only built when a TRACE sink
|
|
832
|
+
# is attached (they'd cost real time at 1e6 tasks otherwise).
|
|
833
|
+
logger.opt(lazy=True).trace(
|
|
834
|
+
'running {}: inputs {} -> outputs {}', lambda: task,
|
|
835
|
+
lambda: [str(p) for p in task.inputs.values()],
|
|
836
|
+
lambda: [str(p) for p in task.outputs.values()],
|
|
837
|
+
)
|
|
838
|
+
for token in task.outputs.values():
|
|
839
|
+
if hasattr(token, '__fspath__'):
|
|
840
|
+
Path(token).parent.mkdir(parents=True, exist_ok=True)
|
|
841
|
+
|
|
842
|
+
fn = exec_function(task.rule.fn, task.rule.uses)
|
|
843
|
+
args = []
|
|
844
|
+
if task.rule.inputs is not None:
|
|
845
|
+
args.append(task.inputs)
|
|
846
|
+
if task.rule.outputs is not None:
|
|
847
|
+
args.append(task.outputs)
|
|
848
|
+
try:
|
|
849
|
+
fn(*args, **task.kwargs)
|
|
850
|
+
except SystemExit as exc:
|
|
851
|
+
# sys.exit(0) / sys.exit() from a wrapped CLI main() means
|
|
852
|
+
# success; only a non-zero code is a failure (handled below).
|
|
853
|
+
# Mirror CPython: only None or an int 0 (incl. False) exits 0;
|
|
854
|
+
# sys.exit(0.0) or sys.exit('msg') exit 1.
|
|
855
|
+
if not (exc.code is None or (isinstance(exc.code, int) and exc.code == 0)):
|
|
856
|
+
raise
|
|
857
|
+
except (Exception, SystemExit) as exc:
|
|
858
|
+
# SystemExit from task code (a CLI main() calling sys.exit) is a
|
|
859
|
+
# task failure, not a request to end the run (review H6).
|
|
860
|
+
# KeyboardInterrupt still propagates: Ctrl-C stops the run.
|
|
754
861
|
elapsed = perf_counter() - start
|
|
862
|
+
tb = traceback.format_exc()
|
|
755
863
|
logger.bind(event='task_failed', task=str(task), rule=task.rule.name,
|
|
756
864
|
key=task.key, seconds=round(elapsed, 6),
|
|
757
865
|
).error(f'failed: {task} after {elapsed:.2f}s')
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
)
|
|
866
|
+
# The traceback at DEBUG: lands in per-task logs (DEBUG sinks)
|
|
867
|
+
# without flooding the INFO console on runs with many failures
|
|
868
|
+
# (review M12).
|
|
869
|
+
logger.debug('traceback for {}:\n{}', task, tb)
|
|
870
|
+
self.metadata.update_task(task, TASK_STATUS_FAILED, exception=tb)
|
|
871
|
+
if isinstance(exc, SystemExit):
|
|
872
|
+
raise TaskExit(
|
|
873
|
+
f'{task} called sys.exit({exc.code!r})'
|
|
874
|
+
) from exc
|
|
761
875
|
raise
|
|
762
876
|
elapsed = perf_counter() - start
|
|
763
877
|
logger.bind(event='task_complete', task=str(task), rule=task.rule.name,
|
|
@@ -216,7 +216,12 @@ def rule(
|
|
|
216
216
|
inputs=inputs,
|
|
217
217
|
outputs=outputs,
|
|
218
218
|
matrix=matrix,
|
|
219
|
-
|
|
219
|
+
# A bare name or Rule is one dependency — list('extract') would
|
|
220
|
+
# split it into characters (review 2026-09-24 L4).
|
|
221
|
+
depends_on=(
|
|
222
|
+
[depends_on] if isinstance(depends_on, (str, Rule))
|
|
223
|
+
else list(depends_on) if depends_on else []
|
|
224
|
+
),
|
|
220
225
|
uses=uses_,
|
|
221
226
|
strict_scope=strict_scope,
|
|
222
227
|
config=dict(config) if config else {},
|
|
@@ -23,13 +23,13 @@ import os
|
|
|
23
23
|
from loguru import logger
|
|
24
24
|
|
|
25
25
|
from ..core.exceptions import RemakeError
|
|
26
|
-
from ..core.planner import upstream_failed
|
|
26
|
+
from ..core.planner import record_failure, upstream_failed
|
|
27
27
|
from .executor import Executor
|
|
28
28
|
|
|
29
29
|
_worker_rmk_cache = {}
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def _run_spec(remakefile, rule_name, kwargs):
|
|
32
|
+
def _run_spec(remakefile, rule_name, kwargs, run_seq=None):
|
|
33
33
|
"""Runs on a dask worker. Returns True on success."""
|
|
34
34
|
from ..loader import load_remake
|
|
35
35
|
from ..metadata.sidecar import SidecarWriter
|
|
@@ -40,6 +40,9 @@ def _run_spec(remakefile, rule_name, kwargs):
|
|
|
40
40
|
rmk = load_remake(remakefile, finalize=False)
|
|
41
41
|
rmk.metadata = SidecarWriter()
|
|
42
42
|
_worker_rmk_cache[remakefile] = rmk
|
|
43
|
+
# The parent invocation's run_seq (durable propagation, bugs/01); set per
|
|
44
|
+
# call because long-lived workers outlive a single invocation.
|
|
45
|
+
rmk.metadata.run_seq = run_seq
|
|
43
46
|
task = rmk.task_from_spec(rule_name, kwargs)
|
|
44
47
|
logfile = task_log_path(task)
|
|
45
48
|
logfile.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -95,14 +98,15 @@ class DaskExecutor(Executor):
|
|
|
95
98
|
nfailed = 0
|
|
96
99
|
nskipped = 0
|
|
97
100
|
done = 0
|
|
98
|
-
failures = {} #
|
|
101
|
+
failures = {} # see planner.record_failure
|
|
102
|
+
run_seq = self.rmk.metadata.current_run_seq()
|
|
99
103
|
client, cluster = self._client()
|
|
100
104
|
try:
|
|
101
105
|
for rule, rule_tasks in groups:
|
|
102
106
|
to_run = []
|
|
103
107
|
for task in rule_tasks:
|
|
104
108
|
if upstream_failed(task, failures):
|
|
105
|
-
failures
|
|
109
|
+
record_failure(failures, task)
|
|
106
110
|
nskipped += 1
|
|
107
111
|
done += 1
|
|
108
112
|
logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
|
|
@@ -113,7 +117,8 @@ class DaskExecutor(Executor):
|
|
|
113
117
|
logger.info(f'{rule.name}: {len(to_run)} task(s) on dask ({self.nproc} workers)')
|
|
114
118
|
futures = {
|
|
115
119
|
client.submit(
|
|
116
|
-
_run_spec, self.remakefile, rule.name, task.kwargs,
|
|
120
|
+
_run_spec, self.remakefile, rule.name, task.kwargs, run_seq,
|
|
121
|
+
pure=False,
|
|
117
122
|
): task
|
|
118
123
|
for task in to_run
|
|
119
124
|
}
|
|
@@ -124,7 +129,7 @@ class DaskExecutor(Executor):
|
|
|
124
129
|
if future.result():
|
|
125
130
|
logger.info(f'{done}/{ntasks}: {task}')
|
|
126
131
|
else:
|
|
127
|
-
failures
|
|
132
|
+
record_failure(failures, task)
|
|
128
133
|
nfailed += 1
|
|
129
134
|
logger.error(f'{done}/{ntasks} failed: {task}')
|
|
130
135
|
finally:
|
|
@@ -22,7 +22,7 @@ from multiprocessing import get_context
|
|
|
22
22
|
from loguru import logger
|
|
23
23
|
|
|
24
24
|
from ..core.exceptions import RemakeError
|
|
25
|
-
from ..core.planner import upstream_failed
|
|
25
|
+
from ..core.planner import record_failure, upstream_failed
|
|
26
26
|
from .executor import Executor
|
|
27
27
|
|
|
28
28
|
_worker_rmk = None
|
|
@@ -53,7 +53,10 @@ def _worker_init(remakefile):
|
|
|
53
53
|
def _worker_run(spec):
|
|
54
54
|
from ..util import task_log_path
|
|
55
55
|
|
|
56
|
-
rule_name, kwargs = spec
|
|
56
|
+
rule_name, kwargs, run_seq = spec
|
|
57
|
+
# Stamp results with the parent invocation's run_seq so durable rerun
|
|
58
|
+
# propagation (bugs/01) works for multiproc runs as it does for SLURM.
|
|
59
|
+
_worker_rmk.metadata.run_seq = run_seq
|
|
57
60
|
task = _worker_rmk.task_from_spec(rule_name, kwargs)
|
|
58
61
|
logfile = task_log_path(task)
|
|
59
62
|
logfile.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -93,7 +96,8 @@ class MultiprocExecutor(Executor):
|
|
|
93
96
|
nfailed = 0
|
|
94
97
|
nskipped = 0
|
|
95
98
|
done = 0
|
|
96
|
-
failures = {} #
|
|
99
|
+
failures = {} # see planner.record_failure
|
|
100
|
+
run_seq = self.rmk.metadata.current_run_seq()
|
|
97
101
|
with ProcessPoolExecutor(
|
|
98
102
|
max_workers=self.nproc,
|
|
99
103
|
mp_context=get_context('spawn'),
|
|
@@ -107,7 +111,7 @@ class MultiprocExecutor(Executor):
|
|
|
107
111
|
to_run = []
|
|
108
112
|
for task in rule_tasks:
|
|
109
113
|
if upstream_failed(task, failures):
|
|
110
|
-
failures
|
|
114
|
+
record_failure(failures, task)
|
|
111
115
|
nskipped += 1
|
|
112
116
|
done += 1
|
|
113
117
|
logger.warning(f'{done}/{ntasks} skipped (upstream failed): {task}')
|
|
@@ -117,7 +121,8 @@ class MultiprocExecutor(Executor):
|
|
|
117
121
|
continue
|
|
118
122
|
logger.info(f'{rule.name}: {len(to_run)} task(s) on {self.nproc} proc(s)')
|
|
119
123
|
futures = {
|
|
120
|
-
pool.submit(_worker_run, (rule.name, t.kwargs)): t
|
|
124
|
+
pool.submit(_worker_run, (rule.name, t.kwargs, run_seq)): t
|
|
125
|
+
for t in to_run
|
|
121
126
|
}
|
|
122
127
|
# Barrier: drain this rule before starting the next.
|
|
123
128
|
for future in as_completed(futures):
|
|
@@ -126,7 +131,7 @@ class MultiprocExecutor(Executor):
|
|
|
126
131
|
if future.result():
|
|
127
132
|
logger.info(f'{done}/{ntasks}: {task}')
|
|
128
133
|
else:
|
|
129
|
-
failures
|
|
134
|
+
record_failure(failures, task)
|
|
130
135
|
nfailed += 1
|
|
131
136
|
logger.error(f'{done}/{ntasks} failed: {task}')
|
|
132
137
|
if nfailed:
|
|
@@ -2,7 +2,7 @@ import math
|
|
|
2
2
|
|
|
3
3
|
from loguru import logger
|
|
4
4
|
|
|
5
|
-
from ..core.planner import upstream_failed
|
|
5
|
+
from ..core.planner import record_failure, upstream_failed
|
|
6
6
|
from .executor import Executor
|
|
7
7
|
|
|
8
8
|
|
|
@@ -12,7 +12,7 @@ class SingleprocExecutor(Executor):
|
|
|
12
12
|
ndigits = math.floor(math.log10(ntasks)) + 1 if ntasks else 1
|
|
13
13
|
nfailed = 0
|
|
14
14
|
nskipped = 0
|
|
15
|
-
failures = {} #
|
|
15
|
+
failures = {} # see planner.record_failure
|
|
16
16
|
for i, task in enumerate(tasks):
|
|
17
17
|
prefix = f'{i + 1:>{ndigits}}/{ntasks}'
|
|
18
18
|
if upstream_failed(task, failures):
|
|
@@ -20,7 +20,7 @@ class SingleprocExecutor(Executor):
|
|
|
20
20
|
# fail noisily on missing inputs. Left unrecorded (pending):
|
|
21
21
|
# fixing the upstream makes the next run pick them up.
|
|
22
22
|
# Counts as a failure for downstream propagation.
|
|
23
|
-
failures
|
|
23
|
+
record_failure(failures, task)
|
|
24
24
|
nskipped += 1
|
|
25
25
|
logger.warning(f'{prefix} skipped (upstream failed): {task}')
|
|
26
26
|
continue
|
|
@@ -32,7 +32,7 @@ class SingleprocExecutor(Executor):
|
|
|
32
32
|
raise # remake run -X: let the debugger see it
|
|
33
33
|
# Failure is recorded by run_task; carry on so independent
|
|
34
34
|
# tasks still run.
|
|
35
|
-
failures
|
|
35
|
+
record_failure(failures, task)
|
|
36
36
|
nfailed += 1
|
|
37
37
|
if nfailed:
|
|
38
38
|
skipped = f' ({nskipped} downstream task(s) skipped)' if nskipped else ''
|
|
@@ -42,8 +42,14 @@ def load_remake(filename, finalize=True):
|
|
|
42
42
|
from ..core.remake import Remake
|
|
43
43
|
|
|
44
44
|
filename = Path(filename)
|
|
45
|
+
if not filename.name or filename.is_dir():
|
|
46
|
+
raise RemakeLoadError(f'Expected a remakefile (.py), got a directory: {filename}')
|
|
45
47
|
if not filename.suffix:
|
|
46
48
|
filename = filename.with_suffix('.py')
|
|
49
|
+
if filename.suffix != '.py':
|
|
50
|
+
raise RemakeLoadError(f'Not a Python remakefile (expected a .py file): {filename}')
|
|
51
|
+
if not (Path.cwd() / filename).is_file():
|
|
52
|
+
raise RemakeLoadError(f'Remakefile not found: {Path.cwd() / filename}')
|
|
47
53
|
remake_module = load_module(filename)
|
|
48
54
|
remakes = [o for o in vars(remake_module).values() if isinstance(o, Remake)]
|
|
49
55
|
if len(remakes) > 1:
|
|
@@ -12,10 +12,11 @@ import json
|
|
|
12
12
|
import random
|
|
13
13
|
import sqlite3
|
|
14
14
|
from pathlib import Path
|
|
15
|
-
from time import perf_counter, sleep
|
|
15
|
+
from time import monotonic, perf_counter, sleep
|
|
16
16
|
|
|
17
17
|
from loguru import logger
|
|
18
18
|
|
|
19
|
+
from ..core.exceptions import RemakeError
|
|
19
20
|
from ..core.scope import io_hash as compute_io_hash
|
|
20
21
|
from ..core.scope import raw_uses_parts
|
|
21
22
|
from ..core.scope import uses_hash as compute_uses_hash
|
|
@@ -102,23 +103,92 @@ INSERT INTO meta(key, value) VALUES ('run_seq', 0);
|
|
|
102
103
|
"""
|
|
103
104
|
|
|
104
105
|
|
|
106
|
+
def _schema_statements():
|
|
107
|
+
"""SQL_SCHEMA as single, idempotent statements, run one by one inside an
|
|
108
|
+
explicit transaction (executescript would COMMIT it first).
|
|
109
|
+
|
|
110
|
+
Idempotent (IF NOT EXISTS / OR IGNORE) so they can run on every open: a
|
|
111
|
+
DB left partially created by an older remake — whose executescript
|
|
112
|
+
committed statement by statement — gets its missing tables and index
|
|
113
|
+
instead of failing on the ones it has. sqlite3.complete_statement is
|
|
114
|
+
tokenizer-aware, so a ';' inside a comment doesn't split a statement."""
|
|
115
|
+
stmts, buf = [], ''
|
|
116
|
+
for line in SQL_SCHEMA.splitlines(keepends=True):
|
|
117
|
+
buf += line
|
|
118
|
+
if sqlite3.complete_statement(buf):
|
|
119
|
+
stmts.append(_idempotent(buf))
|
|
120
|
+
buf = ''
|
|
121
|
+
assert not buf.strip(), 'SQL_SCHEMA ends with an incomplete statement'
|
|
122
|
+
return stmts
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _idempotent(stmt):
|
|
126
|
+
for old, new in (('CREATE TABLE ', 'CREATE TABLE IF NOT EXISTS '),
|
|
127
|
+
('CREATE UNIQUE INDEX ', 'CREATE UNIQUE INDEX IF NOT EXISTS '),
|
|
128
|
+
('CREATE INDEX ', 'CREATE INDEX IF NOT EXISTS '),
|
|
129
|
+
('INSERT INTO ', 'INSERT OR IGNORE INTO ')):
|
|
130
|
+
stmt = stmt.replace(old, new)
|
|
131
|
+
return stmt
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# How long a writer keeps retrying a locked DB before giving up. Long enough
|
|
135
|
+
# to sit out another process's one-time migration or a big sidecar ingest;
|
|
136
|
+
# finite, so a DB that stays locked becomes an error rather than a hang.
|
|
137
|
+
LOCK_RETRY_SECONDS = 600
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _quarantine(path, why):
|
|
141
|
+
"""Move a bad sidecar aside (`*.json.bad`, outside the ingest glob) with
|
|
142
|
+
one warning, instead of failing or re-warning on every command."""
|
|
143
|
+
bad = path.with_name(path.name + '.bad')
|
|
144
|
+
try:
|
|
145
|
+
path.replace(bad)
|
|
146
|
+
except FileNotFoundError:
|
|
147
|
+
return # another process got there first
|
|
148
|
+
logger.warning(f'Quarantined {why} sidecar result: {bad}')
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _is_lock_error(exc):
|
|
152
|
+
msg = str(exc).lower()
|
|
153
|
+
return 'locked' in msg or 'busy' in msg
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _retry_locked(fn, what):
|
|
157
|
+
"""Call fn(), retrying with capped exponential backoff while SQLite
|
|
158
|
+
reports lock contention. Any other OperationalError ("no such table",
|
|
159
|
+
disk full, malformed schema) is re-raised at once instead of being
|
|
160
|
+
retried forever (todos: "Bound and message-match retry_lock_commit")."""
|
|
161
|
+
deadline = monotonic() + LOCK_RETRY_SECONDS
|
|
162
|
+
nattempts = 1
|
|
163
|
+
while True:
|
|
164
|
+
try:
|
|
165
|
+
return fn()
|
|
166
|
+
except sqlite3.OperationalError as oe:
|
|
167
|
+
if not _is_lock_error(oe):
|
|
168
|
+
raise
|
|
169
|
+
if monotonic() >= deadline:
|
|
170
|
+
raise RemakeError(
|
|
171
|
+
f'{what}: database still locked after {LOCK_RETRY_SECONDS}s '
|
|
172
|
+
f'— is another remake process stuck? ({oe})'
|
|
173
|
+
) from oe
|
|
174
|
+
logger.debug(f'{what}: {oe} (attempt {nattempts}); retrying')
|
|
175
|
+
nattempts += 1
|
|
176
|
+
sleep(min(30, 2**nattempts * random.random()))
|
|
177
|
+
|
|
178
|
+
|
|
105
179
|
def retry_lock_commit(fn):
|
|
106
|
-
"""Run fn in an EXCLUSIVE transaction, retrying with
|
|
107
|
-
|
|
180
|
+
"""Run fn in an EXCLUSIVE transaction, retrying with backoff on lock
|
|
181
|
+
contention (concurrent workers/SLURM jobs share the DB)."""
|
|
108
182
|
|
|
109
183
|
def inner(self, *args, **kwargs):
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
except sqlite3.OperationalError as oe:
|
|
119
|
-
logger.debug(f'OperationalError: {oe}')
|
|
120
|
-
nattempts += 1
|
|
121
|
-
sleep(2**nattempts * random.random())
|
|
184
|
+
def attempt():
|
|
185
|
+
with self.conn:
|
|
186
|
+
self.conn.execute('BEGIN EXCLUSIVE')
|
|
187
|
+
ret = fn(self, *args, **kwargs)
|
|
188
|
+
self.conn.commit()
|
|
189
|
+
return ret
|
|
190
|
+
|
|
191
|
+
return _retry_locked(attempt, fn.__name__)
|
|
122
192
|
|
|
123
193
|
return inner
|
|
124
194
|
|
|
@@ -127,18 +197,25 @@ class Sqlite3Backend(MetadataManager):
|
|
|
127
197
|
def __init__(self, dbloc='.remake/remake.db'):
|
|
128
198
|
self.dbloc = str(dbloc)
|
|
129
199
|
self.code_comparer = CodeComparer()
|
|
130
|
-
|
|
131
|
-
create_db = in_memory or not Path(self.dbloc).exists()
|
|
132
|
-
if create_db and not in_memory:
|
|
133
|
-
logger.info(f'Creating sqlite3 database: {self.dbloc}')
|
|
200
|
+
if self.dbloc != ':memory:':
|
|
134
201
|
Path(self.dbloc).parent.mkdir(parents=True, exist_ok=True)
|
|
135
202
|
# No detect_types: timestamps are read as plain strings (TaskRecord
|
|
136
203
|
# .timestamp), and the implicit converter is deprecated in 3.12.
|
|
137
204
|
self.conn = sqlite3.connect(self.dbloc)
|
|
138
|
-
|
|
139
|
-
self.
|
|
140
|
-
|
|
141
|
-
self.
|
|
205
|
+
try:
|
|
206
|
+
self._init_schema()
|
|
207
|
+
except sqlite3.DatabaseError as e:
|
|
208
|
+
self.conn.close()
|
|
209
|
+
if isinstance(e, sqlite3.OperationalError):
|
|
210
|
+
raise
|
|
211
|
+
# e.g. "file is not a database": a corrupt or foreign file.
|
|
212
|
+
raise RemakeError(
|
|
213
|
+
f'{self.dbloc} is not a usable remake database ({e}); move it '
|
|
214
|
+
f'aside to start a fresh one'
|
|
215
|
+
) from None
|
|
216
|
+
except BaseException:
|
|
217
|
+
self.conn.close()
|
|
218
|
+
raise
|
|
142
219
|
self.conn.isolation_level = 'EXCLUSIVE'
|
|
143
220
|
# rule name -> (rule_id, run_code_id, uses_code_id, io_code_id):
|
|
144
221
|
# the rule's row plus this invocation's current interned code ids.
|
|
@@ -169,18 +246,65 @@ class Sqlite3Backend(MetadataManager):
|
|
|
169
246
|
except Exception:
|
|
170
247
|
pass
|
|
171
248
|
|
|
249
|
+
def _init_schema(self):
|
|
250
|
+
"""Create the schema, or migrate an existing DB, in ONE exclusive
|
|
251
|
+
transaction (review 2026-09-24 M5/M6).
|
|
252
|
+
|
|
253
|
+
Whether to create is decided by the presence of the `task` table,
|
|
254
|
+
checked inside the lock — not by the file existing: sqlite3.connect
|
|
255
|
+
creates the file before any schema is written, so a first run killed
|
|
256
|
+
mid-create (disk/quota full, Ctrl-C) used to leave a 0-byte file that
|
|
257
|
+
bricked every later command. Holding the lock across check + DDL also
|
|
258
|
+
serialises concurrent first openers (continuation jobs, `info` next
|
|
259
|
+
to `run`), which used to race to ALTER the same columns; and a crash
|
|
260
|
+
mid-migration now rolls back cleanly and is redone on the next open,
|
|
261
|
+
instead of leaving a half-migrated DB that is skipped forever."""
|
|
262
|
+
conn = self.conn
|
|
263
|
+
conn.isolation_level = None # explicit transaction control below
|
|
264
|
+
_retry_locked(lambda: conn.execute('BEGIN EXCLUSIVE'), 'opening the database')
|
|
265
|
+
try:
|
|
266
|
+
tables = {row[0] for row in conn.execute(
|
|
267
|
+
"SELECT name FROM sqlite_master WHERE type='table'")}
|
|
268
|
+
if 'task' not in tables and self.dbloc != ':memory:':
|
|
269
|
+
logger.info(f'Creating sqlite3 database: {self.dbloc}')
|
|
270
|
+
# Old-shape tables that exist are left alone here (IF NOT EXISTS)
|
|
271
|
+
# and brought up to date column by column below.
|
|
272
|
+
for stmt in _schema_statements():
|
|
273
|
+
conn.execute(stmt)
|
|
274
|
+
needs_vacuum = self._add_missing_columns()
|
|
275
|
+
conn.execute('COMMIT')
|
|
276
|
+
except BaseException:
|
|
277
|
+
# SQLite may already have rolled back by itself (disk full, I/O
|
|
278
|
+
# error); an unconditional ROLLBACK would then raise "no
|
|
279
|
+
# transaction is active" and hide the real error.
|
|
280
|
+
if conn.in_transaction:
|
|
281
|
+
conn.execute('ROLLBACK')
|
|
282
|
+
raise
|
|
283
|
+
if needs_vacuum:
|
|
284
|
+
# VACUUM can't run inside a transaction, so it follows the
|
|
285
|
+
# (committed) migration. Best effort: if another process holds
|
|
286
|
+
# the DB it is skipped — the space is reclaimed on a later VACUUM.
|
|
287
|
+
logger.info('Vacuuming (one-time; reclaims the inline-hash space)')
|
|
288
|
+
try:
|
|
289
|
+
conn.execute('VACUUM')
|
|
290
|
+
except sqlite3.OperationalError as e:
|
|
291
|
+
logger.warning(f'VACUUM skipped ({e}); the DB is correct, just larger')
|
|
292
|
+
|
|
172
293
|
def _add_missing_columns(self):
|
|
173
294
|
"""Lightweight forward-compat for columns added after a DB was first
|
|
174
295
|
created (still no general migration support — see the module docstring).
|
|
175
296
|
A pre-existing record left with io_code_id/run_seq NULL is treated as
|
|
176
297
|
'not yet tracked' by the planner, so upgrading does not force a mass
|
|
177
|
-
rerun.
|
|
298
|
+
rerun. Runs inside _init_schema's exclusive transaction: no commits
|
|
299
|
+
here. Returns True when a VACUUM should follow."""
|
|
300
|
+
needs_vacuum = False
|
|
178
301
|
cols = {row[1] for row in self.conn.execute('PRAGMA table_info(task)')}
|
|
179
302
|
if 'run_seq' not in cols:
|
|
180
303
|
logger.info('Adding task.run_seq column to existing DB')
|
|
181
304
|
self.conn.execute('ALTER TABLE task ADD COLUMN run_seq INTEGER')
|
|
182
305
|
if 'uses_code_id' not in cols:
|
|
183
306
|
self._migrate_inline_hashes_to_code_ids(cols)
|
|
307
|
+
needs_vacuum = True
|
|
184
308
|
rule_cols = {row[1] for row in self.conn.execute('PRAGMA table_info(rule)')}
|
|
185
309
|
if 'remakefile' not in rule_cols:
|
|
186
310
|
logger.info('Adding rule.remakefile column to existing DB')
|
|
@@ -201,14 +325,15 @@ class Sqlite3Backend(MetadataManager):
|
|
|
201
325
|
self.conn.execute(
|
|
202
326
|
'CREATE TABLE meta (key TEXT NOT NULL PRIMARY KEY, value INTEGER NOT NULL)')
|
|
203
327
|
self.conn.execute("INSERT INTO meta(key, value) VALUES ('run_seq', 0)")
|
|
204
|
-
|
|
328
|
+
return needs_vacuum
|
|
205
329
|
|
|
206
330
|
def _migrate_inline_hashes_to_code_ids(self, cols):
|
|
207
331
|
"""One-time in-place migration: the old task.uses_hash/io_hash columns
|
|
208
332
|
stored the full normalised uses/io strings inline per row — duplicated
|
|
209
333
|
across every task of a rule (measured at 99.8% of a 272 MB field DB).
|
|
210
334
|
Intern each distinct value into `code` once, point integer FKs at it,
|
|
211
|
-
drop the text columns
|
|
335
|
+
drop the text columns; the caller VACUUMs afterwards to return the
|
|
336
|
+
space."""
|
|
212
337
|
logger.info('Migrating task.uses_hash/io_hash to code-table FKs')
|
|
213
338
|
self.conn.execute('ALTER TABLE task ADD COLUMN uses_code_id INTEGER')
|
|
214
339
|
self.conn.execute('ALTER TABLE task ADD COLUMN io_code_id INTEGER')
|
|
@@ -242,9 +367,6 @@ class Sqlite3Backend(MetadataManager):
|
|
|
242
367
|
# frees the space (after the VACUUM below) and nothing reads
|
|
243
368
|
# it any more.
|
|
244
369
|
self.conn.execute(f'UPDATE task SET {col} = NULL')
|
|
245
|
-
self.conn.commit()
|
|
246
|
-
logger.info('Vacuuming (one-time; reclaims the inline-hash space)')
|
|
247
|
-
self.conn.execute('VACUUM')
|
|
248
370
|
|
|
249
371
|
@retry_lock_commit
|
|
250
372
|
def _allocate_run_seq(self):
|
|
@@ -471,8 +593,15 @@ class Sqlite3Backend(MetadataManager):
|
|
|
471
593
|
payload = json.loads(path.read_text())
|
|
472
594
|
except FileNotFoundError:
|
|
473
595
|
continue # another process ingested it first
|
|
474
|
-
except json.JSONDecodeError:
|
|
475
|
-
|
|
596
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
597
|
+
_quarantine(path, 'unreadable')
|
|
598
|
+
continue
|
|
599
|
+
if not (isinstance(payload, dict)
|
|
600
|
+
and isinstance(payload.get('status'), int)):
|
|
601
|
+
# Valid JSON of the wrong shape (hand-edited, another
|
|
602
|
+
# remake version) used to crash every plan/info/run
|
|
603
|
+
# inside the ingest transaction (review 2026-09-24 L14).
|
|
604
|
+
_quarantine(path, 'malformed (no integer "status")')
|
|
476
605
|
continue
|
|
477
606
|
logger.trace('sidecar {} ({}): {}', key, rule.name, payload.get('status'))
|
|
478
607
|
pending.append((rule, key, payload, path))
|
|
@@ -517,7 +646,16 @@ class Sqlite3Backend(MetadataManager):
|
|
|
517
646
|
' run_seq = excluded.run_seq, '
|
|
518
647
|
' last_run_timestamp = excluded.last_run_timestamp, '
|
|
519
648
|
' last_run_status = excluded.last_run_status, '
|
|
520
|
-
' exception = excluded.exception'
|
|
649
|
+
' exception = excluded.exception '
|
|
650
|
+
# Never let an older result overwrite a newer one (review
|
|
651
|
+
# 2026-09-24 M17): a sidecar left un-ingested (SLURM element,
|
|
652
|
+
# dead multiproc parent) must not revert a later write such as
|
|
653
|
+
# `remake run-task`. Ordered by run_seq — allocated centrally
|
|
654
|
+
# per invocation — not wall clocks, which skew between compute
|
|
655
|
+
# and login nodes. Without a run_seq on either side (legacy
|
|
656
|
+
# records/sidecars) the sidecar applies, as before.
|
|
657
|
+
'WHERE task.run_seq IS NULL OR excluded.run_seq IS NULL '
|
|
658
|
+
' OR excluded.run_seq >= task.run_seq',
|
|
521
659
|
(
|
|
522
660
|
key,
|
|
523
661
|
rule_id,
|
|
@@ -107,7 +107,10 @@ def _make_executor(name, rmk, nproc=None):
|
|
|
107
107
|
f"{sorted([*builtin, 'multiproc', 'dask'])} or a "
|
|
108
108
|
f'dotted path like mymodule:MyExecutor'
|
|
109
109
|
)
|
|
110
|
-
|
|
110
|
+
try:
|
|
111
|
+
cls = getattr(importlib.import_module(module_name), cls_name)
|
|
112
|
+
except (ImportError, AttributeError) as e:
|
|
113
|
+
raise RemakeError(f'Cannot load executor {name!r}: {e}') from None
|
|
111
114
|
if not (isinstance(cls, type) and issubclass(cls, Executor)):
|
|
112
115
|
raise RemakeError(f'{name!r} is not an Executor subclass')
|
|
113
116
|
return cls(rmk)
|
|
@@ -404,7 +407,9 @@ class RemakeCLI:
|
|
|
404
407
|
force=args.force,
|
|
405
408
|
ignore_code_changes=args.ignore_code_changes,
|
|
406
409
|
)
|
|
407
|
-
|
|
410
|
+
# Exit 0 only when everything needed completed: failed tasks and
|
|
411
|
+
# rules left blocked (deferred matrix never ready) are both non-zero.
|
|
412
|
+
return 1 if nfailed or rmk.blocked_rules else 0
|
|
408
413
|
|
|
409
414
|
def remake_run_task(self, args):
|
|
410
415
|
rmk = self._load(args)
|
|
@@ -433,6 +438,11 @@ class RemakeCLI:
|
|
|
433
438
|
f'remake run {args.remakefile} --executor slurm'
|
|
434
439
|
)
|
|
435
440
|
specs = json.loads(specs_path.read_text())
|
|
441
|
+
if not 0 <= args.index < len(specs):
|
|
442
|
+
raise RemakeError(
|
|
443
|
+
f'array index {args.index} out of range: {specs_path} has '
|
|
444
|
+
f'{len(specs)} task(s)'
|
|
445
|
+
)
|
|
436
446
|
spec = specs[args.index]
|
|
437
447
|
# run_seq was fixed at submission; carry it into the sidecar so its
|
|
438
448
|
# stamp matches the rest of this submission's tasks (older job specs
|
|
@@ -601,11 +611,12 @@ class RemakeCLI:
|
|
|
601
611
|
def remake_ls_tasks(self, args):
|
|
602
612
|
from .core.dag import iter_expand_rule
|
|
603
613
|
from .core.exceptions import Defer
|
|
604
|
-
from .core.planner import make_predicate
|
|
614
|
+
from .core.planner import make_predicate, rule_kwarg_names
|
|
605
615
|
|
|
606
616
|
rmk = self._load(args)
|
|
607
617
|
rmk.finalize()
|
|
608
|
-
predicate = make_predicate(args.query
|
|
618
|
+
predicate = (make_predicate(args.query, rule_kwarg_names(rmk.rules))
|
|
619
|
+
if args.query else None)
|
|
609
620
|
paint = Painter(args.colour)
|
|
610
621
|
|
|
611
622
|
def input_files(task):
|
|
@@ -899,17 +910,21 @@ def remake_cmd(argv=None):
|
|
|
899
910
|
# Logs go to stderr; stdout carries command output only (so --json and
|
|
900
911
|
# piping stay clean).
|
|
901
912
|
logger.remove()
|
|
913
|
+
# Honour --colour / NO_COLOR / a non-TTY stderr like the rest of the
|
|
914
|
+
# output (review 2026-09-24 L25: hardcoded colorize=True filled CI logs
|
|
915
|
+
# and SLURM .err files with escape codes).
|
|
916
|
+
colorize = Painter(getattr(args, 'colour', 'auto'), sys.stderr).enabled
|
|
902
917
|
if args.trace:
|
|
903
|
-
logger.add(sys.stderr, colorize=
|
|
918
|
+
logger.add(sys.stderr, colorize=colorize, level='TRACE')
|
|
904
919
|
elif args.debug:
|
|
905
|
-
logger.add(sys.stderr, colorize=
|
|
920
|
+
logger.add(sys.stderr, colorize=colorize, level='DEBUG')
|
|
906
921
|
elif args.warning:
|
|
907
922
|
logger.add(
|
|
908
|
-
sys.stderr, colorize=
|
|
923
|
+
sys.stderr, colorize=colorize, format='<lvl>{message}</lvl>', level='WARNING'
|
|
909
924
|
)
|
|
910
925
|
else:
|
|
911
926
|
logger.add(
|
|
912
|
-
sys.stderr, colorize=
|
|
927
|
+
sys.stderr, colorize=colorize, format='<lvl>{message}</lvl>', level='INFO'
|
|
913
928
|
)
|
|
914
929
|
|
|
915
930
|
# Anchor execution to the remakefile's directory: cd there so .remake/ and
|
|
@@ -927,7 +942,7 @@ def remake_cmd(argv=None):
|
|
|
927
942
|
except OSError as e:
|
|
928
943
|
print(f'error: cannot enter remakefile directory {rf.parent}: {e}',
|
|
929
944
|
file=sys.stderr)
|
|
930
|
-
return
|
|
945
|
+
return 2
|
|
931
946
|
args.remakefile = rf.name
|
|
932
947
|
|
|
933
948
|
# Per-task-process subcommands (SLURM array elements) get a per-task log
|
|
@@ -941,7 +956,12 @@ def remake_cmd(argv=None):
|
|
|
941
956
|
# structured sink), so a miner can group an invocation's lines and
|
|
942
957
|
# correlate e.g. a plan total with its constituent status queries.
|
|
943
958
|
logger.configure(extra={'run_id': uuid.uuid4().hex[:12]})
|
|
944
|
-
|
|
959
|
+
try:
|
|
960
|
+
Path('.remake').mkdir(parents=True, exist_ok=True)
|
|
961
|
+
except OSError as e:
|
|
962
|
+
print(f'error: cannot create .remake/ metadata dir: {e}', file=sys.stderr)
|
|
963
|
+
os.chdir(orig_cwd)
|
|
964
|
+
return 2
|
|
945
965
|
debug_level = 'TRACE' if args.trace else 'DEBUG'
|
|
946
966
|
# Three always-on file sinks next to the metadata DB, split so the
|
|
947
967
|
# streams don't compete for one rotation window (logs_analysis §3.2):
|
|
@@ -970,6 +990,14 @@ def remake_cmd(argv=None):
|
|
|
970
990
|
|
|
971
991
|
try:
|
|
972
992
|
return cli.dispatch()
|
|
993
|
+
except BrokenPipeError:
|
|
994
|
+
# Output piped into something that stopped reading (`| head`): stop
|
|
995
|
+
# quietly like other CLI tools, not with a traceback (review L26).
|
|
996
|
+
# Point stdout at devnull so the interpreter's final flush can't
|
|
997
|
+
# raise again.
|
|
998
|
+
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
999
|
+
os.dup2(devnull, sys.stdout.fileno())
|
|
1000
|
+
return 141 # 128 + SIGPIPE, what a shell reports for a killed writer
|
|
973
1001
|
except RemakeError as e:
|
|
974
1002
|
# User-facing errors (bad query, >1-task match, unknown rule, ...)
|
|
975
1003
|
# print cleanly and exit 2; keep the traceback only under -X.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""A per-directory lock so two local `remake run`s can't execute the same
|
|
2
|
+
tasks at once (review 2026-09-24 M15: both would run every task and race on
|
|
3
|
+
the same output files).
|
|
4
|
+
|
|
5
|
+
The lock file sits next to the metadata DB (`.remake/run.lock`) and records
|
|
6
|
+
the holder's host and PID. It is created atomically (O_EXCL) and removed when
|
|
7
|
+
the run ends, including on Ctrl-C. A lock left by a process that died on this
|
|
8
|
+
host is detected (its PID is gone) and taken over; one from another host (a
|
|
9
|
+
shared filesystem) can't be checked, so the error says how to clear it.
|
|
10
|
+
"""
|
|
11
|
+
import contextlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import socket
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from loguru import logger
|
|
18
|
+
|
|
19
|
+
from ..core.exceptions import RemakeError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _pid_alive(pid):
|
|
23
|
+
try:
|
|
24
|
+
os.kill(pid, 0)
|
|
25
|
+
except ProcessLookupError:
|
|
26
|
+
return False
|
|
27
|
+
except PermissionError:
|
|
28
|
+
return True # exists, owned by someone else
|
|
29
|
+
return True
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _read_holder(path):
|
|
33
|
+
try:
|
|
34
|
+
return json.loads(path.read_text())
|
|
35
|
+
except (OSError, ValueError):
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@contextlib.contextmanager
|
|
40
|
+
def run_lock(metadata):
|
|
41
|
+
dbloc = getattr(metadata, 'dbloc', None)
|
|
42
|
+
if not dbloc or dbloc == ':memory:':
|
|
43
|
+
yield # nothing on disk to protect (tests, custom backends)
|
|
44
|
+
return
|
|
45
|
+
path = Path(dbloc).parent / 'run.lock'
|
|
46
|
+
me = {'host': socket.gethostname(), 'pid': os.getpid()}
|
|
47
|
+
for attempt in (1, 2):
|
|
48
|
+
try:
|
|
49
|
+
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
50
|
+
except FileExistsError:
|
|
51
|
+
holder = _read_holder(path) or {}
|
|
52
|
+
same_host = holder.get('host') == me['host']
|
|
53
|
+
pid = holder.get('pid')
|
|
54
|
+
if attempt == 1 and same_host and isinstance(pid, int) and not _pid_alive(pid):
|
|
55
|
+
# Re-check right before removing: if another run has just
|
|
56
|
+
# replaced the stale lock with its own, leave that one alone
|
|
57
|
+
# (narrows the takeover race to a re-read-then-unlink window).
|
|
58
|
+
if _read_holder(path) == holder:
|
|
59
|
+
logger.warning(f'Removing stale run lock left by dead process {pid}: {path}')
|
|
60
|
+
path.unlink(missing_ok=True)
|
|
61
|
+
continue
|
|
62
|
+
who = f'{holder.get("host", "?")} pid {holder.get("pid", "?")}'
|
|
63
|
+
raise RemakeError(
|
|
64
|
+
f'another remake run is active in this directory ({who}; lock '
|
|
65
|
+
f'{path}). Wait for it to finish; if it is not running, delete '
|
|
66
|
+
f'{path}'
|
|
67
|
+
) from None
|
|
68
|
+
with os.fdopen(fd, 'w') as f:
|
|
69
|
+
json.dump(me, f)
|
|
70
|
+
break
|
|
71
|
+
try:
|
|
72
|
+
yield
|
|
73
|
+
finally:
|
|
74
|
+
# Only remove our own lock: if it was deleted by hand and another
|
|
75
|
+
# run took the directory, that run's lock must survive our exit.
|
|
76
|
+
if _read_holder(path) == me:
|
|
77
|
+
path.unlink(missing_ok=True)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|