remake 0.8.4__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {remake-0.8.4 → remake-0.9.0}/PKG-INFO +2 -1
- {remake-0.8.4 → remake-0.9.0}/pyproject.toml +4 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/__init__.py +2 -0
- remake-0.9.0/src/remake/__main__.py +8 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/core/__init__.py +1 -1
- remake-0.9.0/src/remake/core/canonical.py +528 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/core/dag.py +48 -3
- remake-0.9.0/src/remake/core/deps.py +201 -0
- remake-0.9.0/src/remake/core/dot.py +84 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/core/exceptions.py +4 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/core/planner.py +210 -92
- {remake-0.8.4 → remake-0.9.0}/src/remake/core/remake.py +485 -20
- {remake-0.8.4 → remake-0.9.0}/src/remake/core/rule.py +8 -4
- remake-0.9.0/src/remake/core/scope.py +556 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/core/task.py +30 -2
- remake-0.9.0/src/remake/core/tokens.py +387 -0
- remake-0.9.0/src/remake/envcheck.py +193 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/executors/dask_executor.py +80 -21
- remake-0.9.0/src/remake/executors/multiproc_executor.py +270 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/executors/singleproc_executor.py +3 -1
- {remake-0.8.4 → remake-0.9.0}/src/remake/executors/slurm_executor.py +79 -20
- {remake-0.8.4 → remake-0.9.0}/src/remake/loader/__init__.py +18 -2
- {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/metadata_manager.py +59 -4
- {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/sidecar.py +15 -5
- {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/sqlite3_backend.py +298 -17
- remake-0.9.0/src/remake/profiles/jasmin.toml +18 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/remake_cmd.py +428 -13
- {remake-0.8.4 → remake-0.9.0}/src/remake/util/__init__.py +0 -1
- {remake-0.8.4 → remake-0.9.0}/src/remake/util/code_compare.py +12 -2
- remake-0.9.0/src/remake/util/config.py +230 -0
- remake-0.9.0/src/remake/util/resources.py +325 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/util/util.py +1 -1
- {remake-0.8.4 → remake-0.9.0}/src/remake/version.py +1 -1
- {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/PKG-INFO +2 -1
- {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/SOURCES.txt +7 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/requires.txt +3 -0
- remake-0.8.4/src/remake/core/scope.py +0 -313
- remake-0.8.4/src/remake/core/tokens.py +0 -91
- remake-0.8.4/src/remake/executors/multiproc_executor.py +0 -143
- remake-0.8.4/src/remake/util/config.py +0 -36
- {remake-0.8.4 → remake-0.9.0}/LICENCE +0 -0
- {remake-0.8.4 → remake-0.9.0}/README.md +0 -0
- {remake-0.8.4 → remake-0.9.0}/setup.cfg +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/executors/__init__.py +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/executors/executor.py +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/__init__.py +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/util/colour.py +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/util/command_line_args.py +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake/util/run_lock.py +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/dependency_links.txt +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/entry_points.txt +0 -0
- {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: remake
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Smart remake tool
|
|
5
5
|
Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
|
|
6
6
|
Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
|
|
@@ -227,6 +227,7 @@ Description-Content-Type: text/markdown
|
|
|
227
227
|
License-File: LICENCE
|
|
228
228
|
Requires-Dist: loguru
|
|
229
229
|
Requires-Dist: networkx
|
|
230
|
+
Requires-Dist: tomli; python_version < "3.11"
|
|
230
231
|
Provides-Extra: debug
|
|
231
232
|
Requires-Dist: ipdb; extra == "debug"
|
|
232
233
|
Provides-Extra: s3
|
|
@@ -17,6 +17,7 @@ requires-python = ">=3.10"
|
|
|
17
17
|
dependencies = [
|
|
18
18
|
"loguru",
|
|
19
19
|
"networkx",
|
|
20
|
+
"tomli; python_version < '3.11'",
|
|
20
21
|
]
|
|
21
22
|
dynamic = ["version"]
|
|
22
23
|
classifiers = [
|
|
@@ -85,6 +86,9 @@ remake = "remake.remake_cmd:remake_cmd"
|
|
|
85
86
|
[tool.setuptools.packages.find]
|
|
86
87
|
where = ["src"]
|
|
87
88
|
|
|
89
|
+
[tool.setuptools.package-data]
|
|
90
|
+
remake = ["profiles/*.toml"]
|
|
91
|
+
|
|
88
92
|
[tool.setuptools.dynamic]
|
|
89
93
|
version = { attr = "remake.version.__version__" }
|
|
90
94
|
|
|
@@ -11,4 +11,4 @@ from .remake import Remake
|
|
|
11
11
|
from .rule import Rule, deferrable, rule
|
|
12
12
|
from .scope import ScopeWarning
|
|
13
13
|
from .task import Task
|
|
14
|
-
from .tokens import FileToken, OutputToken, PathToken, S3Object, ZarrStore
|
|
14
|
+
from .tokens import Dir, Ensure, FileToken, OutputToken, PathToken, S3Object, ZarrStore
|
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
"""Canonical renderings of rule state, for change detection (0.9.0 item 2).
|
|
2
|
+
|
|
3
|
+
Change detection compares what a task last ran with against what it would
|
|
4
|
+
run with now. The renderings here make that comparison mean what it says:
|
|
5
|
+
|
|
6
|
+
- **Deterministic.** Sets are sorted and dict keys are sorted (dict order is
|
|
7
|
+
not semantic under `==`), so `uses={'VARS': {'tas', 'pr'}}` no longer
|
|
8
|
+
reruns on every invocation because string hashing is randomised per
|
|
9
|
+
process (review 2026-09-24 H4).
|
|
10
|
+
- **Version-independent.** ASTs are dumped by our own dumper, which omits
|
|
11
|
+
empty fields and positions, instead of `ast.dump`, whose output changed in
|
|
12
|
+
Python 3.13 (H5).
|
|
13
|
+
- **Lossless where it matters.** numpy arrays hash their bytes (a repr
|
|
14
|
+
truncates past 1000 elements and rounds floats, M4); pandas objects hash
|
|
15
|
+
their values, index, columns and dtypes; `functools.partial`, numpy ufuncs,
|
|
16
|
+
builtins and sourceless functions render by what they compute, not by a
|
|
17
|
+
type label (M3).
|
|
18
|
+
- **Cosmetic changes are not changes.** Comments and whitespace never
|
|
19
|
+
reach an AST. Docstrings do count (review L7 proposed dropping them; MM
|
|
20
|
+
kept them, 2026-11-21: a docstring is part of the code a user reviews).
|
|
21
|
+
|
|
22
|
+
Stored strings carry a format marker (`FORMAT_MARKER`, the first line). A
|
|
23
|
+
stored string without it was written by remake <= 0.9.0a1; the planner
|
|
24
|
+
matches those through `legacy_*` (see `uses_match`/`io_match`) so upgrading
|
|
25
|
+
does not rerun everything, and restamps matched records to the new form.
|
|
26
|
+
"""
|
|
27
|
+
import ast
|
|
28
|
+
import dataclasses
|
|
29
|
+
import enum
|
|
30
|
+
import functools
|
|
31
|
+
import hashlib
|
|
32
|
+
import inspect
|
|
33
|
+
import json
|
|
34
|
+
import linecache
|
|
35
|
+
import marshal
|
|
36
|
+
import os
|
|
37
|
+
import sys
|
|
38
|
+
import textwrap
|
|
39
|
+
import types
|
|
40
|
+
|
|
41
|
+
FORMAT_MARKER = '#remake-canonical-v2'
|
|
42
|
+
|
|
43
|
+
# Fields that carry positions or are otherwise not semantic.
|
|
44
|
+
_SKIP_FIELDS = frozenset({'lineno', 'end_lineno', 'col_offset', 'end_col_offset',
|
|
45
|
+
'type_comment'})
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# --- AST ---------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
def dump_ast(node):
|
|
51
|
+
"""A version-independent dump of an AST: no positions, and empty fields
|
|
52
|
+
([] or None) omitted, so a field added in a later Python with an empty
|
|
53
|
+
default (e.g. 3.12's `type_params`) doesn't change the result."""
|
|
54
|
+
if isinstance(node, ast.AST):
|
|
55
|
+
parts = []
|
|
56
|
+
for name in node._fields:
|
|
57
|
+
if name in _SKIP_FIELDS or not hasattr(node, name):
|
|
58
|
+
continue
|
|
59
|
+
value = getattr(node, name)
|
|
60
|
+
if value is None and name != 'value':
|
|
61
|
+
continue
|
|
62
|
+
if isinstance(value, list) and not value:
|
|
63
|
+
continue
|
|
64
|
+
parts.append(f'{name}={dump_ast(value)}')
|
|
65
|
+
return f'{type(node).__name__}({", ".join(parts)})'
|
|
66
|
+
if isinstance(node, list):
|
|
67
|
+
return '[' + ', '.join(dump_ast(n) for n in node) + ']'
|
|
68
|
+
return repr(node)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _is_rule_decorator(dec):
|
|
72
|
+
target = dec.func if isinstance(dec, ast.Call) else dec
|
|
73
|
+
if isinstance(target, ast.Name):
|
|
74
|
+
return target.id == 'rule'
|
|
75
|
+
if isinstance(target, ast.Attribute):
|
|
76
|
+
return target.attr == 'rule'
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def normalise_def(tree, *, strip_rule_decorator=False):
|
|
81
|
+
"""Normalise a parsed module holding one def/class in place: for a rule
|
|
82
|
+
function, drop the `@rule(...)` decorator and any decorators above it (H2: those configure or wrap the Rule object,
|
|
83
|
+
not the function that runs). Decorators *below* `@rule` wrap the
|
|
84
|
+
function itself and stay. Returns the tree."""
|
|
85
|
+
for node in tree.body:
|
|
86
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
87
|
+
if strip_rule_decorator:
|
|
88
|
+
idx = next((i for i, d in enumerate(node.decorator_list)
|
|
89
|
+
if _is_rule_decorator(d)), None)
|
|
90
|
+
if idx is not None:
|
|
91
|
+
node.decorator_list = node.decorator_list[idx + 1:]
|
|
92
|
+
break
|
|
93
|
+
return tree
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# Reconstructing an AST from `ast.dump` text without eval: the dump is a
|
|
97
|
+
# Python expression of Call/List/Constant nodes, so parse it and rebuild the
|
|
98
|
+
# nodes it names. Used only to read legacy stored strings.
|
|
99
|
+
|
|
100
|
+
class _NotRebuildable(Exception):
|
|
101
|
+
pass
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _rebuild(node):
|
|
105
|
+
if isinstance(node, ast.Call):
|
|
106
|
+
if not isinstance(node.func, ast.Name) or node.args:
|
|
107
|
+
raise _NotRebuildable
|
|
108
|
+
cls = getattr(ast, node.func.id, None)
|
|
109
|
+
if not (isinstance(cls, type) and issubclass(cls, ast.AST)):
|
|
110
|
+
raise _NotRebuildable(node.func.id)
|
|
111
|
+
fields = {kw.arg: _rebuild(kw.value) for kw in node.keywords}
|
|
112
|
+
known = set(cls._fields)
|
|
113
|
+
# Fields this Python's node class lacks (from another version's dump)
|
|
114
|
+
# are dropped: dump_ast only renders this Python's fields anyway.
|
|
115
|
+
obj = cls(**{k: v for k, v in fields.items() if k in known})
|
|
116
|
+
return obj
|
|
117
|
+
if isinstance(node, ast.List):
|
|
118
|
+
return [_rebuild(e) for e in node.elts]
|
|
119
|
+
if isinstance(node, ast.Constant):
|
|
120
|
+
return node.value
|
|
121
|
+
if isinstance(node, ast.Name) and node.id == 'Ellipsis':
|
|
122
|
+
return Ellipsis
|
|
123
|
+
if (isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub)
|
|
124
|
+
and isinstance(node.operand, ast.Constant)):
|
|
125
|
+
return -node.operand.value
|
|
126
|
+
raise _NotRebuildable(type(node).__name__)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def ast_from_dump(text):
|
|
130
|
+
"""The AST an `ast.dump()` string describes, or None if it isn't one we
|
|
131
|
+
can rebuild (unknown node class, not a dump at all)."""
|
|
132
|
+
try:
|
|
133
|
+
expr = ast.parse(text, mode='eval').body
|
|
134
|
+
return _rebuild(expr)
|
|
135
|
+
except (SyntaxError, ValueError, RecursionError, MemoryError, _NotRebuildable,
|
|
136
|
+
TypeError, AttributeError):
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# --- sources -----------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
@functools.lru_cache(maxsize=64)
|
|
143
|
+
def _module_tree(path, mtime_ns):
|
|
144
|
+
linecache.checkcache(path) # the file may have changed since it was cached
|
|
145
|
+
lines = linecache.getlines(path)
|
|
146
|
+
if not lines:
|
|
147
|
+
return None
|
|
148
|
+
try:
|
|
149
|
+
return ast.parse(''.join(lines), filename=path)
|
|
150
|
+
except (SyntaxError, ValueError):
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _lambda_nodes(fn):
|
|
155
|
+
"""The Lambda node(s) in fn's source file that could be fn: those on its
|
|
156
|
+
first line with its argument names. None if the file can't be read."""
|
|
157
|
+
code = fn.__code__
|
|
158
|
+
try:
|
|
159
|
+
path = inspect.getsourcefile(fn)
|
|
160
|
+
mtime_ns = os.stat(path).st_mtime_ns
|
|
161
|
+
except (TypeError, OSError):
|
|
162
|
+
return None
|
|
163
|
+
tree = _module_tree(path, mtime_ns)
|
|
164
|
+
if tree is None:
|
|
165
|
+
return None
|
|
166
|
+
argnames = list(code.co_varnames[:code.co_argcount + code.co_kwonlyargcount])
|
|
167
|
+
found = []
|
|
168
|
+
for node in ast.walk(tree):
|
|
169
|
+
if isinstance(node, ast.Lambda) and node.lineno == code.co_firstlineno:
|
|
170
|
+
a = node.args
|
|
171
|
+
names = [x.arg for x in a.posonlyargs + a.args + a.kwonlyargs]
|
|
172
|
+
if names == argnames:
|
|
173
|
+
found.append(node)
|
|
174
|
+
return found or None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def callable_source_ast(fn, *, strip_rule_decorator=False):
|
|
178
|
+
"""Canonical AST text of a function, lambda or class, or None when no
|
|
179
|
+
source is available. A lambda's `getsource` is the whole line(s) it sits
|
|
180
|
+
on (often an unparseable `@rule(inputs=lambda ...` fragment, and with the
|
|
181
|
+
neighbouring arguments in it); here it is cut down to the lambda itself."""
|
|
182
|
+
if getattr(fn, '__name__', None) == '<lambda>' and hasattr(fn, '__code__'):
|
|
183
|
+
nodes = _lambda_nodes(fn)
|
|
184
|
+
if nodes:
|
|
185
|
+
# Several identical-signature lambdas on one line: all of them.
|
|
186
|
+
return ' | '.join(dump_ast(n) for n in nodes)
|
|
187
|
+
try:
|
|
188
|
+
src = inspect.getsource(fn)
|
|
189
|
+
except (OSError, TypeError):
|
|
190
|
+
return None
|
|
191
|
+
try:
|
|
192
|
+
tree = ast.parse(textwrap.dedent(src))
|
|
193
|
+
except SyntaxError:
|
|
194
|
+
return None
|
|
195
|
+
return dump_ast(normalise_def(tree, strip_rule_decorator=strip_rule_decorator))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _qualname(obj):
|
|
199
|
+
return f'{getattr(obj, "__module__", "?")}.{getattr(obj, "__qualname__", getattr(obj, "__name__", "?"))}'
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _code_digest(code):
|
|
203
|
+
"""A sourceless function's identity: its full code object (constants,
|
|
204
|
+
names and nested code included — not just co_code, review L8)."""
|
|
205
|
+
try:
|
|
206
|
+
return hashlib.sha256(marshal.dumps(code)).hexdigest()[:16]
|
|
207
|
+
except ValueError:
|
|
208
|
+
return hashlib.sha256(code.co_code).hexdigest()[:16]
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
# --- values ------------------------------------------------------------------
|
|
212
|
+
|
|
213
|
+
_MAX_DEPTH = 40
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _sorted_join(items):
|
|
217
|
+
return ', '.join(sorted(items))
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _has_address(text):
|
|
221
|
+
return ' at 0x' in text or 'object at ' in text
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def canonical_value(value, *, project_dir=None, _depth=0):
|
|
225
|
+
"""A deterministic text for a `uses` value (see the module docstring).
|
|
226
|
+
|
|
227
|
+
Plain values render as their repr, with dict keys and set elements
|
|
228
|
+
sorted; most other objects as their repr too, as long as it is stable
|
|
229
|
+
(no memory address). Types can opt in to their own rendering with a
|
|
230
|
+
`__remake_hash__(self) -> str` method."""
|
|
231
|
+
if _depth > _MAX_DEPTH:
|
|
232
|
+
return '<too deep>'
|
|
233
|
+
kw = {'project_dir': project_dir, '_depth': _depth + 1}
|
|
234
|
+
t = type(value)
|
|
235
|
+
hook = getattr(t, '__remake_hash__', None)
|
|
236
|
+
if hook is not None and not isinstance(value, type):
|
|
237
|
+
return f'{_qualname(t)}<{value.__remake_hash__()}>'
|
|
238
|
+
if value is None or t in (bool, int, float, complex, str, bytes):
|
|
239
|
+
return repr(value)
|
|
240
|
+
if isinstance(value, enum.Enum):
|
|
241
|
+
return f'{_qualname(t)}.{value.name}'
|
|
242
|
+
if t is list:
|
|
243
|
+
return '[' + ', '.join(canonical_value(v, **kw) for v in value) + ']'
|
|
244
|
+
if t is tuple:
|
|
245
|
+
inner = ', '.join(canonical_value(v, **kw) for v in value)
|
|
246
|
+
return f'({inner},)' if len(value) == 1 else f'({inner})'
|
|
247
|
+
if t is dict:
|
|
248
|
+
items = (f'{canonical_value(k, **kw)}: {canonical_value(v, **kw)}'
|
|
249
|
+
for k, v in value.items())
|
|
250
|
+
return '{' + _sorted_join(items) + '}'
|
|
251
|
+
if t in (set, frozenset):
|
|
252
|
+
if not value:
|
|
253
|
+
return f'{t.__name__}()'
|
|
254
|
+
inner = '{' + _sorted_join(canonical_value(v, **kw) for v in value) + '}'
|
|
255
|
+
return inner if t is set else f'frozenset({inner})'
|
|
256
|
+
if isinstance(value, types.ModuleType):
|
|
257
|
+
return _module_rendering(value, project_dir)
|
|
258
|
+
if isinstance(value, functools.partial):
|
|
259
|
+
args = ', '.join(canonical_value(a, **kw) for a in value.args)
|
|
260
|
+
kws = _sorted_join(f'{k}={canonical_value(v, **kw)}'
|
|
261
|
+
for k, v in value.keywords.items())
|
|
262
|
+
return f'partial({canonical_value(value.func, **kw)}; {args}; {kws})'
|
|
263
|
+
root = (getattr(t, '__module__', '') or '').split('.')[0]
|
|
264
|
+
if root == 'numpy':
|
|
265
|
+
rendered = _numpy_rendering(value, kw)
|
|
266
|
+
if rendered is not None:
|
|
267
|
+
return rendered
|
|
268
|
+
if root == 'pandas':
|
|
269
|
+
rendered = _pandas_rendering(value)
|
|
270
|
+
if rendered is not None:
|
|
271
|
+
return rendered
|
|
272
|
+
if (inspect.isfunction(value) or inspect.ismethod(value) or inspect.isclass(value)
|
|
273
|
+
or inspect.isbuiltin(value) or inspect.isroutine(value)):
|
|
274
|
+
return _callable_rendering(value, kw)
|
|
275
|
+
# Containers that aren't the exact builtins (dataclasses, namedtuples,
|
|
276
|
+
# dict/list/set subclasses) render through their contents too, or a set
|
|
277
|
+
# inside one would order randomly per process (code review 2026-11-21).
|
|
278
|
+
name = _qualname(t)
|
|
279
|
+
if dataclasses.is_dataclass(value):
|
|
280
|
+
missing = object() # an unset field(init=False) has no attribute
|
|
281
|
+
fields = {f.name: v for f in dataclasses.fields(value)
|
|
282
|
+
if (v := getattr(value, f.name, missing)) is not missing}
|
|
283
|
+
return f'{name}{canonical_value(fields, **kw)}'
|
|
284
|
+
if isinstance(value, tuple) and hasattr(value, '_asdict'):
|
|
285
|
+
return f'{name}{canonical_value(dict(value._asdict()), **kw)}'
|
|
286
|
+
if isinstance(value, dict):
|
|
287
|
+
return f'{name}{canonical_value(dict(value), **kw)}'
|
|
288
|
+
if isinstance(value, (set, frozenset)):
|
|
289
|
+
return f'{name}{canonical_value(set(value), **kw)}'
|
|
290
|
+
if isinstance(value, (list, tuple)):
|
|
291
|
+
return f'{name}{canonical_value(list(value), **kw)}'
|
|
292
|
+
text = repr(value)
|
|
293
|
+
if _has_address(text):
|
|
294
|
+
# The repr embeds a memory address: it would differ every process.
|
|
295
|
+
# Render the instance by its attributes when it has them; otherwise
|
|
296
|
+
# all we can stably say is its type (body changes go unseen).
|
|
297
|
+
state = getattr(value, '__dict__', None)
|
|
298
|
+
if isinstance(state, dict):
|
|
299
|
+
return f'{_qualname(t)}{canonical_value(state, **kw)}'
|
|
300
|
+
return f'<unrenderable {_qualname(t)}>'
|
|
301
|
+
return text
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _callable_rendering(fn, kw=None):
|
|
305
|
+
if inspect.ismethod(fn):
|
|
306
|
+
# Pass the depth on: an object holding a bound method of itself
|
|
307
|
+
# would otherwise recurse without limit.
|
|
308
|
+
return f'method {_qualname(fn)} of {canonical_value(fn.__self__, **(kw or {}))}'
|
|
309
|
+
src = callable_source_ast(fn)
|
|
310
|
+
if src is not None:
|
|
311
|
+
return f'source {src}'
|
|
312
|
+
code = getattr(fn, '__code__', None)
|
|
313
|
+
if code is not None:
|
|
314
|
+
return f'code {_qualname(fn)} {_code_digest(code)}'
|
|
315
|
+
if inspect.isclass(fn):
|
|
316
|
+
# Sourceless class (exec/REPL): only its name is stable; body
|
|
317
|
+
# changes go unseen.
|
|
318
|
+
return f'class {_qualname(fn)}'
|
|
319
|
+
return f'builtin {_qualname(fn)}'
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _numpy_rendering(value, kw):
|
|
323
|
+
import numpy as np
|
|
324
|
+
|
|
325
|
+
if isinstance(value, np.ndarray):
|
|
326
|
+
if value.dtype.hasobject:
|
|
327
|
+
# Object arrays hold pointers: hash their elements instead.
|
|
328
|
+
return (f'ndarray(object, {value.shape}, '
|
|
329
|
+
f'{canonical_value(value.tolist(), **kw)})')
|
|
330
|
+
digest = hashlib.sha256(np.ascontiguousarray(value).tobytes()).hexdigest()
|
|
331
|
+
return f'ndarray({value.dtype.str}, {value.shape}, {digest})'
|
|
332
|
+
if isinstance(value, np.generic):
|
|
333
|
+
return f'numpy.{type(value).__name__}({value.item()!r})'
|
|
334
|
+
if isinstance(value, np.ufunc):
|
|
335
|
+
return f'numpy.ufunc {value.__name__}'
|
|
336
|
+
if isinstance(value, np.dtype):
|
|
337
|
+
return f'numpy.dtype({value.str})'
|
|
338
|
+
return None
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _pandas_rendering(value):
|
|
342
|
+
import pandas as pd
|
|
343
|
+
|
|
344
|
+
if isinstance(value, (pd.DataFrame, pd.Series, pd.Index)):
|
|
345
|
+
try:
|
|
346
|
+
hashed = pd.util.hash_pandas_object(value, index=not isinstance(value, pd.Index))
|
|
347
|
+
digest = hashlib.sha256(hashed.values.tobytes()).hexdigest()
|
|
348
|
+
except TypeError:
|
|
349
|
+
# Unhashable object cells (lists, dicts): render the contents
|
|
350
|
+
# instead (ultrareview 2026-11-21).
|
|
351
|
+
if isinstance(value, pd.DataFrame):
|
|
352
|
+
data = value.to_dict(orient='split')
|
|
353
|
+
elif isinstance(value, pd.Series):
|
|
354
|
+
data = {'index': list(value.index), 'data': list(value)}
|
|
355
|
+
else:
|
|
356
|
+
data = list(value)
|
|
357
|
+
digest = hashlib.sha256(canonical_value(data).encode()).hexdigest()
|
|
358
|
+
if isinstance(value, pd.DataFrame):
|
|
359
|
+
meta = repr([(str(c), str(d)) for c, d in value.dtypes.items()])
|
|
360
|
+
else:
|
|
361
|
+
meta = repr((str(getattr(value, 'name', None)), str(value.dtype)))
|
|
362
|
+
return f'{type(value).__name__}({meta}, {value.shape}, {digest})'
|
|
363
|
+
return None
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
_ENV_PATHS = None
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _environment_roots():
|
|
370
|
+
"""Directories whose modules are environment (stdlib, site-packages)."""
|
|
371
|
+
global _ENV_PATHS
|
|
372
|
+
if _ENV_PATHS is None:
|
|
373
|
+
import site
|
|
374
|
+
import sysconfig
|
|
375
|
+
|
|
376
|
+
roots = set()
|
|
377
|
+
for key in ('stdlib', 'platstdlib', 'purelib', 'platlib'):
|
|
378
|
+
p = sysconfig.get_paths().get(key)
|
|
379
|
+
if p:
|
|
380
|
+
roots.add(os.path.realpath(p))
|
|
381
|
+
try:
|
|
382
|
+
roots.update(os.path.realpath(p) for p in site.getsitepackages())
|
|
383
|
+
except AttributeError:
|
|
384
|
+
pass
|
|
385
|
+
user = getattr(site, 'getusersitepackages', lambda: None)()
|
|
386
|
+
if user:
|
|
387
|
+
roots.add(os.path.realpath(user))
|
|
388
|
+
roots.add(os.path.realpath(sys.prefix))
|
|
389
|
+
roots.add(os.path.realpath(sys.base_prefix))
|
|
390
|
+
_ENV_PATHS = tuple(sorted(roots))
|
|
391
|
+
return _ENV_PATHS
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def is_project_module(module, project_dir):
|
|
395
|
+
"""A module whose file lives under `project_dir` (the remakefile's
|
|
396
|
+
directory) and not in the interpreter's environment: project code, whose
|
|
397
|
+
edits can change results. Editable installs and PYTHONPATH checkouts
|
|
398
|
+
elsewhere count as environment, like any installed library."""
|
|
399
|
+
path = getattr(module, '__file__', None)
|
|
400
|
+
if not path or not project_dir:
|
|
401
|
+
return False
|
|
402
|
+
path = os.path.realpath(path)
|
|
403
|
+
if any(path.startswith(root + os.sep) for root in _environment_roots()):
|
|
404
|
+
return False
|
|
405
|
+
project = os.path.realpath(project_dir)
|
|
406
|
+
return path.startswith(project + os.sep)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _module_rendering(module, project_dir):
|
|
410
|
+
if project_dir and is_project_module(module, project_dir):
|
|
411
|
+
try:
|
|
412
|
+
with open(module.__file__, 'rb') as f:
|
|
413
|
+
tree = ast.parse(f.read())
|
|
414
|
+
return f'module {module.__name__} {dump_ast(tree)}'
|
|
415
|
+
except (OSError, SyntaxError, ValueError):
|
|
416
|
+
pass
|
|
417
|
+
return f'module {module.__name__}'
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
# --- uses ----------------------------------------------------------------------
|
|
421
|
+
|
|
422
|
+
def uses_parts(uses, project_dir=None):
|
|
423
|
+
"""{name: rendering} for a uses dict, name-sorted."""
|
|
424
|
+
return {name: canonical_value(uses[name], project_dir=project_dir)
|
|
425
|
+
for name in sorted(uses)}
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def uses_hash(uses, project_dir=None):
|
|
429
|
+
"""The stored text for a uses dict: the marker line, then JSON of
|
|
430
|
+
{name: rendering}. Empty uses is '' (as before 0.9.0, so equal)."""
|
|
431
|
+
if not uses:
|
|
432
|
+
return ''
|
|
433
|
+
return FORMAT_MARKER + '\n' + json.dumps(uses_parts(uses, project_dir),
|
|
434
|
+
sort_keys=True, ensure_ascii=False)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def is_canonical(stored):
|
|
438
|
+
return stored.startswith(FORMAT_MARKER + '\n')
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def parse_canonical(stored):
|
|
442
|
+
return json.loads(stored[len(FORMAT_MARKER) + 1:])
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
# --- inputs/outputs ------------------------------------------------------------
|
|
446
|
+
|
|
447
|
+
def _sorted_dict_ast(tree):
|
|
448
|
+
"""Sort a dict display's keys (evaluated dict specs only: inside a
|
|
449
|
+
callable's source, `{**a, 'k': v}` order is semantic)."""
|
|
450
|
+
expr = tree.body[0].value if (tree.body and isinstance(tree.body[0], ast.Expr)) else None
|
|
451
|
+
if isinstance(expr, ast.Dict) and all(k is not None for k in expr.keys):
|
|
452
|
+
pairs = sorted(zip(expr.keys, expr.values), key=lambda kv: dump_ast(kv[0]))
|
|
453
|
+
expr.keys = [k for k, _ in pairs]
|
|
454
|
+
expr.values = [v for _, v in pairs]
|
|
455
|
+
return tree
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def io_part(spec):
|
|
459
|
+
"""Canonical text for one inputs/outputs spec: '' for none; a dict spec
|
|
460
|
+
by its repr's AST with keys sorted (reordering keys is not a rerun
|
|
461
|
+
trigger, matching `==`); a callable by its source AST."""
|
|
462
|
+
if spec is None:
|
|
463
|
+
return ''
|
|
464
|
+
if callable(spec) and not isinstance(spec, dict):
|
|
465
|
+
src = callable_source_ast(spec)
|
|
466
|
+
if src is not None:
|
|
467
|
+
return 'callable ' + src
|
|
468
|
+
return 'callable ' + _callable_rendering(spec)
|
|
469
|
+
from .tokens import FileToken
|
|
470
|
+
|
|
471
|
+
# A FileToken is what a plain path string becomes anyway, and Ensure
|
|
472
|
+
# options are not identity: `Ensure('a.nc')` and `'a.nc'` hash the same.
|
|
473
|
+
spec = {k: (v.path if type(v) is FileToken else v) for k, v in spec.items()}
|
|
474
|
+
try:
|
|
475
|
+
tree = ast.parse(repr(spec))
|
|
476
|
+
except SyntaxError:
|
|
477
|
+
return 'dict ' + canonical_value(spec)
|
|
478
|
+
return 'dict ' + dump_ast(_sorted_dict_ast(tree))
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def io_hash(rule):
|
|
482
|
+
return FORMAT_MARKER + '\n' + json.dumps(
|
|
483
|
+
{'inputs': io_part(rule.inputs), 'outputs': io_part(rule.outputs)},
|
|
484
|
+
sort_keys=True, ensure_ascii=False)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
# --- legacy (remake <= 0.9.0a1) ----------------------------------------------
|
|
488
|
+
|
|
489
|
+
def normalise_legacy_dump(text, *, dict_spec):
|
|
490
|
+
"""Canonical text for a legacy `ast.dump` segment (an io spec or a uses
|
|
491
|
+
function), or None if it can't be rebuilt."""
|
|
492
|
+
tree = ast_from_dump(text)
|
|
493
|
+
if not isinstance(tree, ast.Module):
|
|
494
|
+
return None
|
|
495
|
+
if dict_spec:
|
|
496
|
+
expr = tree.body[0].value if (tree.body and isinstance(tree.body[0], ast.Expr)) else None
|
|
497
|
+
if isinstance(expr, ast.Dict):
|
|
498
|
+
return 'dict ' + dump_ast(_sorted_dict_ast(tree))
|
|
499
|
+
return None
|
|
500
|
+
return dump_ast(normalise_def(tree))
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _literal(node):
|
|
504
|
+
"""ast.literal_eval, plus `set()` and `frozenset({...})`."""
|
|
505
|
+
if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
|
|
506
|
+
and node.func.id in ('set', 'frozenset') and not node.keywords
|
|
507
|
+
and len(node.args) <= 1):
|
|
508
|
+
inner = _literal(node.args[0]) if node.args else ()
|
|
509
|
+
return set(inner) if node.func.id == 'set' else frozenset(inner)
|
|
510
|
+
if isinstance(node, (ast.Set, ast.List, ast.Tuple)):
|
|
511
|
+
items = [_literal(e) for e in node.elts]
|
|
512
|
+
return {ast.Set: set, ast.List: list, ast.Tuple: tuple}[type(node)](items)
|
|
513
|
+
if isinstance(node, ast.Dict):
|
|
514
|
+
return {_literal(k): _literal(v) for k, v in zip(node.keys, node.values)}
|
|
515
|
+
return ast.literal_eval(node)
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def normalise_legacy_value(text):
|
|
519
|
+
"""Canonical text for a legacy `repr` rendering of a plain value: parsed
|
|
520
|
+
back when it is a literal (so a set's random order or a dict's key order
|
|
521
|
+
stops mattering); otherwise the repr itself. (A legacy dataclass or
|
|
522
|
+
namedtuple therefore doesn't match here; it matches on the exact repr
|
|
523
|
+
comparison tried first, and is then restamped.)"""
|
|
524
|
+
try:
|
|
525
|
+
node = ast.parse(text, mode='eval').body
|
|
526
|
+
return canonical_value(_literal(node))
|
|
527
|
+
except (SyntaxError, ValueError, TypeError, RecursionError, MemoryError):
|
|
528
|
+
return text
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Rule-level DAG construction and task expansion. Pure functions, no I/O.
|
|
2
2
|
|
|
3
3
|
The rule DAG is the only graph remake builds — there is no task-level DAG.
|
|
4
|
-
See
|
|
4
|
+
See design_docs/design.md, "No task-level DAG".
|
|
5
5
|
"""
|
|
6
6
|
import itertools
|
|
7
7
|
|
|
@@ -117,6 +117,43 @@ def _check_scalar_kwargs(rule, kwargs):
|
|
|
117
117
|
)
|
|
118
118
|
|
|
119
119
|
|
|
120
|
+
def _canonical_scalar(value):
|
|
121
|
+
"""A matrix value as its exact plain type: numpy scalars via .item(),
|
|
122
|
+
enum members via .value, str/int/float subclasses as the base type
|
|
123
|
+
(review 2026-09-24 M19). Their reprs — and so task keys — otherwise
|
|
124
|
+
change across the SLURM JSON round-trip and numpy 1 -> 2. Anything else
|
|
125
|
+
is returned as is (and rejected by _check_scalar_kwargs)."""
|
|
126
|
+
import enum
|
|
127
|
+
|
|
128
|
+
t = type(value)
|
|
129
|
+
if value is None or t in (str, int, float, bool):
|
|
130
|
+
return value
|
|
131
|
+
if isinstance(value, enum.Enum):
|
|
132
|
+
value = value.value
|
|
133
|
+
if (getattr(type(value), '__module__', '') or '').split('.')[0] == 'numpy' \
|
|
134
|
+
and hasattr(value, 'item'):
|
|
135
|
+
import numpy as np
|
|
136
|
+
|
|
137
|
+
# Only numbers, bools and strings: datetime64/timedelta64 .item()
|
|
138
|
+
# gives an int of ns (or a datetime), silently changing the value;
|
|
139
|
+
# they stay rejected, as in 0.8 (ultrareview 2026-11-21).
|
|
140
|
+
if not isinstance(value, (np.number, np.bool_, np.str_)):
|
|
141
|
+
return value
|
|
142
|
+
value = value.item()
|
|
143
|
+
for base in (bool, int, float):
|
|
144
|
+
if isinstance(value, base):
|
|
145
|
+
return base(value)
|
|
146
|
+
if isinstance(value, str):
|
|
147
|
+
return ''.join([value]) # the base str, bypassing a subclass __str__
|
|
148
|
+
return value
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _canonical_kwargs(kwargs):
|
|
152
|
+
out = {k: _canonical_scalar(v) for k, v in kwargs.items()}
|
|
153
|
+
changed = any(type(out[k]) is not type(v) for k, v in kwargs.items())
|
|
154
|
+
return out, changed
|
|
155
|
+
|
|
156
|
+
|
|
120
157
|
def iter_expand_rule(rule, predicate=None):
|
|
121
158
|
"""Generator form of expand_rule — yields Tasks one at a time."""
|
|
122
159
|
kwargs_list = resolve_matrix(rule.matrix)
|
|
@@ -124,10 +161,18 @@ def iter_expand_rule(rule, predicate=None):
|
|
|
124
161
|
# Deferred half of the signature contract: parameter names were
|
|
125
162
|
# unknowable at decoration time for callable matrices.
|
|
126
163
|
_check_expanded_kwargs(rule, kwargs_list[0])
|
|
127
|
-
|
|
164
|
+
from .task import task_key
|
|
165
|
+
|
|
166
|
+
for original in kwargs_list:
|
|
167
|
+
kw, changed = _canonical_kwargs(original)
|
|
128
168
|
_check_scalar_kwargs(rule, kw)
|
|
129
169
|
if predicate is None or predicate({**kw, 'rule': rule.name}):
|
|
130
|
-
|
|
170
|
+
task = Task(rule=rule, kwargs=kw)
|
|
171
|
+
if changed:
|
|
172
|
+
legacy = task_key(rule.name, original)
|
|
173
|
+
if legacy != task.key:
|
|
174
|
+
task.legacy_key = legacy
|
|
175
|
+
yield task
|
|
131
176
|
|
|
132
177
|
|
|
133
178
|
def _check_expanded_kwargs(rule, kwargs):
|