remake 0.8.4__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {remake-0.8.4 → remake-0.9.0}/PKG-INFO +2 -1
  2. {remake-0.8.4 → remake-0.9.0}/pyproject.toml +4 -0
  3. {remake-0.8.4 → remake-0.9.0}/src/remake/__init__.py +2 -0
  4. remake-0.9.0/src/remake/__main__.py +8 -0
  5. {remake-0.8.4 → remake-0.9.0}/src/remake/core/__init__.py +1 -1
  6. remake-0.9.0/src/remake/core/canonical.py +528 -0
  7. {remake-0.8.4 → remake-0.9.0}/src/remake/core/dag.py +48 -3
  8. remake-0.9.0/src/remake/core/deps.py +201 -0
  9. remake-0.9.0/src/remake/core/dot.py +84 -0
  10. {remake-0.8.4 → remake-0.9.0}/src/remake/core/exceptions.py +4 -0
  11. {remake-0.8.4 → remake-0.9.0}/src/remake/core/planner.py +210 -92
  12. {remake-0.8.4 → remake-0.9.0}/src/remake/core/remake.py +485 -20
  13. {remake-0.8.4 → remake-0.9.0}/src/remake/core/rule.py +8 -4
  14. remake-0.9.0/src/remake/core/scope.py +556 -0
  15. {remake-0.8.4 → remake-0.9.0}/src/remake/core/task.py +30 -2
  16. remake-0.9.0/src/remake/core/tokens.py +387 -0
  17. remake-0.9.0/src/remake/envcheck.py +193 -0
  18. {remake-0.8.4 → remake-0.9.0}/src/remake/executors/dask_executor.py +80 -21
  19. remake-0.9.0/src/remake/executors/multiproc_executor.py +270 -0
  20. {remake-0.8.4 → remake-0.9.0}/src/remake/executors/singleproc_executor.py +3 -1
  21. {remake-0.8.4 → remake-0.9.0}/src/remake/executors/slurm_executor.py +79 -20
  22. {remake-0.8.4 → remake-0.9.0}/src/remake/loader/__init__.py +18 -2
  23. {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/metadata_manager.py +59 -4
  24. {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/sidecar.py +15 -5
  25. {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/sqlite3_backend.py +298 -17
  26. remake-0.9.0/src/remake/profiles/jasmin.toml +18 -0
  27. {remake-0.8.4 → remake-0.9.0}/src/remake/remake_cmd.py +428 -13
  28. {remake-0.8.4 → remake-0.9.0}/src/remake/util/__init__.py +0 -1
  29. {remake-0.8.4 → remake-0.9.0}/src/remake/util/code_compare.py +12 -2
  30. remake-0.9.0/src/remake/util/config.py +230 -0
  31. remake-0.9.0/src/remake/util/resources.py +325 -0
  32. {remake-0.8.4 → remake-0.9.0}/src/remake/util/util.py +1 -1
  33. {remake-0.8.4 → remake-0.9.0}/src/remake/version.py +1 -1
  34. {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/PKG-INFO +2 -1
  35. {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/SOURCES.txt +7 -0
  36. {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/requires.txt +3 -0
  37. remake-0.8.4/src/remake/core/scope.py +0 -313
  38. remake-0.8.4/src/remake/core/tokens.py +0 -91
  39. remake-0.8.4/src/remake/executors/multiproc_executor.py +0 -143
  40. remake-0.8.4/src/remake/util/config.py +0 -36
  41. {remake-0.8.4 → remake-0.9.0}/LICENCE +0 -0
  42. {remake-0.8.4 → remake-0.9.0}/README.md +0 -0
  43. {remake-0.8.4 → remake-0.9.0}/setup.cfg +0 -0
  44. {remake-0.8.4 → remake-0.9.0}/src/remake/executors/__init__.py +0 -0
  45. {remake-0.8.4 → remake-0.9.0}/src/remake/executors/executor.py +0 -0
  46. {remake-0.8.4 → remake-0.9.0}/src/remake/metadata/__init__.py +0 -0
  47. {remake-0.8.4 → remake-0.9.0}/src/remake/util/colour.py +0 -0
  48. {remake-0.8.4 → remake-0.9.0}/src/remake/util/command_line_args.py +0 -0
  49. {remake-0.8.4 → remake-0.9.0}/src/remake/util/run_lock.py +0 -0
  50. {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/dependency_links.txt +0 -0
  51. {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/entry_points.txt +0 -0
  52. {remake-0.8.4 → remake-0.9.0}/src/remake.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: remake
3
- Version: 0.8.4
3
+ Version: 0.9.0
4
4
  Summary: Smart remake tool
5
5
  Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
6
6
  Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
@@ -227,6 +227,7 @@ Description-Content-Type: text/markdown
227
227
  License-File: LICENCE
228
228
  Requires-Dist: loguru
229
229
  Requires-Dist: networkx
230
+ Requires-Dist: tomli; python_version < "3.11"
230
231
  Provides-Extra: debug
231
232
  Requires-Dist: ipdb; extra == "debug"
232
233
  Provides-Extra: s3
@@ -17,6 +17,7 @@ requires-python = ">=3.10"
17
17
  dependencies = [
18
18
  "loguru",
19
19
  "networkx",
20
+ "tomli; python_version < '3.11'",
20
21
  ]
21
22
  dynamic = ["version"]
22
23
  classifiers = [
@@ -85,6 +86,9 @@ remake = "remake.remake_cmd:remake_cmd"
85
86
  [tool.setuptools.packages.find]
86
87
  where = ["src"]
87
88
 
89
+ [tool.setuptools.package-data]
90
+ remake = ["profiles/*.toml"]
91
+
88
92
  [tool.setuptools.dynamic]
89
93
  version = { attr = "remake.version.__version__" }
90
94
 
@@ -2,6 +2,8 @@ from .version import __version__
2
2
 
3
3
  from .core import (
4
4
  Defer,
5
+ Dir,
6
+ Ensure,
5
7
  FileToken,
6
8
  OutputToken,
7
9
  PathToken,
@@ -0,0 +1,8 @@
1
+ """`python -m remake ...` runs the remake CLI (used by `run-all`, which runs
2
+ each remakefile in its own process)."""
3
+ import sys
4
+
5
+ from .remake_cmd import remake_cmd
6
+
7
+ if __name__ == '__main__':
8
+ sys.exit(remake_cmd(sys.argv))
@@ -11,4 +11,4 @@ from .remake import Remake
11
11
  from .rule import Rule, deferrable, rule
12
12
  from .scope import ScopeWarning
13
13
  from .task import Task
14
- from .tokens import FileToken, OutputToken, PathToken, S3Object, ZarrStore
14
+ from .tokens import Dir, Ensure, FileToken, OutputToken, PathToken, S3Object, ZarrStore
@@ -0,0 +1,528 @@
1
+ """Canonical renderings of rule state, for change detection (0.9.0 item 2).
2
+
3
+ Change detection compares what a task last ran with against what it would
4
+ run with now. The renderings here make that comparison mean what it says:
5
+
6
+ - **Deterministic.** Sets are sorted and dict keys are sorted (dict order is
7
+ not semantic under `==`), so `uses={'VARS': {'tas', 'pr'}}` no longer
8
+ reruns on every invocation because string hashing is randomised per
9
+ process (review 2026-09-24 H4).
10
+ - **Version-independent.** ASTs are dumped by our own dumper, which omits
11
+ empty fields and positions, instead of `ast.dump`, whose output changed in
12
+ Python 3.13 (H5).
13
+ - **Lossless where it matters.** numpy arrays hash their bytes (a repr
14
+ truncates past 1000 elements and rounds floats, M4); pandas objects hash
15
+ their values, index, columns and dtypes; `functools.partial`, numpy ufuncs,
16
+ builtins and sourceless functions render by what they compute, not by a
17
+ type label (M3).
18
+ - **Cosmetic changes are not changes.** Comments and whitespace never
19
+ reach an AST. Docstrings do count (review L7 proposed dropping them; MM
20
+ kept them, 2026-11-21: a docstring is part of the code a user reviews).
21
+
22
+ Stored strings carry a format marker (`FORMAT_MARKER`, the first line). A
23
+ stored string without it was written by remake <= 0.9.0a1; the planner
24
+ matches those through `legacy_*` (see `uses_match`/`io_match`) so upgrading
25
+ does not rerun everything, and restamps matched records to the new form.
26
+ """
27
+ import ast
28
+ import dataclasses
29
+ import enum
30
+ import functools
31
+ import hashlib
32
+ import inspect
33
+ import json
34
+ import linecache
35
+ import marshal
36
+ import os
37
+ import sys
38
+ import textwrap
39
+ import types
40
+
41
+ FORMAT_MARKER = '#remake-canonical-v2'
42
+
43
+ # Fields that carry positions or are otherwise not semantic.
44
+ _SKIP_FIELDS = frozenset({'lineno', 'end_lineno', 'col_offset', 'end_col_offset',
45
+ 'type_comment'})
46
+
47
+
48
+ # --- AST ---------------------------------------------------------------------
49
+
50
+ def dump_ast(node):
51
+ """A version-independent dump of an AST: no positions, and empty fields
52
+ ([] or None) omitted, so a field added in a later Python with an empty
53
+ default (e.g. 3.12's `type_params`) doesn't change the result."""
54
+ if isinstance(node, ast.AST):
55
+ parts = []
56
+ for name in node._fields:
57
+ if name in _SKIP_FIELDS or not hasattr(node, name):
58
+ continue
59
+ value = getattr(node, name)
60
+ if value is None and name != 'value':
61
+ continue
62
+ if isinstance(value, list) and not value:
63
+ continue
64
+ parts.append(f'{name}={dump_ast(value)}')
65
+ return f'{type(node).__name__}({", ".join(parts)})'
66
+ if isinstance(node, list):
67
+ return '[' + ', '.join(dump_ast(n) for n in node) + ']'
68
+ return repr(node)
69
+
70
+
71
+ def _is_rule_decorator(dec):
72
+ target = dec.func if isinstance(dec, ast.Call) else dec
73
+ if isinstance(target, ast.Name):
74
+ return target.id == 'rule'
75
+ if isinstance(target, ast.Attribute):
76
+ return target.attr == 'rule'
77
+ return False
78
+
79
+
80
+ def normalise_def(tree, *, strip_rule_decorator=False):
81
+ """Normalise a parsed module holding one def/class in place: for a rule
82
+ function, drop the `@rule(...)` decorator and any decorators above it (H2: those configure or wrap the Rule object,
83
+ not the function that runs). Decorators *below* `@rule` wrap the
84
+ function itself and stay. Returns the tree."""
85
+ for node in tree.body:
86
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
87
+ if strip_rule_decorator:
88
+ idx = next((i for i, d in enumerate(node.decorator_list)
89
+ if _is_rule_decorator(d)), None)
90
+ if idx is not None:
91
+ node.decorator_list = node.decorator_list[idx + 1:]
92
+ break
93
+ return tree
94
+
95
+
96
+ # Reconstructing an AST from `ast.dump` text without eval: the dump is a
97
+ # Python expression of Call/List/Constant nodes, so parse it and rebuild the
98
+ # nodes it names. Used only to read legacy stored strings.
99
+
100
+ class _NotRebuildable(Exception):
101
+ pass
102
+
103
+
104
+ def _rebuild(node):
105
+ if isinstance(node, ast.Call):
106
+ if not isinstance(node.func, ast.Name) or node.args:
107
+ raise _NotRebuildable
108
+ cls = getattr(ast, node.func.id, None)
109
+ if not (isinstance(cls, type) and issubclass(cls, ast.AST)):
110
+ raise _NotRebuildable(node.func.id)
111
+ fields = {kw.arg: _rebuild(kw.value) for kw in node.keywords}
112
+ known = set(cls._fields)
113
+ # Fields this Python's node class lacks (from another version's dump)
114
+ # are dropped: dump_ast only renders this Python's fields anyway.
115
+ obj = cls(**{k: v for k, v in fields.items() if k in known})
116
+ return obj
117
+ if isinstance(node, ast.List):
118
+ return [_rebuild(e) for e in node.elts]
119
+ if isinstance(node, ast.Constant):
120
+ return node.value
121
+ if isinstance(node, ast.Name) and node.id == 'Ellipsis':
122
+ return Ellipsis
123
+ if (isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub)
124
+ and isinstance(node.operand, ast.Constant)):
125
+ return -node.operand.value
126
+ raise _NotRebuildable(type(node).__name__)
127
+
128
+
129
+ def ast_from_dump(text):
130
+ """The AST an `ast.dump()` string describes, or None if it isn't one we
131
+ can rebuild (unknown node class, not a dump at all)."""
132
+ try:
133
+ expr = ast.parse(text, mode='eval').body
134
+ return _rebuild(expr)
135
+ except (SyntaxError, ValueError, RecursionError, MemoryError, _NotRebuildable,
136
+ TypeError, AttributeError):
137
+ return None
138
+
139
+
140
+ # --- sources -----------------------------------------------------------------
141
+
142
+ @functools.lru_cache(maxsize=64)
143
+ def _module_tree(path, mtime_ns):
144
+ linecache.checkcache(path) # the file may have changed since it was cached
145
+ lines = linecache.getlines(path)
146
+ if not lines:
147
+ return None
148
+ try:
149
+ return ast.parse(''.join(lines), filename=path)
150
+ except (SyntaxError, ValueError):
151
+ return None
152
+
153
+
154
+ def _lambda_nodes(fn):
155
+ """The Lambda node(s) in fn's source file that could be fn: those on its
156
+ first line with its argument names. None if the file can't be read."""
157
+ code = fn.__code__
158
+ try:
159
+ path = inspect.getsourcefile(fn)
160
+ mtime_ns = os.stat(path).st_mtime_ns
161
+ except (TypeError, OSError):
162
+ return None
163
+ tree = _module_tree(path, mtime_ns)
164
+ if tree is None:
165
+ return None
166
+ argnames = list(code.co_varnames[:code.co_argcount + code.co_kwonlyargcount])
167
+ found = []
168
+ for node in ast.walk(tree):
169
+ if isinstance(node, ast.Lambda) and node.lineno == code.co_firstlineno:
170
+ a = node.args
171
+ names = [x.arg for x in a.posonlyargs + a.args + a.kwonlyargs]
172
+ if names == argnames:
173
+ found.append(node)
174
+ return found or None
175
+
176
+
177
+ def callable_source_ast(fn, *, strip_rule_decorator=False):
178
+ """Canonical AST text of a function, lambda or class, or None when no
179
+ source is available. A lambda's `getsource` is the whole line(s) it sits
180
+ on (often an unparseable `@rule(inputs=lambda ...` fragment, and with the
181
+ neighbouring arguments in it); here it is cut down to the lambda itself."""
182
+ if getattr(fn, '__name__', None) == '<lambda>' and hasattr(fn, '__code__'):
183
+ nodes = _lambda_nodes(fn)
184
+ if nodes:
185
+ # Several identical-signature lambdas on one line: all of them.
186
+ return ' | '.join(dump_ast(n) for n in nodes)
187
+ try:
188
+ src = inspect.getsource(fn)
189
+ except (OSError, TypeError):
190
+ return None
191
+ try:
192
+ tree = ast.parse(textwrap.dedent(src))
193
+ except SyntaxError:
194
+ return None
195
+ return dump_ast(normalise_def(tree, strip_rule_decorator=strip_rule_decorator))
196
+
197
+
198
+ def _qualname(obj):
199
+ return f'{getattr(obj, "__module__", "?")}.{getattr(obj, "__qualname__", getattr(obj, "__name__", "?"))}'
200
+
201
+
202
+ def _code_digest(code):
203
+ """A sourceless function's identity: its full code object (constants,
204
+ names and nested code included — not just co_code, review L8)."""
205
+ try:
206
+ return hashlib.sha256(marshal.dumps(code)).hexdigest()[:16]
207
+ except ValueError:
208
+ return hashlib.sha256(code.co_code).hexdigest()[:16]
209
+
210
+
211
+ # --- values ------------------------------------------------------------------
212
+
213
+ _MAX_DEPTH = 40
214
+
215
+
216
+ def _sorted_join(items):
217
+ return ', '.join(sorted(items))
218
+
219
+
220
+ def _has_address(text):
221
+ return ' at 0x' in text or 'object at ' in text
222
+
223
+
224
+ def canonical_value(value, *, project_dir=None, _depth=0):
225
+ """A deterministic text for a `uses` value (see the module docstring).
226
+
227
+ Plain values render as their repr, with dict keys and set elements
228
+ sorted; most other objects as their repr too, as long as it is stable
229
+ (no memory address). Types can opt in to their own rendering with a
230
+ `__remake_hash__(self) -> str` method."""
231
+ if _depth > _MAX_DEPTH:
232
+ return '<too deep>'
233
+ kw = {'project_dir': project_dir, '_depth': _depth + 1}
234
+ t = type(value)
235
+ hook = getattr(t, '__remake_hash__', None)
236
+ if hook is not None and not isinstance(value, type):
237
+ return f'{_qualname(t)}<{value.__remake_hash__()}>'
238
+ if value is None or t in (bool, int, float, complex, str, bytes):
239
+ return repr(value)
240
+ if isinstance(value, enum.Enum):
241
+ return f'{_qualname(t)}.{value.name}'
242
+ if t is list:
243
+ return '[' + ', '.join(canonical_value(v, **kw) for v in value) + ']'
244
+ if t is tuple:
245
+ inner = ', '.join(canonical_value(v, **kw) for v in value)
246
+ return f'({inner},)' if len(value) == 1 else f'({inner})'
247
+ if t is dict:
248
+ items = (f'{canonical_value(k, **kw)}: {canonical_value(v, **kw)}'
249
+ for k, v in value.items())
250
+ return '{' + _sorted_join(items) + '}'
251
+ if t in (set, frozenset):
252
+ if not value:
253
+ return f'{t.__name__}()'
254
+ inner = '{' + _sorted_join(canonical_value(v, **kw) for v in value) + '}'
255
+ return inner if t is set else f'frozenset({inner})'
256
+ if isinstance(value, types.ModuleType):
257
+ return _module_rendering(value, project_dir)
258
+ if isinstance(value, functools.partial):
259
+ args = ', '.join(canonical_value(a, **kw) for a in value.args)
260
+ kws = _sorted_join(f'{k}={canonical_value(v, **kw)}'
261
+ for k, v in value.keywords.items())
262
+ return f'partial({canonical_value(value.func, **kw)}; {args}; {kws})'
263
+ root = (getattr(t, '__module__', '') or '').split('.')[0]
264
+ if root == 'numpy':
265
+ rendered = _numpy_rendering(value, kw)
266
+ if rendered is not None:
267
+ return rendered
268
+ if root == 'pandas':
269
+ rendered = _pandas_rendering(value)
270
+ if rendered is not None:
271
+ return rendered
272
+ if (inspect.isfunction(value) or inspect.ismethod(value) or inspect.isclass(value)
273
+ or inspect.isbuiltin(value) or inspect.isroutine(value)):
274
+ return _callable_rendering(value, kw)
275
+ # Containers that aren't the exact builtins (dataclasses, namedtuples,
276
+ # dict/list/set subclasses) render through their contents too, or a set
277
+ # inside one would order randomly per process (code review 2026-11-21).
278
+ name = _qualname(t)
279
+ if dataclasses.is_dataclass(value):
280
+ missing = object() # an unset field(init=False) has no attribute
281
+ fields = {f.name: v for f in dataclasses.fields(value)
282
+ if (v := getattr(value, f.name, missing)) is not missing}
283
+ return f'{name}{canonical_value(fields, **kw)}'
284
+ if isinstance(value, tuple) and hasattr(value, '_asdict'):
285
+ return f'{name}{canonical_value(dict(value._asdict()), **kw)}'
286
+ if isinstance(value, dict):
287
+ return f'{name}{canonical_value(dict(value), **kw)}'
288
+ if isinstance(value, (set, frozenset)):
289
+ return f'{name}{canonical_value(set(value), **kw)}'
290
+ if isinstance(value, (list, tuple)):
291
+ return f'{name}{canonical_value(list(value), **kw)}'
292
+ text = repr(value)
293
+ if _has_address(text):
294
+ # The repr embeds a memory address: it would differ every process.
295
+ # Render the instance by its attributes when it has them; otherwise
296
+ # all we can stably say is its type (body changes go unseen).
297
+ state = getattr(value, '__dict__', None)
298
+ if isinstance(state, dict):
299
+ return f'{_qualname(t)}{canonical_value(state, **kw)}'
300
+ return f'<unrenderable {_qualname(t)}>'
301
+ return text
302
+
303
+
304
+ def _callable_rendering(fn, kw=None):
305
+ if inspect.ismethod(fn):
306
+ # Pass the depth on: an object holding a bound method of itself
307
+ # would otherwise recurse without limit.
308
+ return f'method {_qualname(fn)} of {canonical_value(fn.__self__, **(kw or {}))}'
309
+ src = callable_source_ast(fn)
310
+ if src is not None:
311
+ return f'source {src}'
312
+ code = getattr(fn, '__code__', None)
313
+ if code is not None:
314
+ return f'code {_qualname(fn)} {_code_digest(code)}'
315
+ if inspect.isclass(fn):
316
+ # Sourceless class (exec/REPL): only its name is stable; body
317
+ # changes go unseen.
318
+ return f'class {_qualname(fn)}'
319
+ return f'builtin {_qualname(fn)}'
320
+
321
+
322
+ def _numpy_rendering(value, kw):
323
+ import numpy as np
324
+
325
+ if isinstance(value, np.ndarray):
326
+ if value.dtype.hasobject:
327
+ # Object arrays hold pointers: hash their elements instead.
328
+ return (f'ndarray(object, {value.shape}, '
329
+ f'{canonical_value(value.tolist(), **kw)})')
330
+ digest = hashlib.sha256(np.ascontiguousarray(value).tobytes()).hexdigest()
331
+ return f'ndarray({value.dtype.str}, {value.shape}, {digest})'
332
+ if isinstance(value, np.generic):
333
+ return f'numpy.{type(value).__name__}({value.item()!r})'
334
+ if isinstance(value, np.ufunc):
335
+ return f'numpy.ufunc {value.__name__}'
336
+ if isinstance(value, np.dtype):
337
+ return f'numpy.dtype({value.str})'
338
+ return None
339
+
340
+
341
+ def _pandas_rendering(value):
342
+ import pandas as pd
343
+
344
+ if isinstance(value, (pd.DataFrame, pd.Series, pd.Index)):
345
+ try:
346
+ hashed = pd.util.hash_pandas_object(value, index=not isinstance(value, pd.Index))
347
+ digest = hashlib.sha256(hashed.values.tobytes()).hexdigest()
348
+ except TypeError:
349
+ # Unhashable object cells (lists, dicts): render the contents
350
+ # instead (ultrareview 2026-11-21).
351
+ if isinstance(value, pd.DataFrame):
352
+ data = value.to_dict(orient='split')
353
+ elif isinstance(value, pd.Series):
354
+ data = {'index': list(value.index), 'data': list(value)}
355
+ else:
356
+ data = list(value)
357
+ digest = hashlib.sha256(canonical_value(data).encode()).hexdigest()
358
+ if isinstance(value, pd.DataFrame):
359
+ meta = repr([(str(c), str(d)) for c, d in value.dtypes.items()])
360
+ else:
361
+ meta = repr((str(getattr(value, 'name', None)), str(value.dtype)))
362
+ return f'{type(value).__name__}({meta}, {value.shape}, {digest})'
363
+ return None
364
+
365
+
366
+ _ENV_PATHS = None
367
+
368
+
369
+ def _environment_roots():
370
+ """Directories whose modules are environment (stdlib, site-packages)."""
371
+ global _ENV_PATHS
372
+ if _ENV_PATHS is None:
373
+ import site
374
+ import sysconfig
375
+
376
+ roots = set()
377
+ for key in ('stdlib', 'platstdlib', 'purelib', 'platlib'):
378
+ p = sysconfig.get_paths().get(key)
379
+ if p:
380
+ roots.add(os.path.realpath(p))
381
+ try:
382
+ roots.update(os.path.realpath(p) for p in site.getsitepackages())
383
+ except AttributeError:
384
+ pass
385
+ user = getattr(site, 'getusersitepackages', lambda: None)()
386
+ if user:
387
+ roots.add(os.path.realpath(user))
388
+ roots.add(os.path.realpath(sys.prefix))
389
+ roots.add(os.path.realpath(sys.base_prefix))
390
+ _ENV_PATHS = tuple(sorted(roots))
391
+ return _ENV_PATHS
392
+
393
+
394
+ def is_project_module(module, project_dir):
395
+ """A module whose file lives under `project_dir` (the remakefile's
396
+ directory) and not in the interpreter's environment: project code, whose
397
+ edits can change results. Editable installs and PYTHONPATH checkouts
398
+ elsewhere count as environment, like any installed library."""
399
+ path = getattr(module, '__file__', None)
400
+ if not path or not project_dir:
401
+ return False
402
+ path = os.path.realpath(path)
403
+ if any(path.startswith(root + os.sep) for root in _environment_roots()):
404
+ return False
405
+ project = os.path.realpath(project_dir)
406
+ return path.startswith(project + os.sep)
407
+
408
+
409
+ def _module_rendering(module, project_dir):
410
+ if project_dir and is_project_module(module, project_dir):
411
+ try:
412
+ with open(module.__file__, 'rb') as f:
413
+ tree = ast.parse(f.read())
414
+ return f'module {module.__name__} {dump_ast(tree)}'
415
+ except (OSError, SyntaxError, ValueError):
416
+ pass
417
+ return f'module {module.__name__}'
418
+
419
+
420
+ # --- uses ----------------------------------------------------------------------
421
+
422
+ def uses_parts(uses, project_dir=None):
423
+ """{name: rendering} for a uses dict, name-sorted."""
424
+ return {name: canonical_value(uses[name], project_dir=project_dir)
425
+ for name in sorted(uses)}
426
+
427
+
428
+ def uses_hash(uses, project_dir=None):
429
+ """The stored text for a uses dict: the marker line, then JSON of
430
+ {name: rendering}. Empty uses is '' (as before 0.9.0, so equal)."""
431
+ if not uses:
432
+ return ''
433
+ return FORMAT_MARKER + '\n' + json.dumps(uses_parts(uses, project_dir),
434
+ sort_keys=True, ensure_ascii=False)
435
+
436
+
437
+ def is_canonical(stored):
438
+ return stored.startswith(FORMAT_MARKER + '\n')
439
+
440
+
441
+ def parse_canonical(stored):
442
+ return json.loads(stored[len(FORMAT_MARKER) + 1:])
443
+
444
+
445
+ # --- inputs/outputs ------------------------------------------------------------
446
+
447
+ def _sorted_dict_ast(tree):
448
+ """Sort a dict display's keys (evaluated dict specs only: inside a
449
+ callable's source, `{**a, 'k': v}` order is semantic)."""
450
+ expr = tree.body[0].value if (tree.body and isinstance(tree.body[0], ast.Expr)) else None
451
+ if isinstance(expr, ast.Dict) and all(k is not None for k in expr.keys):
452
+ pairs = sorted(zip(expr.keys, expr.values), key=lambda kv: dump_ast(kv[0]))
453
+ expr.keys = [k for k, _ in pairs]
454
+ expr.values = [v for _, v in pairs]
455
+ return tree
456
+
457
+
458
+ def io_part(spec):
459
+ """Canonical text for one inputs/outputs spec: '' for none; a dict spec
460
+ by its repr's AST with keys sorted (reordering keys is not a rerun
461
+ trigger, matching `==`); a callable by its source AST."""
462
+ if spec is None:
463
+ return ''
464
+ if callable(spec) and not isinstance(spec, dict):
465
+ src = callable_source_ast(spec)
466
+ if src is not None:
467
+ return 'callable ' + src
468
+ return 'callable ' + _callable_rendering(spec)
469
+ from .tokens import FileToken
470
+
471
+ # A FileToken is what a plain path string becomes anyway, and Ensure
472
+ # options are not identity: `Ensure('a.nc')` and `'a.nc'` hash the same.
473
+ spec = {k: (v.path if type(v) is FileToken else v) for k, v in spec.items()}
474
+ try:
475
+ tree = ast.parse(repr(spec))
476
+ except SyntaxError:
477
+ return 'dict ' + canonical_value(spec)
478
+ return 'dict ' + dump_ast(_sorted_dict_ast(tree))
479
+
480
+
481
+ def io_hash(rule):
482
+ return FORMAT_MARKER + '\n' + json.dumps(
483
+ {'inputs': io_part(rule.inputs), 'outputs': io_part(rule.outputs)},
484
+ sort_keys=True, ensure_ascii=False)
485
+
486
+
487
+ # --- legacy (remake <= 0.9.0a1) ----------------------------------------------
488
+
489
+ def normalise_legacy_dump(text, *, dict_spec):
490
+ """Canonical text for a legacy `ast.dump` segment (an io spec or a uses
491
+ function), or None if it can't be rebuilt."""
492
+ tree = ast_from_dump(text)
493
+ if not isinstance(tree, ast.Module):
494
+ return None
495
+ if dict_spec:
496
+ expr = tree.body[0].value if (tree.body and isinstance(tree.body[0], ast.Expr)) else None
497
+ if isinstance(expr, ast.Dict):
498
+ return 'dict ' + dump_ast(_sorted_dict_ast(tree))
499
+ return None
500
+ return dump_ast(normalise_def(tree))
501
+
502
+
503
+ def _literal(node):
504
+ """ast.literal_eval, plus `set()` and `frozenset({...})`."""
505
+ if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
506
+ and node.func.id in ('set', 'frozenset') and not node.keywords
507
+ and len(node.args) <= 1):
508
+ inner = _literal(node.args[0]) if node.args else ()
509
+ return set(inner) if node.func.id == 'set' else frozenset(inner)
510
+ if isinstance(node, (ast.Set, ast.List, ast.Tuple)):
511
+ items = [_literal(e) for e in node.elts]
512
+ return {ast.Set: set, ast.List: list, ast.Tuple: tuple}[type(node)](items)
513
+ if isinstance(node, ast.Dict):
514
+ return {_literal(k): _literal(v) for k, v in zip(node.keys, node.values)}
515
+ return ast.literal_eval(node)
516
+
517
+
518
+ def normalise_legacy_value(text):
519
+ """Canonical text for a legacy `repr` rendering of a plain value: parsed
520
+ back when it is a literal (so a set's random order or a dict's key order
521
+ stops mattering); otherwise the repr itself. (A legacy dataclass or
522
+ namedtuple therefore doesn't match here; it matches on the exact repr
523
+ comparison tried first, and is then restamped.)"""
524
+ try:
525
+ node = ast.parse(text, mode='eval').body
526
+ return canonical_value(_literal(node))
527
+ except (SyntaxError, ValueError, TypeError, RecursionError, MemoryError):
528
+ return text
@@ -1,7 +1,7 @@
1
1
  """Rule-level DAG construction and task expansion. Pure functions, no I/O.
2
2
 
3
3
  The rule DAG is the only graph remake builds — there is no task-level DAG.
4
- See remake3_design.md, "No task-level DAG".
4
+ See design_docs/design.md, "No task-level DAG".
5
5
  """
6
6
  import itertools
7
7
 
@@ -117,6 +117,43 @@ def _check_scalar_kwargs(rule, kwargs):
117
117
  )
118
118
 
119
119
 
120
+ def _canonical_scalar(value):
121
+ """A matrix value as its exact plain type: numpy scalars via .item(),
122
+ enum members via .value, str/int/float subclasses as the base type
123
+ (review 2026-09-24 M19). Their reprs — and so task keys — otherwise
124
+ change across the SLURM JSON round-trip and numpy 1 -> 2. Anything else
125
+ is returned as is (and rejected by _check_scalar_kwargs)."""
126
+ import enum
127
+
128
+ t = type(value)
129
+ if value is None or t in (str, int, float, bool):
130
+ return value
131
+ if isinstance(value, enum.Enum):
132
+ value = value.value
133
+ if (getattr(type(value), '__module__', '') or '').split('.')[0] == 'numpy' \
134
+ and hasattr(value, 'item'):
135
+ import numpy as np
136
+
137
+ # Only numbers, bools and strings: datetime64/timedelta64 .item()
138
+ # gives an int of ns (or a datetime), silently changing the value;
139
+ # they stay rejected, as in 0.8 (ultrareview 2026-11-21).
140
+ if not isinstance(value, (np.number, np.bool_, np.str_)):
141
+ return value
142
+ value = value.item()
143
+ for base in (bool, int, float):
144
+ if isinstance(value, base):
145
+ return base(value)
146
+ if isinstance(value, str):
147
+ return ''.join([value]) # the base str, bypassing a subclass __str__
148
+ return value
149
+
150
+
151
+ def _canonical_kwargs(kwargs):
152
+ out = {k: _canonical_scalar(v) for k, v in kwargs.items()}
153
+ changed = any(type(out[k]) is not type(v) for k, v in kwargs.items())
154
+ return out, changed
155
+
156
+
120
157
  def iter_expand_rule(rule, predicate=None):
121
158
  """Generator form of expand_rule — yields Tasks one at a time."""
122
159
  kwargs_list = resolve_matrix(rule.matrix)
@@ -124,10 +161,18 @@ def iter_expand_rule(rule, predicate=None):
124
161
  # Deferred half of the signature contract: parameter names were
125
162
  # unknowable at decoration time for callable matrices.
126
163
  _check_expanded_kwargs(rule, kwargs_list[0])
127
- for kw in kwargs_list:
164
+ from .task import task_key
165
+
166
+ for original in kwargs_list:
167
+ kw, changed = _canonical_kwargs(original)
128
168
  _check_scalar_kwargs(rule, kw)
129
169
  if predicate is None or predicate({**kw, 'rule': rule.name}):
130
- yield Task(rule=rule, kwargs=kw)
170
+ task = Task(rule=rule, kwargs=kw)
171
+ if changed:
172
+ legacy = task_key(rule.name, original)
173
+ if legacy != task.key:
174
+ task.legacy_key = legacy
175
+ yield task
131
176
 
132
177
 
133
178
  def _check_expanded_kwargs(rule, kwargs):