remake 0.6.3__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. remake-0.8.0/PKG-INFO +255 -0
  2. remake-0.8.0/README.md +18 -0
  3. remake-0.8.0/pyproject.toml +89 -0
  4. remake-0.8.0/src/remake/__init__.py +28 -0
  5. remake-0.8.0/src/remake/core/__init__.py +14 -0
  6. remake-0.8.0/src/remake/core/dag.py +142 -0
  7. remake-0.8.0/src/remake/core/exceptions.py +38 -0
  8. remake-0.8.0/src/remake/core/planner.py +482 -0
  9. remake-0.8.0/src/remake/core/remake.py +751 -0
  10. remake-0.8.0/src/remake/core/rule.py +231 -0
  11. remake-0.8.0/src/remake/core/scope.py +313 -0
  12. remake-0.8.0/src/remake/core/task.py +61 -0
  13. remake-0.8.0/src/remake/core/tokens.py +91 -0
  14. {remake-0.6.3/remake/executor → remake-0.8.0/src/remake/executors}/__init__.py +3 -3
  15. remake-0.8.0/src/remake/executors/dask_executor.py +139 -0
  16. remake-0.8.0/src/remake/executors/executor.py +24 -0
  17. remake-0.8.0/src/remake/executors/multiproc_executor.py +138 -0
  18. remake-0.8.0/src/remake/executors/singleproc_executor.py +40 -0
  19. remake-0.8.0/src/remake/executors/slurm_executor.py +327 -0
  20. remake-0.8.0/src/remake/loader/__init__.py +59 -0
  21. remake-0.8.0/src/remake/metadata/__init__.py +10 -0
  22. remake-0.8.0/src/remake/metadata/metadata_manager.py +126 -0
  23. remake-0.8.0/src/remake/metadata/sidecar.py +68 -0
  24. remake-0.8.0/src/remake/metadata/sqlite3_backend.py +570 -0
  25. remake-0.8.0/src/remake/remake_cmd.py +955 -0
  26. remake-0.8.0/src/remake/util/__init__.py +5 -0
  27. remake-0.8.0/src/remake/util/code_compare.py +84 -0
  28. remake-0.8.0/src/remake/util/colour.py +64 -0
  29. remake-0.8.0/src/remake/util/command_line_args.py +40 -0
  30. remake-0.8.0/src/remake/util/config.py +36 -0
  31. remake-0.8.0/src/remake/util/util.py +53 -0
  32. remake-0.8.0/src/remake/version.py +5 -0
  33. remake-0.8.0/src/remake.egg-info/PKG-INFO +255 -0
  34. remake-0.8.0/src/remake.egg-info/SOURCES.txt +38 -0
  35. {remake-0.6.3 → remake-0.8.0/src}/remake.egg-info/entry_points.txt +0 -1
  36. remake-0.8.0/src/remake.egg-info/requires.txt +11 -0
  37. remake-0.6.3/MANIFEST.in +0 -4
  38. remake-0.6.3/PKG-INFO +0 -135
  39. remake-0.6.3/README.md +0 -109
  40. remake-0.6.3/remake/__init__.py +0 -23
  41. remake-0.6.3/remake/bcolors.py +0 -18
  42. remake-0.6.3/remake/examples/Makefile +0 -18
  43. remake-0.6.3/remake/examples/README.md +0 -26
  44. remake-0.6.3/remake/examples/__init__.py +0 -0
  45. remake-0.6.3/remake/examples/data/in.txt +0 -1
  46. remake-0.6.3/remake/examples/data/inputs/a.00.in +0 -0
  47. remake-0.6.3/remake/examples/data/inputs/a.01.in +0 -0
  48. remake-0.6.3/remake/examples/data/inputs/a.02.in +0 -0
  49. remake-0.6.3/remake/examples/data/inputs/a.10.in +0 -0
  50. remake-0.6.3/remake/examples/data/inputs/a.11.in +0 -0
  51. remake-0.6.3/remake/examples/data/inputs/a.12.in +0 -0
  52. remake-0.6.3/remake/examples/data/inputs/a.in +0 -1
  53. remake-0.6.3/remake/examples/data/inputs/b.in +0 -0
  54. remake-0.6.3/remake/examples/data/inputs/c.in +0 -0
  55. remake-0.6.3/remake/examples/data/inputs/ex8_in1.txt +0 -0
  56. remake-0.6.3/remake/examples/data/inputs/in1.txt +0 -12
  57. remake-0.6.3/remake/examples/data/inputs/in2.txt +0 -0
  58. remake-0.6.3/remake/examples/demo.py +0 -46
  59. remake-0.6.3/remake/examples/ex1.1.py +0 -28
  60. remake-0.6.3/remake/examples/ex1.2.py +0 -28
  61. remake-0.6.3/remake/examples/ex1.py +0 -28
  62. remake-0.6.3/remake/examples/ex2.py +0 -73
  63. remake-0.6.3/remake/examples/ex3.py +0 -39
  64. remake-0.6.3/remake/examples/ex4.py +0 -40
  65. remake-0.6.3/remake/examples/ex5.py +0 -27
  66. remake-0.6.3/remake/examples/ex6.py +0 -53
  67. remake-0.6.3/remake/examples/ex7.py +0 -30
  68. remake-0.6.3/remake/examples/ex8.py +0 -39
  69. remake-0.6.3/remake/examples/ex_slurm.py +0 -74
  70. remake-0.6.3/remake/examples/test_all_examples.py +0 -160
  71. remake-0.6.3/remake/executor/base_executor.py +0 -38
  72. remake-0.6.3/remake/executor/multiproc_executor.py +0 -120
  73. remake-0.6.3/remake/executor/singleproc_executor.py +0 -23
  74. remake-0.6.3/remake/executor/slurm_executor.py +0 -201
  75. remake-0.6.3/remake/experimental/__init__.py +0 -0
  76. remake-0.6.3/remake/experimental/networkx_displays.py +0 -34
  77. remake-0.6.3/remake/flags.py +0 -25
  78. remake-0.6.3/remake/formatter.py +0 -111
  79. remake-0.6.3/remake/loader.py +0 -22
  80. remake-0.6.3/remake/metadata.py +0 -355
  81. remake-0.6.3/remake/monitor.py +0 -359
  82. remake-0.6.3/remake/remake_base.py +0 -443
  83. remake-0.6.3/remake/remake_cmd.py +0 -586
  84. remake-0.6.3/remake/remake_exceptions.py +0 -24
  85. remake-0.6.3/remake/setup_logging.py +0 -103
  86. remake-0.6.3/remake/special_paths.py +0 -68
  87. remake-0.6.3/remake/task.py +0 -293
  88. remake-0.6.3/remake/task_control.py +0 -690
  89. remake-0.6.3/remake/task_query_set.py +0 -139
  90. remake-0.6.3/remake/task_rule.py +0 -199
  91. remake-0.6.3/remake/util.py +0 -106
  92. remake-0.6.3/remake/version.py +0 -24
  93. remake-0.6.3/remake.egg-info/PKG-INFO +0 -135
  94. remake-0.6.3/remake.egg-info/SOURCES.txt +0 -64
  95. remake-0.6.3/remake.egg-info/requires.txt +0 -18
  96. remake-0.6.3/setup.py +0 -68
  97. {remake-0.6.3 → remake-0.8.0}/LICENCE +0 -0
  98. {remake-0.6.3 → remake-0.8.0}/setup.cfg +0 -0
  99. {remake-0.6.3 → remake-0.8.0/src}/remake.egg-info/dependency_links.txt +0 -0
  100. {remake-0.6.3 → remake-0.8.0/src}/remake.egg-info/top_level.txt +0 -0
remake-0.8.0/PKG-INFO ADDED
@@ -0,0 +1,255 @@
1
+ Metadata-Version: 2.4
2
+ Name: remake
3
+ Version: 0.8.0
4
+ Summary: Smart remake tool
5
+ Author-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
6
+ Maintainer-email: Mark Muetzelfeldt <mark.muetzelfeldt@reading.ac.uk>
7
+ License: Apache License
8
+ Version 2.0, January 2004
9
+ http://www.apache.org/licenses/
10
+
11
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
12
+
13
+ 1. Definitions.
14
+
15
+ "License" shall mean the terms and conditions for use, reproduction,
16
+ and distribution as defined by Sections 1 through 9 of this document.
17
+
18
+ "Licensor" shall mean the copyright owner or entity authorized by
19
+ the copyright owner that is granting the License.
20
+
21
+ "Legal Entity" shall mean the union of the acting entity and all
22
+ other entities that control, are controlled by, or are under common
23
+ control with that entity. For the purposes of this definition,
24
+ "control" means (i) the power, direct or indirect, to cause the
25
+ direction or management of such entity, whether by contract or
26
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
27
+ outstanding shares, or (iii) beneficial ownership of such entity.
28
+
29
+ "You" (or "Your") shall mean an individual or Legal Entity
30
+ exercising permissions granted by this License.
31
+
32
+ "Source" form shall mean the preferred form for making modifications,
33
+ including but not limited to software source code, documentation
34
+ source, and configuration files.
35
+
36
+ "Object" form shall mean any form resulting from mechanical
37
+ transformation or translation of a Source form, including but
38
+ not limited to compiled object code, generated documentation,
39
+ and conversions to other media types.
40
+
41
+ "Work" shall mean the work of authorship, whether in Source or
42
+ Object form, made available under the License, as indicated by a
43
+ copyright notice that is included in or attached to the work
44
+ (an example is provided in the Appendix below).
45
+
46
+ "Derivative Works" shall mean any work, whether in Source or Object
47
+ form, that is based on (or derived from) the Work and for which the
48
+ editorial revisions, annotations, elaborations, or other modifications
49
+ represent, as a whole, an original work of authorship. For the purposes
50
+ of this License, Derivative Works shall not include works that remain
51
+ separable from, or merely link (or bind by name) to the interfaces of,
52
+ the Work and Derivative Works thereof.
53
+
54
+ "Contribution" shall mean any work of authorship, including
55
+ the original version of the Work and any modifications or additions
56
+ to that Work or Derivative Works thereof, that is intentionally
57
+ submitted to Licensor for inclusion in the Work by the copyright owner
58
+ or by an individual or Legal Entity authorized to submit on behalf of
59
+ the copyright owner. For the purposes of this definition, "submitted"
60
+ means any form of electronic, verbal, or written communication sent
61
+ to the Licensor or its representatives, including but not limited to
62
+ communication on electronic mailing lists, source code control systems,
63
+ and issue tracking systems that are managed by, or on behalf of, the
64
+ Licensor for the purpose of discussing and improving the Work, but
65
+ excluding communication that is conspicuously marked or otherwise
66
+ designated in writing by the copyright owner as "Not a Contribution."
67
+
68
+ "Contributor" shall mean Licensor and any individual or Legal Entity
69
+ on behalf of whom a Contribution has been received by Licensor and
70
+ subsequently incorporated within the Work.
71
+
72
+ 2. Grant of Copyright License. Subject to the terms and conditions of
73
+ this License, each Contributor hereby grants to You a perpetual,
74
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
75
+ copyright license to reproduce, prepare Derivative Works of,
76
+ publicly display, publicly perform, sublicense, and distribute the
77
+ Work and such Derivative Works in Source or Object form.
78
+
79
+ 3. Grant of Patent License. Subject to the terms and conditions of
80
+ this License, each Contributor hereby grants to You a perpetual,
81
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
82
+ (except as stated in this section) patent license to make, have made,
83
+ use, offer to sell, sell, import, and otherwise transfer the Work,
84
+ where such license applies only to those patent claims licensable
85
+ by such Contributor that are necessarily infringed by their
86
+ Contribution(s) alone or by combination of their Contribution(s)
87
+ with the Work to which such Contribution(s) was submitted. If You
88
+ institute patent litigation against any entity (including a
89
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
90
+ or a Contribution incorporated within the Work constitutes direct
91
+ or contributory patent infringement, then any patent licenses
92
+ granted to You under this License for that Work shall terminate
93
+ as of the date such litigation is filed.
94
+
95
+ 4. Redistribution. You may reproduce and distribute copies of the
96
+ Work or Derivative Works thereof in any medium, with or without
97
+ modifications, and in Source or Object form, provided that You
98
+ meet the following conditions:
99
+
100
+ (a) You must give any other recipients of the Work or
101
+ Derivative Works a copy of this License; and
102
+
103
+ (b) You must cause any modified files to carry prominent notices
104
+ stating that You changed the files; and
105
+
106
+ (c) You must retain, in the Source form of any Derivative Works
107
+ that You distribute, all copyright, patent, trademark, and
108
+ attribution notices from the Source form of the Work,
109
+ excluding those notices that do not pertain to any part of
110
+ the Derivative Works; and
111
+
112
+ (d) If the Work includes a "NOTICE" text file as part of its
113
+ distribution, then any Derivative Works that You distribute must
114
+ include a readable copy of the attribution notices contained
115
+ within such NOTICE file, excluding those notices that do not
116
+ pertain to any part of the Derivative Works, in at least one
117
+ of the following places: within a NOTICE text file distributed
118
+ as part of the Derivative Works; within the Source form or
119
+ documentation, if provided along with the Derivative Works; or,
120
+ within a display generated by the Derivative Works, if and
121
+ wherever such third-party notices normally appear. The contents
122
+ of the NOTICE file are for informational purposes only and
123
+ do not modify the License. You may add Your own attribution
124
+ notices within Derivative Works that You distribute, alongside
125
+ or as an addendum to the NOTICE text from the Work, provided
126
+ that such additional attribution notices cannot be construed
127
+ as modifying the License.
128
+
129
+ You may add Your own copyright statement to Your modifications and
130
+ may provide additional or different license terms and conditions
131
+ for use, reproduction, or distribution of Your modifications, or
132
+ for any such Derivative Works as a whole, provided Your use,
133
+ reproduction, and distribution of the Work otherwise complies with
134
+ the conditions stated in this License.
135
+
136
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
137
+ any Contribution intentionally submitted for inclusion in the Work
138
+ by You to the Licensor shall be under the terms and conditions of
139
+ this License, without any additional terms or conditions.
140
+ Notwithstanding the above, nothing herein shall supersede or modify
141
+ the terms of any separate license agreement you may have executed
142
+ with Licensor regarding such Contributions.
143
+
144
+ 6. Trademarks. This License does not grant permission to use the trade
145
+ names, trademarks, service marks, or product names of the Licensor,
146
+ except as required for reasonable and customary use in describing the
147
+ origin of the Work and reproducing the content of the NOTICE file.
148
+
149
+ 7. Disclaimer of Warranty. Unless required by applicable law or
150
+ agreed to in writing, Licensor provides the Work (and each
151
+ Contributor provides its Contributions) on an "AS IS" BASIS,
152
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
153
+ implied, including, without limitation, any warranties or conditions
154
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
155
+ PARTICULAR PURPOSE. You are solely responsible for determining the
156
+ appropriateness of using or redistributing the Work and assume any
157
+ risks associated with Your exercise of permissions under this License.
158
+
159
+ 8. Limitation of Liability. In no event and under no legal theory,
160
+ whether in tort (including negligence), contract, or otherwise,
161
+ unless required by applicable law (such as deliberate and grossly
162
+ negligent acts) or agreed to in writing, shall any Contributor be
163
+ liable to You for damages, including any direct, indirect, special,
164
+ incidental, or consequential damages of any character arising as a
165
+ result of this License or out of the use or inability to use the
166
+ Work (including but not limited to damages for loss of goodwill,
167
+ work stoppage, computer failure or malfunction, or any and all
168
+ other commercial damages or losses), even if such Contributor
169
+ has been advised of the possibility of such damages.
170
+
171
+ 9. Accepting Warranty or Additional Liability. While redistributing
172
+ the Work or Derivative Works thereof, You may choose to offer,
173
+ and charge a fee for, acceptance of support, warranty, indemnity,
174
+ or other liability obligations and/or rights consistent with this
175
+ License. However, in accepting such obligations, You may act only
176
+ on Your own behalf and on Your sole responsibility, not on behalf
177
+ of any other Contributor, and only if You agree to indemnify,
178
+ defend, and hold each Contributor harmless for any liability
179
+ incurred by, or claims asserted against, such Contributor by reason
180
+ of your accepting any such warranty or additional liability.
181
+
182
+ END OF TERMS AND CONDITIONS
183
+
184
+ APPENDIX: How to apply the Apache License to your work.
185
+
186
+ To apply the Apache License to your work, attach the following
187
+ boilerplate notice, with the fields enclosed by brackets "[]"
188
+ replaced with your own identifying information. (Don't include
189
+ the brackets!) The text should be enclosed in the appropriate
190
+ comment syntax for the file format. We also recommend that a
191
+ file or class name and description of purpose be included on the
192
+ same "printed page" as the copyright notice for easier
193
+ identification within third-party archives.
194
+
195
+ Copyright 2020 Mark R. Muetzelfeldt
196
+
197
+ Licensed under the Apache License, Version 2.0 (the "License");
198
+ you may not use this file except in compliance with the License.
199
+ You may obtain a copy of the License at
200
+
201
+ http://www.apache.org/licenses/LICENSE-2.0
202
+
203
+ Unless required by applicable law or agreed to in writing, software
204
+ distributed under the License is distributed on an "AS IS" BASIS,
205
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
206
+ See the License for the specific language governing permissions and
207
+ limitations under the License.
208
+
209
+
210
+ Project-URL: Homepage, https://github.com/markmuetz/remake
211
+ Project-URL: Documentation, https://markmuetz.github.io/remake
212
+ Project-URL: Bug Tracker, https://github.com/markmuetz/remake/issues
213
+ Classifier: Environment :: Console
214
+ Classifier: Intended Audience :: Science/Research
215
+ Classifier: Natural Language :: English
216
+ Classifier: Operating System :: POSIX :: Linux
217
+ Classifier: Programming Language :: Python :: 3.10
218
+ Classifier: Programming Language :: Python :: 3.11
219
+ Classifier: Programming Language :: Python :: 3.12
220
+ Classifier: Programming Language :: Python :: 3.13
221
+ Classifier: Programming Language :: Python :: 3.14
222
+ Classifier: Development Status :: 3 - Alpha
223
+ Classifier: License :: OSI Approved :: Apache Software License
224
+ Classifier: Topic :: Scientific/Engineering
225
+ Requires-Python: >=3.10
226
+ Description-Content-Type: text/markdown
227
+ License-File: LICENCE
228
+ Requires-Dist: loguru
229
+ Requires-Dist: networkx
230
+ Provides-Extra: debug
231
+ Requires-Dist: ipdb; extra == "debug"
232
+ Provides-Extra: s3
233
+ Requires-Dist: boto3; extra == "s3"
234
+ Provides-Extra: dask
235
+ Requires-Dist: distributed>=2024.8.0; extra == "dask"
236
+ Dynamic: license-file
237
+
238
+ # remake
239
+
240
+ Remake is a smart Python build tool, similar to `make`. It makes it easy to build pipelines with complex dependencies, and deploy these to high-performance computing systems. It is particularly suited to use for scientific workflows, due to its ability to reliably recreate any set of output files, based on running only those tasks that are necessary.
241
+
242
+ **Documentation: <https://markmuetz.github.io/remake>**
243
+
244
+ ## Status
245
+
246
+ remake 0.8.0 is a ground-up rewrite. See the
247
+ [changelog](https://markmuetz.github.io/remake/changelog/) for what's new and
248
+ [getting started](https://markmuetz.github.io/remake/getting-started/) for a
249
+ walkthrough.
250
+
251
+ ## Installation
252
+
253
+ ```bash
254
+ pip install remake
255
+ ```
remake-0.8.0/README.md ADDED
@@ -0,0 +1,18 @@
1
+ # remake
2
+
3
+ Remake is a smart Python build tool, similar to `make`. It makes it easy to build pipelines with complex dependencies, and deploy these to high-performance computing systems. It is particularly suited to use for scientific workflows, due to its ability to reliably recreate any set of output files, based on running only those tasks that are necessary.
4
+
5
+ **Documentation: <https://markmuetz.github.io/remake>**
6
+
7
+ ## Status
8
+
9
+ remake 0.8.0 is a ground-up rewrite. See the
10
+ [changelog](https://markmuetz.github.io/remake/changelog/) for what's new and
11
+ [getting started](https://markmuetz.github.io/remake/getting-started/) for a
12
+ walkthrough.
13
+
14
+ ## Installation
15
+
16
+ ```bash
17
+ pip install remake
18
+ ```
@@ -0,0 +1,89 @@
1
+ [build-system]
2
+ requires = ["setuptools >= 64"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "remake"
7
+ description = "Smart remake tool"
8
+ readme = "README.md"
9
+ license = { file = "LICENCE" }
10
+ authors = [
11
+ { name = "Mark Muetzelfeldt", email = "mark.muetzelfeldt@reading.ac.uk" },
12
+ ]
13
+ maintainers = [
14
+ { name = "Mark Muetzelfeldt", email = "mark.muetzelfeldt@reading.ac.uk" },
15
+ ]
16
+ requires-python = ">=3.10"
17
+ dependencies = [
18
+ "loguru",
19
+ "networkx",
20
+ ]
21
+ dynamic = ["version"]
22
+ classifiers = [
23
+ "Environment :: Console",
24
+ "Intended Audience :: Science/Research",
25
+ "Natural Language :: English",
26
+ "Operating System :: POSIX :: Linux",
27
+ "Programming Language :: Python :: 3.10",
28
+ "Programming Language :: Python :: 3.11",
29
+ "Programming Language :: Python :: 3.12",
30
+ "Programming Language :: Python :: 3.13",
31
+ "Programming Language :: Python :: 3.14",
32
+ "Development Status :: 3 - Alpha",
33
+ "License :: OSI Approved :: Apache Software License",
34
+ "Topic :: Scientific/Engineering",
35
+ ]
36
+
37
+ [project.urls]
38
+ Homepage = "https://github.com/markmuetz/remake"
39
+ Documentation = "https://markmuetz.github.io/remake"
40
+ "Bug Tracker" = "https://github.com/markmuetz/remake/issues"
41
+
42
+ [project.optional-dependencies]
43
+ debug = [
44
+ "ipdb",
45
+ ]
46
+ s3 = ["boto3"]
47
+ dask = [
48
+ "distributed>=2024.8.0",
49
+ ]
50
+
51
+ [dependency-groups]
52
+ dev = [
53
+ "pytest",
54
+ "pytest-cov",
55
+ "hypothesis",
56
+ "xarray>=2024.7.0",
57
+ "netcdf4>=1.7.2",
58
+ "h5netcdf>=1.8.1",
59
+ "h5py",
60
+ "zarr<3",
61
+ "dask",
62
+ "pyyaml>=6.0.3",
63
+ "distributed>=2024.8.0",
64
+ ]
65
+ docs = [
66
+ "mkdocs",
67
+ "mkdocs-material",
68
+ "mkdocstrings[python]",
69
+ ]
70
+
71
+ [project.scripts]
72
+ remake = "remake.remake_cmd:remake_cmd"
73
+
74
+ [tool.setuptools.packages.find]
75
+ where = ["src"]
76
+
77
+ [tool.setuptools.dynamic]
78
+ version = { attr = "remake.version.__version__" }
79
+
80
+ [tool.pytest.ini_options]
81
+ testpaths = ["tests"]
82
+
83
+ [tool.coverage.run]
84
+ source = ["remake"]
85
+ branch = true
86
+
87
+ [tool.coverage.report]
88
+ show_missing = true
89
+ fail_under = 85
@@ -0,0 +1,28 @@
1
+ from .version import __version__
2
+
3
+ from .core import (
4
+ Defer,
5
+ FileToken,
6
+ OutputToken,
7
+ PathToken,
8
+ Remake,
9
+ RemakeError,
10
+ Rule,
11
+ S3Object,
12
+ ScopeError,
13
+ ScopeWarning,
14
+ SignatureError,
15
+ Task,
16
+ ZarrStore,
17
+ deferrable,
18
+ rule,
19
+ )
20
+ from .executors import (
21
+ DaskExecutor,
22
+ Executor,
23
+ MultiprocExecutor,
24
+ SingleprocExecutor,
25
+ SlurmExecutor,
26
+ )
27
+ from .loader import load_remake
28
+ from .metadata import MetadataManager, Sqlite3Backend, TaskRecord
@@ -0,0 +1,14 @@
1
+ from .dag import build_rule_dag, expand_rule, resolve_matrix
2
+ from .exceptions import (
3
+ Defer,
4
+ RemakeError,
5
+ RemakeLoadError,
6
+ ScopeError,
7
+ SignatureError,
8
+ )
9
+ from .planner import plan
10
+ from .remake import Remake
11
+ from .rule import Rule, deferrable, rule
12
+ from .scope import ScopeWarning
13
+ from .task import Task
14
+ from .tokens import FileToken, OutputToken, PathToken, S3Object, ZarrStore
@@ -0,0 +1,142 @@
1
+ """Rule-level DAG construction and task expansion. Pure functions, no I/O.
2
+
3
+ The rule DAG is the only graph remake builds — there is no task-level DAG.
4
+ See remake3_design.md, "No task-level DAG".
5
+ """
6
+ import itertools
7
+
8
+ import networkx as nx
9
+
10
+ from .exceptions import Defer, SignatureError
11
+ from .rule import is_deferrable
12
+ from .task import Task
13
+
14
+
15
+ def build_rule_dag(rules):
16
+ """Directed rule-level DAG from explicit depends_on declarations."""
17
+ rules_by_name = {rule.name: rule for rule in rules}
18
+ g = nx.DiGraph()
19
+ for rule in rules:
20
+ g.add_node(rule)
21
+ resolved = []
22
+ for dep in rule.depends_on:
23
+ if isinstance(dep, str):
24
+ if dep not in rules_by_name:
25
+ raise ValueError(
26
+ f'{rule.name}: depends_on references unknown rule {dep!r}'
27
+ )
28
+ dep = rules_by_name[dep]
29
+ resolved.append(dep)
30
+ g.add_edge(dep, rule)
31
+ rule.depends_on = resolved
32
+ if not nx.is_directed_acyclic_graph(g):
33
+ cycle = nx.find_cycle(g)
34
+ raise ValueError(f'Rule dependencies contain a cycle: {cycle}')
35
+ return g
36
+
37
+
38
+ def resolve_matrix(matrix):
39
+ """Normalise all matrix forms to the canonical list[dict].
40
+
41
+ May raise Defer if matrix is a `@deferrable` callable whose required
42
+ upstream outputs do not yet exist. A non-`@deferrable` matrix that raises
43
+ Defer is a programming error (SignatureError) — the contract must be
44
+ declared explicitly.
45
+ """
46
+ if matrix is None:
47
+ return [{}]
48
+ if callable(matrix) and not isinstance(matrix, (dict, list)):
49
+ try:
50
+ return matrix()
51
+ except Defer:
52
+ if not is_deferrable(matrix):
53
+ name = getattr(matrix, '__name__', repr(matrix))
54
+ raise SignatureError(
55
+ f'matrix callable {name!r} raised Defer but is not marked '
56
+ f'@deferrable. Decorate it with @deferrable to declare that '
57
+ f'it derives its task list from upstream outputs and so may '
58
+ f'defer.'
59
+ ) from None
60
+ raise
61
+ if isinstance(matrix, list):
62
+ return matrix
63
+ # dict: cartesian product of axes. A scalar key is one kwarg per value; a
64
+ # tuple key binds several kwargs together per value (the remake2 grouped
65
+ # form), letting you supply an explicit, pre-filtered sequence of combos.
66
+ axes = []
67
+ for key, values in matrix.items():
68
+ if isinstance(key, tuple):
69
+ for v in values:
70
+ if not isinstance(v, tuple) or len(v) != len(key):
71
+ raise SignatureError(
72
+ f'matrix tuple key {key} expects value tuples of length '
73
+ f'{len(key)}, got {v!r}'
74
+ )
75
+ axes.append([dict(zip(key, v)) for v in values])
76
+ else:
77
+ axes.append([{key: v} for v in values])
78
+ return [
79
+ {k: v for axis in combo for k, v in axis.items()}
80
+ for combo in itertools.product(*axes)
81
+ ]
82
+
83
+
84
+ def expand_rule(rule, predicate=None):
85
+ """Expand the matrix for one rule into Task objects (no I/O).
86
+
87
+ predicate: optional callable(namespace) -> bool, applied before Task
88
+ construction so filtered-out tasks are never created. The namespace is
89
+ the task kwargs plus 'rule' (the rule name — it wins over a matrix key
90
+ of the same name), so queries can select by rule: "rule == 'extract'".
91
+ """
92
+ return list(iter_expand_rule(rule, predicate))
93
+
94
+
95
+ # Matrix kwarg values must survive the JSON round-trip used to ship SLURM
96
+ # job specs to compute nodes (a tuple/set comes back as a list, changing both
97
+ # Task.key and any kwarg-derived output path). Restrict them to JSON scalars,
98
+ # checked at plan time before anything is submitted. bool is an int subclass.
99
+ _SCALAR_TYPES = (str, int, float, bool, type(None))
100
+
101
+
102
+ def _check_scalar_kwargs(rule, kwargs):
103
+ for key, value in kwargs.items():
104
+ if not isinstance(value, _SCALAR_TYPES):
105
+ raise SignatureError(
106
+ f'{rule.name}: matrix value {key}={value!r} is a '
107
+ f'{type(value).__name__}; matrix kwarg values must be scalars '
108
+ f'(str, int, float, bool, None). Non-scalars do not survive the '
109
+ f'JSON round-trip used for SLURM job specs (e.g. a tuple returns '
110
+ f'as a list), which silently changes the task key and any output '
111
+ f'path built from it. Encode the value as a string instead — e.g. '
112
+ f'a short label, or a JSON string parsed inside the rule.'
113
+ )
114
+
115
+
116
+ def iter_expand_rule(rule, predicate=None):
117
+ """Generator form of expand_rule — yields Tasks one at a time."""
118
+ kwargs_list = resolve_matrix(rule.matrix)
119
+ if callable(rule.matrix) and kwargs_list:
120
+ # Deferred half of the signature contract: parameter names were
121
+ # unknowable at decoration time for callable matrices.
122
+ _check_expanded_kwargs(rule, kwargs_list[0])
123
+ for kw in kwargs_list:
124
+ _check_scalar_kwargs(rule, kw)
125
+ if predicate is None or predicate({**kw, 'rule': rule.name}):
126
+ yield Task(rule=rule, kwargs=kw)
127
+
128
+
129
+ def _check_expanded_kwargs(rule, kwargs):
130
+ import inspect
131
+
132
+ from .rule import check_io_spec
133
+
134
+ names = list(inspect.signature(rule.fn).parameters)
135
+ expected = [n for n in names if n not in ('inputs', 'outputs')]
136
+ if set(expected) != set(kwargs):
137
+ raise SignatureError(
138
+ f'{rule.name}: parameters {sorted(expected)} do not match resolved '
139
+ f'matrix keys {sorted(kwargs)}'
140
+ )
141
+ check_io_spec(rule.name, 'inputs', rule.inputs, set(kwargs))
142
+ check_io_spec(rule.name, 'outputs', rule.outputs, set(kwargs))
@@ -0,0 +1,38 @@
1
+ class RemakeError(Exception):
2
+ pass
3
+
4
+
5
+ class RemakeLoadError(RemakeError):
6
+ pass
7
+
8
+
9
+ class SignatureError(RemakeError):
10
+ """Rule function signature does not match its declarations."""
11
+
12
+ pass
13
+
14
+
15
+ class ScopeError(RemakeError):
16
+ """Rule function uses undeclared names from outer scope (strict mode)."""
17
+
18
+ pass
19
+
20
+
21
+ class Defer(Exception):
22
+ """Raised by a `@deferrable` matrix callable to signal that the rule
23
+ cannot be expanded this wave — its task list derives from an upstream
24
+ output that does not yet exist. A control-flow signal, not an error
25
+ (hence not a RemakeError): the planner defers the rule and the replan
26
+ loop / SLURM continuation job retries it once the upstream completes.
27
+
28
+ The planner additionally defers a `@deferrable` rule when an upstream is
29
+ *rerunning* this wave (its on-disk output is stale), so the matrix never
30
+ expands from an about-to-be-overwritten output.
31
+
32
+ Accepts path strings as context, surfaced to the user to show what is
33
+ blocking resolution.
34
+ """
35
+
36
+ def __init__(self, *paths):
37
+ self.paths = [str(p) for p in paths]
38
+ super().__init__(', '.join(self.paths) if self.paths else 'deferred')