restage 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {restage-0.3.1 → restage-0.4.0}/.github/workflows/wheels.yml +2 -2
- {restage-0.3.1/src/restage.egg-info → restage-0.4.0}/PKG-INFO +12 -4
- {restage-0.3.1 → restage-0.4.0}/pyproject.toml +13 -5
- {restage-0.3.1 → restage-0.4.0}/src/restage/energy.py +1 -1
- {restage-0.3.1 → restage-0.4.0}/src/restage/instr.py +20 -4
- {restage-0.3.1 → restage-0.4.0}/src/restage/splitrun.py +50 -56
- {restage-0.3.1 → restage-0.4.0}/src/restage/tables.py +2 -2
- {restage-0.3.1 → restage-0.4.0/src/restage.egg-info}/PKG-INFO +12 -4
- {restage-0.3.1 → restage-0.4.0}/src/restage.egg-info/SOURCES.txt +0 -2
- {restage-0.3.1 → restage-0.4.0}/src/restage.egg-info/entry_points.txt +1 -0
- restage-0.4.0/src/restage.egg-info/requires.txt +8 -0
- {restage-0.3.1 → restage-0.4.0}/test/test_single.py +64 -18
- restage-0.3.1/src/restage.egg-info/requires.txt +0 -11
- restage-0.3.1/test/splitRunTest_first.c +0 -12419
- restage-0.3.1/test/splitRunTest_second.c +0 -10944
- {restage-0.3.1 → restage-0.4.0}/.github/workflows/pip.yml +0 -0
- {restage-0.3.1 → restage-0.4.0}/.gitignore +0 -0
- {restage-0.3.1 → restage-0.4.0}/README.md +0 -0
- {restage-0.3.1 → restage-0.4.0}/setup.cfg +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/__init__.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/bifrost_choppers.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/cache.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/cspec_choppers.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/database.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/emulate.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/mcpl.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/range.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/run.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage/scan.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage.egg-info/dependency_links.txt +0 -0
- {restage-0.3.1 → restage-0.4.0}/src/restage.egg-info/top_level.txt +0 -0
- {restage-0.3.1 → restage-0.4.0}/test/test_cache.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/test/test_database.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/test/test_energy.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/test/test_range.py +0 -0
- {restage-0.3.1 → restage-0.4.0}/test/test_scan.py +0 -0
|
@@ -26,7 +26,7 @@ jobs:
|
|
|
26
26
|
- name: Check metadata
|
|
27
27
|
run: pipx run twine check dist/*
|
|
28
28
|
|
|
29
|
-
- uses: actions/upload-artifact@
|
|
29
|
+
- uses: actions/upload-artifact@v4
|
|
30
30
|
with:
|
|
31
31
|
path: dist/*
|
|
32
32
|
|
|
@@ -44,7 +44,7 @@ jobs:
|
|
|
44
44
|
steps:
|
|
45
45
|
- uses: actions/setup-python@v4
|
|
46
46
|
|
|
47
|
-
- uses: actions/download-artifact@
|
|
47
|
+
- uses: actions/download-artifact@v4
|
|
48
48
|
with:
|
|
49
49
|
name: artifact
|
|
50
50
|
path: dist
|
|
@@ -1,14 +1,22 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: restage
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Author-email: Gregory Tucker <gregory.tucker@ess.eu>
|
|
5
|
+
License: BSD-3-Clause
|
|
5
6
|
Classifier: License :: OSI Approved :: BSD License
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Requires-Python: >=3.9
|
|
6
15
|
Description-Content-Type: text/markdown
|
|
7
16
|
Requires-Dist: zenlog>=1.1
|
|
8
17
|
Requires-Dist: platformdirs>=3.11
|
|
9
18
|
Requires-Dist: psutil>=5.9.6
|
|
10
|
-
Requires-Dist:
|
|
11
|
-
Requires-Dist: mccode-antlr[hdf5]==0.7.0
|
|
19
|
+
Requires-Dist: mccode-antlr[hdf5]>=0.10.2
|
|
12
20
|
Provides-Extra: test
|
|
13
21
|
Requires-Dist: pytest; extra == "test"
|
|
14
22
|
Requires-Dist: chopcal; extra == "test"
|
|
@@ -5,18 +5,26 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "restage"
|
|
7
7
|
dependencies = [
|
|
8
|
-
'zenlog>=1.1
|
|
8
|
+
'zenlog>=1.1',
|
|
9
9
|
'platformdirs>=3.11',
|
|
10
10
|
'psutil>=5.9.6',
|
|
11
|
-
|
|
12
|
-
'mccode-antlr[hdf5]==0.7.0',
|
|
11
|
+
'mccode-antlr[hdf5]>=0.10.2',
|
|
13
12
|
]
|
|
14
13
|
readme = "README.md"
|
|
14
|
+
license = {text = "BSD-3-Clause"}
|
|
15
|
+
requires-python = ">=3.9"
|
|
15
16
|
authors = [
|
|
16
17
|
{ name = "Gregory Tucker", email = "gregory.tucker@ess.eu" },
|
|
17
18
|
]
|
|
18
19
|
classifiers = [
|
|
19
20
|
"License :: OSI Approved :: BSD License",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.9",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
20
28
|
]
|
|
21
29
|
dynamic = ["version"]
|
|
22
30
|
|
|
@@ -26,6 +34,7 @@ test = ["pytest", "chopcal"]
|
|
|
26
34
|
[project.scripts]
|
|
27
35
|
restage = "restage.run:entrypoint"
|
|
28
36
|
splitrun = "restage.splitrun:entrypoint"
|
|
37
|
+
restage_bifrost_choppers = "restage.bifrost_choppers:script"
|
|
29
38
|
|
|
30
39
|
[tool.setuptools_scm]
|
|
31
40
|
|
|
@@ -34,10 +43,9 @@ legacy_tox_ini = """
|
|
|
34
43
|
[tox]
|
|
35
44
|
min_version = 4.0
|
|
36
45
|
env_list =
|
|
46
|
+
py313
|
|
37
47
|
py312
|
|
38
48
|
py311
|
|
39
|
-
py310
|
|
40
|
-
py39
|
|
41
49
|
type
|
|
42
50
|
|
|
43
51
|
[testenv]
|
|
@@ -28,7 +28,7 @@ def bifrost_translate_energy_to_chopper_parameters(parameters: dict):
|
|
|
28
28
|
name = ''.join(name)
|
|
29
29
|
if name not in parameters:
|
|
30
30
|
parameters[name] = 0
|
|
31
|
-
order = get_and_remove(parameters, 'order',
|
|
31
|
+
order = get_and_remove(parameters, 'order', 14)
|
|
32
32
|
time = get_and_remove(parameters, 'time', get_and_remove(parameters, 't', 170/180/(2 * 15 * 14)))
|
|
33
33
|
return one_generic_energy_to_chopper_parameters(calculate, time, order, parameters)
|
|
34
34
|
|
|
@@ -1,25 +1,41 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Utilities for interfacing with mccode_antlr.instr.Instr objects
|
|
3
3
|
"""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
4
6
|
from pathlib import Path
|
|
5
7
|
from typing import Union
|
|
6
8
|
from mccode_antlr.instr import Instr
|
|
9
|
+
from mccode_antlr.reader import Registry
|
|
7
10
|
|
|
8
11
|
|
|
9
|
-
def load_instr(filepath: Union[str, Path]) -> Instr:
|
|
12
|
+
def load_instr(filepath: Union[str, Path], extra_registries: list[Registry] | None = None) -> Instr:
|
|
10
13
|
"""Loads an Instr object from a .instr file or a HDF5 file"""
|
|
11
|
-
from mccode_antlr.loader import load_mcstas_instr
|
|
12
14
|
from mccode_antlr.io import load_hdf5
|
|
15
|
+
from mccode_antlr.loader import load_mcstas_instr
|
|
13
16
|
|
|
14
17
|
if not isinstance(filepath, Path):
|
|
15
18
|
filepath = Path(filepath)
|
|
16
19
|
if not filepath.exists() or not filepath.is_file():
|
|
17
20
|
raise ValueError('The provided filepath does not exist or is not a file')
|
|
18
21
|
|
|
22
|
+
# FIXME this hack should be removed ASAP
|
|
23
|
+
if extra_registries is None:
|
|
24
|
+
from mccode_antlr.reader import GitHubRegistry
|
|
25
|
+
mcpl_input_once_registry = GitHubRegistry(
|
|
26
|
+
name='mcpl_input_once',
|
|
27
|
+
url='https://github.com/g5t/mccode-mcpl-input-once',
|
|
28
|
+
version='main',
|
|
29
|
+
filename='pooch-registry.txt'
|
|
30
|
+
)
|
|
31
|
+
extra_registries = [mcpl_input_once_registry]
|
|
32
|
+
|
|
19
33
|
if filepath.suffix == '.instr':
|
|
20
|
-
return load_mcstas_instr(filepath)
|
|
34
|
+
return load_mcstas_instr(filepath, registries=extra_registries)
|
|
21
35
|
|
|
22
|
-
|
|
36
|
+
instr = load_hdf5(filepath)
|
|
37
|
+
instr.registries += tuple(extra_registries)
|
|
38
|
+
return instr
|
|
23
39
|
|
|
24
40
|
|
|
25
41
|
def collect_parameter_dict(instr: Instr, kwargs: dict, strict: bool = True) -> dict:
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Union
|
|
4
3
|
from pathlib import Path
|
|
5
|
-
from .range import Singular, MRange
|
|
6
4
|
from .tables import SimulationEntry, InstrEntry
|
|
7
|
-
from mccode_antlr.compiler.c import CBinaryTarget
|
|
8
5
|
|
|
6
|
+
def mcpl_parameter_split(s: str) -> tuple[str, str]:
|
|
7
|
+
k, v = s.split(':', 1)
|
|
8
|
+
return k, v
|
|
9
9
|
|
|
10
10
|
def make_splitrun_parser():
|
|
11
11
|
from argparse import ArgumentParser
|
|
@@ -40,6 +40,12 @@ def make_splitrun_parser():
|
|
|
40
40
|
# splitrun controlling parameters
|
|
41
41
|
aa('--split-at', nargs=1, type=str, default=['mcpl_split'],
|
|
42
42
|
help='Component at which to split -- DEFAULT: mcpl_split')
|
|
43
|
+
aa('--mcpl-output-component', nargs=1, type=str, default=None,
|
|
44
|
+
help='Inserted MCPL file producing component, MCPL_output(.comp) if not provided')
|
|
45
|
+
aa('--mcpl-input-component', nargs=1, type=str, default=None,
|
|
46
|
+
help='Inserted MCPL file consuming component, MCPL_input(.comp) if not provided')
|
|
47
|
+
aa('--mcpl-input-parameters', nargs='+', type=mcpl_parameter_split, metavar='key:value')
|
|
48
|
+
aa('--mcpl-output-parameters', nargs='+', type=mcpl_parameter_split, metavar='key:value')
|
|
43
49
|
aa('-P', action='append', default=[], help='Cache parameter matching precision')
|
|
44
50
|
|
|
45
51
|
# Other McCode runtime arguments exist, but are likely not used during a scan:
|
|
@@ -78,31 +84,6 @@ def regular_mccode_runtime_dict(args: dict) -> dict:
|
|
|
78
84
|
return t
|
|
79
85
|
|
|
80
86
|
|
|
81
|
-
def mccode_runtime_dict_to_args_list(args: dict) -> list[str]:
|
|
82
|
-
"""Convert a dictionary of McCode runtime arguments to a string.
|
|
83
|
-
|
|
84
|
-
:parameter args: A dictionary of McCode runtime arguments.
|
|
85
|
-
:return: A list of arguments suitable for use in a command line call to a McCode compiled instrument.
|
|
86
|
-
"""
|
|
87
|
-
# convert to a standardized string:
|
|
88
|
-
out = []
|
|
89
|
-
if 'seed' in args and args['seed'] is not None:
|
|
90
|
-
out.append(f'--seed={args["seed"]}')
|
|
91
|
-
if 'ncount' in args and args['ncount'] is not None:
|
|
92
|
-
out.append(f'--ncount={args["ncount"]}')
|
|
93
|
-
if 'dir' in args and args['dir'] is not None:
|
|
94
|
-
out.append(f'--dir={args["dir"]}')
|
|
95
|
-
if 'trace' in args and args['trace']:
|
|
96
|
-
out.append('--trace')
|
|
97
|
-
if 'gravitation' in args and args['gravitation']:
|
|
98
|
-
out.append('--gravitation')
|
|
99
|
-
if 'bufsiz' in args and args['bufsiz'] is not None:
|
|
100
|
-
out.append(f'--bufsiz={args["bufsiz"]}')
|
|
101
|
-
if 'format' in args and args['format'] is not None:
|
|
102
|
-
out.append(f'--format={args["format"]}')
|
|
103
|
-
return out
|
|
104
|
-
|
|
105
|
-
|
|
106
87
|
def parse_splitrun_precision(unparsed: list[str]) -> dict[str, float]:
|
|
107
88
|
precision = {}
|
|
108
89
|
for p in unparsed:
|
|
@@ -113,43 +94,34 @@ def parse_splitrun_precision(unparsed: list[str]) -> dict[str, float]:
|
|
|
113
94
|
return precision
|
|
114
95
|
|
|
115
96
|
|
|
116
|
-
def
|
|
117
|
-
"""Take the list of arguments and sort them into the correct order for splitrun"""
|
|
118
|
-
# TODO this is a bit of a hack, but it works for now
|
|
119
|
-
first, last = [], []
|
|
120
|
-
k = 0
|
|
121
|
-
while k < len(args):
|
|
122
|
-
if args[k].startswith('-'):
|
|
123
|
-
first.append(args[k])
|
|
124
|
-
k += 1
|
|
125
|
-
if '=' not in first[-1] and k < len(args) and not args[k].startswith('-') and '=' not in args[k]:
|
|
126
|
-
first.append(args[k])
|
|
127
|
-
k += 1
|
|
128
|
-
else:
|
|
129
|
-
last.append(args[k])
|
|
130
|
-
k += 1
|
|
131
|
-
return first + last
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def parse_splitrun():
|
|
97
|
+
def parse_splitrun(parser):
|
|
135
98
|
from .range import parse_scan_parameters
|
|
99
|
+
from mccode_antlr.run.runner import sort_args
|
|
136
100
|
import sys
|
|
137
101
|
sys.argv[1:] = sort_args(sys.argv[1:])
|
|
138
102
|
|
|
139
|
-
args =
|
|
103
|
+
args = parser.parse_args()
|
|
104
|
+
if args.mcpl_input_parameters is not None:
|
|
105
|
+
args.mcpl_input_parameters = dict(args.mcpl_input_parameters)
|
|
106
|
+
if args.mcpl_output_parameters is not None:
|
|
107
|
+
args.mcpl_output_parameters = dict(args.mcpl_output_parameters)
|
|
140
108
|
parameters = parse_scan_parameters(args.parameters)
|
|
141
109
|
precision = parse_splitrun_precision(args.P)
|
|
142
110
|
return args, parameters, precision
|
|
143
111
|
|
|
144
112
|
|
|
145
113
|
def entrypoint():
|
|
146
|
-
args, parameters, precision = parse_splitrun()
|
|
114
|
+
args, parameters, precision = parse_splitrun(make_splitrun_parser())
|
|
147
115
|
splitrun_from_file(args, parameters, precision)
|
|
148
116
|
|
|
149
117
|
|
|
150
118
|
def splitrun_from_file(args, parameters, precision):
|
|
151
119
|
from .instr import load_instr
|
|
152
120
|
instr = load_instr(args.instrument[0])
|
|
121
|
+
splitrun_args(instr, parameters, precision, args)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def splitrun_args(instr, parameters, precision, args, **kwargs):
|
|
153
125
|
splitrun(instr, parameters, precision, split_at=args.split_at[0], grid=args.mesh,
|
|
154
126
|
seed=args.seed[0] if args.seed is not None else None,
|
|
155
127
|
ncount=args.ncount[0] if args.ncount is not None else None,
|
|
@@ -164,6 +136,11 @@ def splitrun_from_file(args, parameters, precision):
|
|
|
164
136
|
parallel=args.parallel,
|
|
165
137
|
gpu=args.gpu,
|
|
166
138
|
process_count=args.process_count,
|
|
139
|
+
mcpl_output_component=args.mcpl_output_component[0] if args.mcpl_output_component is not None else None,
|
|
140
|
+
mcpl_output_parameters=args.mcpl_output_parameters,
|
|
141
|
+
mcpl_input_component=args.mcpl_input_component[0] if args.mcpl_input_component is not None else None,
|
|
142
|
+
mcpl_input_parameters=args.mcpl_input_parameters,
|
|
143
|
+
**kwargs
|
|
167
144
|
)
|
|
168
145
|
|
|
169
146
|
|
|
@@ -174,8 +151,11 @@ def splitrun(instr, parameters, precision: dict[str, float], split_at=None, grid
|
|
|
174
151
|
parallel=False, gpu=False, process_count=0,
|
|
175
152
|
callback=None, callback_arguments: dict[str, str] | None = None,
|
|
176
153
|
output_split_instrs=True,
|
|
154
|
+
mcpl_output_component=None, mcpl_output_parameters: dict[str, str] | None = None,
|
|
155
|
+
mcpl_input_component=None, mcpl_input_parameters: dict[str, str] | None = None,
|
|
177
156
|
**runtime_arguments):
|
|
178
157
|
from zenlog import log
|
|
158
|
+
from mccode_antlr.common import ComponentParameter, Expr
|
|
179
159
|
from .energy import get_energy_parameter_names
|
|
180
160
|
if split_at is None:
|
|
181
161
|
split_at = 'mcpl_split'
|
|
@@ -183,7 +163,17 @@ def splitrun(instr, parameters, precision: dict[str, float], split_at=None, grid
|
|
|
183
163
|
if not instr.has_component_named(split_at):
|
|
184
164
|
log.error(f'The specified split-at component, {split_at}, does not exist in the instrument file')
|
|
185
165
|
# splitting defines an instrument parameter in both returned instrument, 'mcpl_filename'.
|
|
186
|
-
|
|
166
|
+
if mcpl_output_parameters is not None:
|
|
167
|
+
mcpl_output_parameters = tuple(ComponentParameter(k, Expr.parse(v)) for k, v in mcpl_output_parameters.items())
|
|
168
|
+
if mcpl_input_parameters is not None:
|
|
169
|
+
mcpl_input_parameters = tuple(ComponentParameter(k, Expr.parse(v)) for k, v in mcpl_input_parameters.items())
|
|
170
|
+
pre, post = instr.mcpl_split(split_at,
|
|
171
|
+
output_component=mcpl_output_component,
|
|
172
|
+
output_parameters=mcpl_output_parameters,
|
|
173
|
+
input_component=mcpl_input_component,
|
|
174
|
+
input_parameters=mcpl_input_parameters,
|
|
175
|
+
remove_unused_parameters=True
|
|
176
|
+
)
|
|
187
177
|
if output_split_instrs:
|
|
188
178
|
for p in (pre, post):
|
|
189
179
|
with open(f'{p.name}.instr', 'w') as f:
|
|
@@ -204,7 +194,8 @@ def splitrun(instr, parameters, precision: dict[str, float], split_at=None, grid
|
|
|
204
194
|
dry_run=dry_run, parallel=parallel, gpu=gpu, process_count=process_count)
|
|
205
195
|
|
|
206
196
|
splitrun_combined(pre_entry, pre, post, pre_parameters, post_parameters, grid, precision,
|
|
207
|
-
dry_run=dry_run,
|
|
197
|
+
dry_run=dry_run, parallel=parallel, gpu=gpu, process_count=process_count,
|
|
198
|
+
callback=callback, callback_arguments=callback_arguments, **runtime_arguments)
|
|
208
199
|
|
|
209
200
|
|
|
210
201
|
def splitrun_pre(instr, parameters, grid, precision: dict[str, float],
|
|
@@ -259,6 +250,7 @@ def _pre_step(instr, entry, names, precision, translate, kw, min_pc, max_pc, dry
|
|
|
259
250
|
|
|
260
251
|
def splitrun_combined(pre_entry, pre, post, pre_parameters, post_parameters, grid, precision: dict[str, float],
|
|
261
252
|
summary=True, dry_run=False, callback=None, callback_arguments: dict[str, str] | None = None,
|
|
253
|
+
parallel=False, gpu=False, process_count=0,
|
|
262
254
|
**runtime_arguments):
|
|
263
255
|
from pathlib import Path
|
|
264
256
|
from .cache import cache_instr, cache_get_simulation
|
|
@@ -267,7 +259,7 @@ def splitrun_combined(pre_entry, pre, post, pre_parameters, post_parameters, gri
|
|
|
267
259
|
from .instr import collect_parameter_dict
|
|
268
260
|
from .tables import best_simulation_entry_match
|
|
269
261
|
from .emulate import mccode_sim_io, mccode_dat_io, mccode_dat_line
|
|
270
|
-
instr_entry = cache_instr(post, mpi=
|
|
262
|
+
instr_entry = cache_instr(post, mpi=parallel, acc=gpu)
|
|
271
263
|
args = regular_mccode_runtime_dict(runtime_arguments)
|
|
272
264
|
sit_kw = {'seed': args.get('seed'), 'ncount': args.get('ncount'), 'gravitation': args.get('gravitation', False)}
|
|
273
265
|
# recombine the parameters to ensure the 'correct' scan is performed
|
|
@@ -308,7 +300,7 @@ def splitrun_combined(pre_entry, pre, post, pre_parameters, post_parameters, gri
|
|
|
308
300
|
# TODO Use the following line instead of the one after it when McCode is fixed to use zero-padded folder names
|
|
309
301
|
# # runtime_arguments['dir'] = args["dir"].joinpath(str(number).zfill(n_zeros))
|
|
310
302
|
runtime_arguments['dir'] = args['dir'].joinpath(str(number))
|
|
311
|
-
do_secondary_simulation(sim_entry, instr_entry, secondary_pars, runtime_arguments, dry_run=dry_run)
|
|
303
|
+
do_secondary_simulation(sim_entry, instr_entry, secondary_pars, runtime_arguments, dry_run=dry_run, process_count=process_count)
|
|
312
304
|
if summary and not dry_run:
|
|
313
305
|
# the data file has *all* **scanned** parameters recorded for each step:
|
|
314
306
|
detectors, line = mccode_dat_line(runtime_arguments['dir'], {k: v for k,v in zip(names, values)})
|
|
@@ -332,7 +324,8 @@ def splitrun_combined(pre_entry, pre, post, pre_parameters, post_parameters, gri
|
|
|
332
324
|
|
|
333
325
|
|
|
334
326
|
def _args_pars_mcpl(args: dict, params: dict, mcpl_filename) -> str:
|
|
335
|
-
|
|
327
|
+
"""Combine the arguments, parameters, and mcpl filename into a single command-arguments string:"""
|
|
328
|
+
from mccode_antlr.run.runner import mccode_runtime_dict_to_args_list
|
|
336
329
|
first = ' '.join(mccode_runtime_dict_to_args_list(args))
|
|
337
330
|
second = ' '.join([f'{k}={v}' for k, v in params.items()])
|
|
338
331
|
third = f'mcpl_filename={mcpl_filename}'
|
|
@@ -457,7 +450,8 @@ def repeat_simulation_until(count, runner, args: dict, parameters, work_dir: Pat
|
|
|
457
450
|
combine_mccode_sims_in_directories(outputs, work_dir)
|
|
458
451
|
|
|
459
452
|
|
|
460
|
-
def do_secondary_simulation(p_sit: SimulationEntry, entry: InstrEntry, pars: dict, args: dict, dry_run: bool = False
|
|
453
|
+
def do_secondary_simulation(p_sit: SimulationEntry, entry: InstrEntry, pars: dict, args: dict, dry_run: bool = False,
|
|
454
|
+
process_count: int = 0):
|
|
461
455
|
from zenlog import log
|
|
462
456
|
from pathlib import Path
|
|
463
457
|
from shutil import copy
|
|
@@ -475,7 +469,7 @@ def do_secondary_simulation(p_sit: SimulationEntry, entry: InstrEntry, pars: dic
|
|
|
475
469
|
|
|
476
470
|
mcpl_path = mcpl_real_filename(Path(p_sit.output_path).joinpath(mcpl_filename))
|
|
477
471
|
executable = Path(entry.binary_path)
|
|
478
|
-
target = CBinaryTarget(mpi=entry.mpi, acc=entry.acc, count=
|
|
472
|
+
target = CBinaryTarget(mpi=entry.mpi, acc=entry.acc, count=process_count, nexus=False)
|
|
479
473
|
run_compiled_instrument(executable, target, _args_pars_mcpl(args, pars, mcpl_path), capture=False, dry_run=dry_run)
|
|
480
474
|
|
|
481
475
|
if not dry_run:
|
|
@@ -10,8 +10,8 @@ def uuid():
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
def utc_timestamp() -> float:
|
|
13
|
-
from datetime import datetime
|
|
14
|
-
return datetime.
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
return datetime.now(timezone.utc).timestamp()
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
COMMON_COLUMNS = ['seed', 'ncount', 'output_path', 'gravitation', 'creation', 'last_access']
|
|
@@ -1,14 +1,22 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: restage
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Author-email: Gregory Tucker <gregory.tucker@ess.eu>
|
|
5
|
+
License: BSD-3-Clause
|
|
5
6
|
Classifier: License :: OSI Approved :: BSD License
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Requires-Python: >=3.9
|
|
6
15
|
Description-Content-Type: text/markdown
|
|
7
16
|
Requires-Dist: zenlog>=1.1
|
|
8
17
|
Requires-Dist: platformdirs>=3.11
|
|
9
18
|
Requires-Dist: psutil>=5.9.6
|
|
10
|
-
Requires-Dist:
|
|
11
|
-
Requires-Dist: mccode-antlr[hdf5]==0.7.0
|
|
19
|
+
Requires-Dist: mccode-antlr[hdf5]>=0.10.2
|
|
12
20
|
Provides-Extra: test
|
|
13
21
|
Requires-Dist: pytest; extra == "test"
|
|
14
22
|
Requires-Dist: chopcal; extra == "test"
|
|
@@ -23,8 +23,6 @@ src/restage.egg-info/dependency_links.txt
|
|
|
23
23
|
src/restage.egg-info/entry_points.txt
|
|
24
24
|
src/restage.egg-info/requires.txt
|
|
25
25
|
src/restage.egg-info/top_level.txt
|
|
26
|
-
test/splitRunTest_first.c
|
|
27
|
-
test/splitRunTest_second.c
|
|
28
26
|
test/test_cache.py
|
|
29
27
|
test/test_database.py
|
|
30
28
|
test/test_energy.py
|
|
@@ -1,4 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import unittest
|
|
4
|
+
from mccode_antlr.compiler.check import simple_instr_compiles
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def compiled_test(method, compiler: str | None = None):
|
|
8
|
+
if compiler is None:
|
|
9
|
+
compiler = 'cc'
|
|
10
|
+
if simple_instr_compiles(compiler):
|
|
11
|
+
return method
|
|
12
|
+
|
|
13
|
+
@unittest.skip(f"Skipping due to lack of working {compiler}")
|
|
14
|
+
def skipped_method(*args, **kwargs):
|
|
15
|
+
return method(*args, **kwargs)
|
|
16
|
+
|
|
17
|
+
return skipped_method
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def gpu_compiled_test(method):
|
|
21
|
+
return compiled_test(method, 'acc')
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def mpi_compiled_test(method):
|
|
25
|
+
return compiled_test(method, 'mpi/cc')
|
|
2
26
|
|
|
3
27
|
|
|
4
28
|
class SingleTestCase(unittest.TestCase):
|
|
@@ -14,7 +38,7 @@ class SingleTestCase(unittest.TestCase):
|
|
|
14
38
|
self.assertTrue(args.mesh)
|
|
15
39
|
|
|
16
40
|
def test_mixed_parsing(self):
|
|
17
|
-
from
|
|
41
|
+
from mccode_antlr.run.runner import sort_args
|
|
18
42
|
args = self.parser.parse_args(sort_args(['test.instr', '-m', 'a=1', 'b=2', '--split-at=here']))
|
|
19
43
|
self.assertEqual(args.instrument, ['test.instr'])
|
|
20
44
|
self.assertEqual(args.parameters, ['a=1', 'b=2'])
|
|
@@ -80,6 +104,17 @@ class SingleTestCase(unittest.TestCase):
|
|
|
80
104
|
self.assertEqual(values[3], 'blah')
|
|
81
105
|
self.assertEqual(values[4], '/data')
|
|
82
106
|
|
|
107
|
+
def test_mcpl_split_parameters(self):
|
|
108
|
+
args = self.parser.parse_args(['test.instr', 'a=1.0', 'b=2', 'c=3:5', 'd=blah', 'e=/data',
|
|
109
|
+
'--mcpl-input-parameters', 'preload:1', 'v_smear:0.01'])
|
|
110
|
+
self.assertEqual(args.parameters, ['a=1.0', 'b=2', 'c=3:5', 'd=blah', 'e=/data'])
|
|
111
|
+
self.assertEqual(args.mcpl_input_parameters, [('preload', '1'), ('v_smear', '0.01')])
|
|
112
|
+
args = self.parser.parse_args(['new_tst.instr', '--mcpl-output-parameters', 'preload:1',
|
|
113
|
+
'--mcpl-input-component=MCPL_input_once'])
|
|
114
|
+
self.assertEqual(args.parameters, [])
|
|
115
|
+
self.assertEqual(args.mcpl_output_parameters, [('preload', '1')])
|
|
116
|
+
self.assertEqual(args.mcpl_input_component, ['MCPL_input_once'])
|
|
117
|
+
|
|
83
118
|
|
|
84
119
|
class DictWranglingTestCase(unittest.TestCase):
|
|
85
120
|
def test_regularization(self):
|
|
@@ -113,24 +148,21 @@ class SplitRunTestCase(unittest.TestCase):
|
|
|
113
148
|
|
|
114
149
|
def _define_instr(self):
|
|
115
150
|
from math import pi, asin, sqrt
|
|
116
|
-
from
|
|
117
|
-
from mccode_antlr.
|
|
118
|
-
from mccode_antlr.
|
|
119
|
-
from mccode_antlr.reader import Reader, MCSTAS_REGISTRY, LocalRegistry
|
|
120
|
-
from pathlib import Path
|
|
151
|
+
from mccode_antlr.reader import MCSTAS_REGISTRY
|
|
152
|
+
from mccode_antlr.reader import GitHubRegistry
|
|
153
|
+
from mccode_antlr.loader.loader import parse_mccode_instr
|
|
121
154
|
|
|
122
155
|
def parse(contents):
|
|
123
|
-
parser = McInstrParser(CommonTokenStream(McInstrLexer(InputStream(contents))))
|
|
124
156
|
# registries = [LocalRegistry(name='test_files', root=Path(__file__).parent.as_posix()), MCSTAS_REGISTRY]
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
157
|
+
mcpl_input_once_registry = GitHubRegistry(
|
|
158
|
+
name='mcpl_input_once',
|
|
159
|
+
url='https://github.com/g5t/mccode-mcpl-input-once',
|
|
160
|
+
version='main',
|
|
161
|
+
filename='pooch-registry.txt'
|
|
162
|
+
)
|
|
163
|
+
registries = [MCSTAS_REGISTRY, mcpl_input_once_registry]
|
|
164
|
+
return parse_mccode_instr(contents, registries, '<test string>')
|
|
165
|
+
|
|
134
166
|
d_spacing = 3.355 # (002) for Highly-ordered Pyrolytic Graphite
|
|
135
167
|
mean_energy = 5.0
|
|
136
168
|
energy_width = 1.0
|
|
@@ -202,7 +234,8 @@ class SplitRunTestCase(unittest.TestCase):
|
|
|
202
234
|
output.mkdir(parents=True)
|
|
203
235
|
|
|
204
236
|
# run the scan
|
|
205
|
-
splitrun(self.instr, scan, precision={}, split_at='split_at', grid=False, ncount=10_000, dir=output
|
|
237
|
+
splitrun(self.instr, scan, precision={}, split_at='split_at', grid=False, ncount=10_000, dir=output,
|
|
238
|
+
mcpl_input_component='MCPL_input_once')
|
|
206
239
|
|
|
207
240
|
# check the scan directory for output
|
|
208
241
|
for x in self.dir.glob('**/*.dat'):
|
|
@@ -210,14 +243,27 @@ class SplitRunTestCase(unittest.TestCase):
|
|
|
210
243
|
|
|
211
244
|
# It would be nice to check that the produced mccode.sim and mccode.dat files look right.
|
|
212
245
|
|
|
246
|
+
@mpi_compiled_test
|
|
213
247
|
def test_parallel_scan(self):
|
|
248
|
+
"""This test requires mpicc and MCPL shared libraries to work
|
|
249
|
+
|
|
250
|
+
On some systems (Fedora at least) specifying which MPI to use requires using
|
|
251
|
+
the `module` system, e.g.,
|
|
252
|
+
$ module load mpi/openmpi-x86_64
|
|
253
|
+
And for some unexplored reason, MCPL shared libraries are not found in their
|
|
254
|
+
default installed location, /usr/local/lib64/libmcpl.so; so this test must
|
|
255
|
+
be invoked with that location specified, e.g.,
|
|
256
|
+
$ LD_LIBRARY_PATH=/usr/local/lib64 pytest test/test_single.py -k test_parallel_scan
|
|
257
|
+
"""
|
|
214
258
|
from restage.splitrun import splitrun
|
|
215
259
|
from restage.range import parse_scan_parameters
|
|
216
260
|
scan = parse_scan_parameters([f'a1={self.min_a1}:0.5:{self.max_a1}', f'a2={2 * self.min_a1}:{2 * self.max_a1}'])
|
|
217
261
|
output = self.dir.joinpath('test_parallel_scan')
|
|
218
262
|
if not output.exists():
|
|
219
263
|
output.mkdir(parents=True)
|
|
220
|
-
splitrun(self.instr, scan, precision={}, split_at='split_at', grid=False, ncount=100_000, dir=output,
|
|
264
|
+
splitrun(self.instr, scan, precision={}, split_at='split_at', grid=False, ncount=100_000, dir=output,
|
|
265
|
+
parallel=True, process_count=4,
|
|
266
|
+
mcpl_input_component='MCPL_input_once', mcpl_input_parameters={'preload': '1'})
|
|
221
267
|
|
|
222
268
|
# check the scan directory for output
|
|
223
269
|
for x in self.dir.glob('**/*.dat'):
|