optmapper 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {optmapper-0.1.0/src/optmapper.egg-info → optmapper-0.2.0}/PKG-INFO +1 -1
  2. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/_version.py +3 -3
  3. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/cli.py +7 -0
  4. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/config.py +54 -14
  5. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/macro.py +20 -0
  6. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/work.py +69 -37
  7. {optmapper-0.1.0 → optmapper-0.2.0/src/optmapper.egg-info}/PKG-INFO +1 -1
  8. optmapper-0.2.0/src/optmapper.egg-info/scm_version.json +8 -0
  9. {optmapper-0.1.0 → optmapper-0.2.0}/tests/conftest.py +1 -1
  10. {optmapper-0.1.0 → optmapper-0.2.0}/tests/test_cli.py +11 -7
  11. {optmapper-0.1.0 → optmapper-0.2.0}/tests/test_config.py +38 -1
  12. {optmapper-0.1.0 → optmapper-0.2.0}/tests/test_macro.py +22 -0
  13. optmapper-0.1.0/src/optmapper.egg-info/scm_version.json +0 -8
  14. {optmapper-0.1.0 → optmapper-0.2.0}/LICENSE +0 -0
  15. {optmapper-0.1.0 → optmapper-0.2.0}/README.md +0 -0
  16. {optmapper-0.1.0 → optmapper-0.2.0}/pyproject.toml +0 -0
  17. {optmapper-0.1.0 → optmapper-0.2.0}/setup.cfg +0 -0
  18. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/__init__.py +0 -0
  19. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/geometry.py +0 -0
  20. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/log_utils.py +0 -0
  21. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/schedulers.py +0 -0
  22. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/sites/local.yaml +0 -0
  23. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/sites/nersc.yaml +0 -0
  24. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/sites/slurm.yaml +0 -0
  25. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper/templates/optmap.mac +0 -0
  26. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper.egg-info/SOURCES.txt +0 -0
  27. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper.egg-info/dependency_links.txt +0 -0
  28. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper.egg-info/entry_points.txt +0 -0
  29. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper.egg-info/not-zip-safe +0 -0
  30. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper.egg-info/requires.txt +0 -0
  31. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper.egg-info/scm_file_list.json +0 -0
  32. {optmapper-0.1.0 → optmapper-0.2.0}/src/optmapper.egg-info/top_level.txt +0 -0
  33. {optmapper-0.1.0 → optmapper-0.2.0}/tests/optmap_dets.gdml +0 -0
  34. {optmapper-0.1.0 → optmapper-0.2.0}/tests/test_geometry.py +0 -0
  35. {optmapper-0.1.0 → optmapper-0.2.0}/tests/test_work.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: optmapper
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Production of LEGEND optical maps on batch systems
5
5
  Author-email: Luigi Pertoldi <gipert@pm.me>
6
6
  Maintainer: The LEGEND Collaboration
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.1.0'
22
- __version_tuple__ = version_tuple = (0, 1, 0)
21
+ __version__ = version = '0.2.0'
22
+ __version_tuple__ = version_tuple = (0, 2, 0)
23
23
 
24
- __commit_id__ = commit_id = 'g44eb9ebce'
24
+ __commit_id__ = commit_id = 'g4dbc0aba8'
@@ -9,6 +9,7 @@ import shutil
9
9
  import sys
10
10
  from pathlib import Path
11
11
 
12
+ from . import _version
12
13
  from .config import Config, ConfigError
13
14
  from .log_utils import setup_log
14
15
  from .schedulers import Job, LocalScheduler, get_scheduler
@@ -79,6 +80,12 @@ def optmapper_cli(argv: list[str] | None = None) -> None:
79
80
  description="%(prog)s: produce LEGEND optical maps. Prepares the geometry, "
80
81
  "submits the production jobs according to the configuration and exits.",
81
82
  )
83
+ parser.add_argument(
84
+ "--version",
85
+ action="version",
86
+ help="""Print %(prog)s version and exit""",
87
+ version=_version.__version__,
88
+ )
82
89
  parser.add_argument("config", help="production configuration file (YAML or JSON)")
83
90
  parser.add_argument("--verbose", "-v", action="count", default=0, help="increase verbosity")
84
91
  parser.add_argument(
@@ -10,7 +10,7 @@ from collections.abc import Mapping
10
10
  from dataclasses import dataclass, field
11
11
  from importlib import resources
12
12
  from pathlib import Path
13
- from typing import Any
13
+ from typing import Any, Literal
14
14
 
15
15
  import dbetto
16
16
 
@@ -100,22 +100,28 @@ class GeometryConfig:
100
100
  class EmissionConfig:
101
101
  """Energy spectrum of the generated optical photons.
102
102
 
103
- Either `spectrum`, the dotted path of a function with signature
103
+ One of `spectrum`, the dotted path of a function with signature
104
104
  ``f(filename: str, output_macro: bool)`` like
105
- :func:`pygeomoptics.lar.g4gps_lar_emissions_spectrum`, or a `gaussian` with a
106
- `mean` (energy or wavelength) and a `sigma` (energy).
105
+ :func:`pygeomoptics.lar.g4gps_lar_emissions_spectrum`, `g4gps_spectrum_macro`, a
106
+ macro file written by such a function with ``output_macro=True``, or a `gaussian`
107
+ with a `mean` (energy or wavelength) and a `sigma` (energy).
107
108
  """
108
109
 
109
110
  spectrum: str | None = None
111
+ g4gps_spectrum_macro: str | None = None
110
112
  gaussian: dict | None = None
111
113
 
112
114
  @classmethod
113
- def from_dict(cls, d: Mapping) -> EmissionConfig:
114
- _check_keys(d, {"spectrum", "gaussian"}, "emission")
115
+ def from_dict(cls, d: Mapping, base: Path) -> EmissionConfig:
116
+ _check_keys(d, {f.name for f in dataclasses.fields(cls)}, "emission")
115
117
  e = cls(**d)
116
- if (e.spectrum is None) == (e.gaussian is None):
117
- msg = "emission: specify exactly one of 'spectrum' or 'gaussian'"
118
+ n_given = sum(v is not None for v in (e.spectrum, e.g4gps_spectrum_macro, e.gaussian))
119
+ if n_given != 1:
120
+ msg = (
121
+ "emission: specify exactly one of 'spectrum', 'g4gps_spectrum_macro' or 'gaussian'"
122
+ )
118
123
  raise ConfigError(msg)
124
+ e.g4gps_spectrum_macro = _resolve_path(e.g4gps_spectrum_macro, base)
119
125
  if e.gaussian is not None:
120
126
  e.gaussian = dict(e.gaussian)
121
127
  _check_keys(e.gaussian, {"mean", "sigma"}, "emission.gaussian")
@@ -195,17 +201,24 @@ class OptmapConfig:
195
201
 
196
202
  @dataclass
197
203
  class ProcessingConfig:
198
- stp_files_per_map: int = 32
199
204
  parallel_maps: int = 8
200
205
  procs_per_map: int = 16
201
206
  merge_procs: int = 8
202
207
  bufsize: int = 50_000
203
- keep_stp: bool = False
208
+ keep_stp: Literal["output", "scratch"] | None = None
204
209
 
205
210
  @classmethod
206
211
  def from_dict(cls, d: Mapping) -> ProcessingConfig:
207
212
  _check_keys(d, {f.name for f in dataclasses.fields(cls)}, "processing")
208
- return cls(**d)
213
+ p = cls(**d)
214
+ if p.keep_stp is True:
215
+ p.keep_stp = "output"
216
+ elif p.keep_stp is False:
217
+ p.keep_stp = None
218
+ if p.keep_stp not in (None, "output", "scratch"):
219
+ msg = f"processing.keep_stp must be false, true, output or scratch, got {p.keep_stp!r}"
220
+ raise ConfigError(msg)
221
+ return p
209
222
 
210
223
 
211
224
  @dataclass
@@ -243,17 +256,30 @@ class ExecutionConfig:
243
256
  def from_dict(cls, d: Mapping) -> ExecutionConfig:
244
257
  d = dict(d)
245
258
  site = load_site(d.pop("site", "local"))
246
- merged = deep_merge(site, d)
259
+ # expand slurm.all separately, so that any user option overrides the site preset
260
+ merged = deep_merge(_expand_slurm_all(site), _expand_slurm_all(d))
247
261
  _check_keys(merged, {f.name for f in dataclasses.fields(cls)}, "execution")
248
262
  e = cls(**merged)
249
263
  if e.scheduler not in ("slurm", "local"):
250
264
  msg = f"execution.scheduler must be slurm or local, got {e.scheduler!r}"
251
265
  raise ConfigError(msg)
252
- _check_keys(e.slurm, {"node", "merge"}, "execution.slurm")
253
266
  e.env = {k: str(v) for k, v in e.env.items()}
254
267
  return e
255
268
 
256
269
 
270
+ def _expand_slurm_all(d: Mapping) -> dict:
271
+ """Move the ``slurm.all`` options of an execution config to ``slurm.node`` and ``slurm.merge``.
272
+
273
+ Job type specific options take precedence over the ones in ``slurm.all``.
274
+ """
275
+ d = dict(d)
276
+ slurm = d.get("slurm", {})
277
+ _check_keys(slurm, {"all", "node", "merge"}, "execution.slurm")
278
+ if "all" in slurm:
279
+ d["slurm"] = {k: {**slurm["all"], **slurm.get(k, {})} for k in ("node", "merge")}
280
+ return d
281
+
282
+
257
283
  @dataclass
258
284
  class Config:
259
285
  name: str
@@ -288,7 +314,7 @@ class Config:
288
314
  name=str(d["name"]),
289
315
  output_dir=str(_resolve_path(d["output_dir"], base)),
290
316
  geometry=GeometryConfig.from_dict(d["geometry"], base),
291
- emission=EmissionConfig.from_dict(d["emission"]),
317
+ emission=EmissionConfig.from_dict(d["emission"], base),
292
318
  optmap=OptmapConfig.from_dict(d["optmap"]),
293
319
  statistics=StatisticsConfig.from_dict(d.get("statistics", {})),
294
320
  processing=ProcessingConfig.from_dict(d.get("processing", {})),
@@ -335,6 +361,20 @@ class Config:
335
361
  def final_map(self) -> Path:
336
362
  return Path(self.output_dir) / f"{self.name}.lh5"
337
363
 
364
+ @property
365
+ def scratch_dir(self) -> Path:
366
+ """Scratch area of this production, expands environment variables of the caller."""
367
+ base = self.execution.scratch_dir or str(Path(self.output_dir) / "scratch")
368
+ return Path(os.path.expandvars(base)) / self.name
369
+
370
+ def stp_dir(self, index: int) -> Path | None:
371
+ """Where the remage output files of a node are kept, ``None`` if they are deleted."""
372
+ if self.processing.keep_stp == "output":
373
+ return Path(self.output_dir) / "stp" / f"node{index:04d}"
374
+ if self.processing.keep_stp == "scratch":
375
+ return self.scratch_dir / "stp" / f"node{index:04d}"
376
+ return None
377
+
338
378
  def node_map(self, index: int) -> Path:
339
379
  """Output map of a single node, the final map if there is only one node."""
340
380
  if self.statistics.nodes == 1:
@@ -72,6 +72,23 @@ def load_spectrum_function(spec: str) -> Callable[[str, bool], None]:
72
72
  return func
73
73
 
74
74
 
75
+ def read_spectrum_macro(path: str) -> list[str]:
76
+ """Read a spectrum macro written by a pygeom-optics ``g4gps_*`` function."""
77
+ try:
78
+ lines = Path(path).read_text().strip().splitlines()
79
+ except OSError as e:
80
+ msg = f"cannot read emission spectrum macro: {e}"
81
+ raise ConfigError(msg) from e
82
+ cmds = [ln.strip() for ln in lines if ln.strip() and not ln.lstrip().startswith("#")]
83
+ if not cmds or not all(c.startswith("/gps/") for c in cmds):
84
+ msg = (
85
+ f"emission spectrum macro '{path}' must only contain /gps/ commands, "
86
+ "was it written with output_macro=True?"
87
+ )
88
+ raise ConfigError(msg)
89
+ return lines
90
+
91
+
75
92
  @contextmanager
76
93
  def _optics_plugin(plugin: str | None) -> Iterator[None]:
77
94
  """Apply a pygeom-optics plugin, restoring the original material properties afterwards."""
@@ -107,6 +124,9 @@ def energy_commands(emission: EmissionConfig, optics_plugin: str | None = None)
107
124
  f"/gps/ene/sigma {sigma.m:.6g} eV",
108
125
  ]
109
126
 
127
+ if emission.g4gps_spectrum_macro is not None:
128
+ return read_spectrum_macro(emission.g4gps_spectrum_macro)
129
+
110
130
  assert emission.spectrum is not None
111
131
  func = load_spectrum_function(emission.spectrum)
112
132
  with _optics_plugin(optics_plugin), tempfile.TemporaryDirectory() as tmpdir:
@@ -9,6 +9,7 @@ import re
9
9
  import resource
10
10
  import shlex
11
11
  import shutil
12
+ import signal
12
13
  import subprocess
13
14
  import time
14
15
  from concurrent.futures import ThreadPoolExecutor
@@ -43,8 +44,13 @@ def _available_cpus() -> int:
43
44
 
44
45
  def merge_maps(cfg: Config, inputs: list[Path], output: Path, workdir: Path) -> None:
45
46
  """Merge optical maps into `output`, temporary files are written to `workdir`."""
47
+ from reboost.log_utils import setup_log as reboost_setup_log # noqa: PLC0415
46
48
  from reboost.optmap.create import merge_optical_maps # noqa: PLC0415
47
49
 
50
+ # the merge runs in this process, so log like ``reboost-optmap --verbose`` would
51
+ if not logging.getLogger("reboost").handlers:
52
+ reboost_setup_log(logging.INFO)
53
+
48
54
  output.parent.mkdir(parents=True, exist_ok=True)
49
55
  tmp = workdir / f".{output.name}.part"
50
56
  tmp.unlink(missing_ok=True)
@@ -65,7 +71,8 @@ def _reboost_create_args(cfg: Config, workdir: Path) -> list[str]:
65
71
  settings = workdir / "optmap-settings.json"
66
72
  dbetto.utils.write_dict(cfg.optmap.reboost_settings(), str(settings))
67
73
  args = [
68
- *("--bufsize", str(cfg.processing.bufsize), "create"),
74
+ # log the progress, to see how far it got if it is killed
75
+ *("--verbose", "--bufsize", str(cfg.processing.bufsize), "create"),
69
76
  *("--settings", str(settings), "--geom", str(cfg.gdml_file)),
70
77
  *("--n-procs", str(cfg.processing.procs_per_map)),
71
78
  ]
@@ -76,12 +83,39 @@ def _reboost_create_args(cfg: Config, workdir: Path) -> list[str]:
76
83
  return args
77
84
 
78
85
 
86
+ def _exit_status(returncode: int) -> str:
87
+ """Describe the exit code of a `subprocess`, which is negative if killed by a signal."""
88
+ if returncode >= 0:
89
+ return f"exit code {returncode}"
90
+ try:
91
+ name = signal.Signals(-returncode).name
92
+ except ValueError:
93
+ name = f"signal {-returncode}"
94
+ msg = f"killed by {name}"
95
+ if -returncode == signal.SIGKILL:
96
+ msg += " (out of memory?)"
97
+ return msg
98
+
99
+
79
100
  def _create_map(create_args: list[str], inputs: list[Path], output: Path) -> None:
80
- """Create a map with ``reboost-optmap``, in a separate process as it spawns its own pool."""
101
+ """Create a map with ``reboost-optmap``, in a separate process as it spawns its own pool.
102
+
103
+ The output is forwarded line by line to the log as it is written, so that it is not lost
104
+ if a process is killed (e.g. out of memory) before the map creation finishes.
105
+ """
81
106
  cmd = ["reboost-optmap", *create_args, "--", *map(str, inputs), str(output)]
82
- res = subprocess.run(cmd, capture_output=True, text=True, check=False)
83
- if res.returncode != 0:
84
- msg = f"map creation failed for {output.name}:\n{res.stdout}{res.stderr}"
107
+ # by default, Python block-buffers output to a pipe, which would be lost on SIGKILL
108
+ env = {**os.environ, "PYTHONUNBUFFERED": "1"}
109
+ with subprocess.Popen(
110
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env
111
+ ) as proc:
112
+ for line in proc.stdout: # type: ignore[union-attr]
113
+ log.info("%s: %s", output.stem, line.rstrip())
114
+ if proc.returncode != 0:
115
+ msg = (
116
+ f"map creation failed for {output.name}: {_exit_status(proc.returncode)}, "
117
+ "see its output above"
118
+ )
85
119
  raise RuntimeError(msg)
86
120
  log.info("created %s from %d stp files", output.name, len(inputs))
87
121
 
@@ -91,10 +125,7 @@ def run_node(cfg: Config, index: int) -> None:
91
125
  from .macro import render_macro # noqa: PLC0415
92
126
 
93
127
  job_id = os.environ.get("SLURM_JOB_ID", str(os.getpid()))
94
- scratch_base = os.path.expandvars(
95
- cfg.execution.scratch_dir or str(Path(cfg.output_dir) / "scratch")
96
- )
97
- scratch = Path(scratch_base) / cfg.name / f"node{index:04d}-{job_id}"
128
+ scratch = cfg.scratch_dir / f"node{index:04d}-{job_id}"
98
129
  # a requeued Slurm job keeps its ID, do not pick up files from the previous attempt
99
130
  shutil.rmtree(scratch, ignore_errors=True)
100
131
  scratch.mkdir(parents=True)
@@ -122,10 +153,9 @@ def run_node(cfg: Config, index: int) -> None:
122
153
  map_dir.mkdir()
123
154
  create_args = _reboost_create_args(cfg, scratch)
124
155
 
156
+ stp_dir = scratch / "stp"
157
+ stp_dir.mkdir()
125
158
  for run in range(cfg.statistics.runs_per_node):
126
- stp_dir = scratch / "stp" / f"run{run:03d}"
127
- stp_dir.mkdir(parents=True)
128
-
129
159
  cmd = [
130
160
  "remage",
131
161
  *cfg.advanced.remage_args,
@@ -143,33 +173,35 @@ def run_node(cfg: Config, index: int) -> None:
143
173
  log.debug("running: %s", shlex.join(cmd))
144
174
  subprocess.run(cmd, check=True)
145
175
 
146
- stp_files = sorted(stp_dir.glob("*.lh5"), key=_natural_key)
147
- if not stp_files:
148
- msg = f"remage did not produce any output file in {stp_dir}"
149
- raise RuntimeError(msg)
176
+ stp_files = sorted(stp_dir.glob("*.lh5"), key=_natural_key)
177
+ if not stp_files:
178
+ msg = f"remage did not produce any output file in {stp_dir}"
179
+ raise RuntimeError(msg)
150
180
 
151
- chunks = [
152
- stp_files[i : i + proc.stp_files_per_map]
153
- for i in range(0, len(stp_files), proc.stp_files_per_map)
181
+ # split the stp files into parallel_maps contiguous chunks of about equal size
182
+ n_maps = min(proc.parallel_maps, len(stp_files))
183
+ chunks = [
184
+ stp_files[k * len(stp_files) // n_maps : (k + 1) * len(stp_files) // n_maps]
185
+ for k in range(n_maps)
186
+ ]
187
+ with (
188
+ _step(f"creating {len(chunks)} map(s) from {len(stp_files)} stp files"),
189
+ ThreadPoolExecutor(n_maps) as pool,
190
+ ):
191
+ futures = [
192
+ pool.submit(_create_map, create_args, chunk, map_dir / f"{k:04d}.lh5")
193
+ for k, chunk in enumerate(chunks)
154
194
  ]
155
- with (
156
- _step(f"run {run}: creating {len(chunks)} map(s) from {len(stp_files)} stp files"),
157
- ThreadPoolExecutor(proc.parallel_maps) as pool,
158
- ):
159
- futures = [
160
- pool.submit(_create_map, create_args, chunk, map_dir / f"run{run:03d}-{k:04d}.lh5")
161
- for k, chunk in enumerate(chunks)
162
- ]
163
- for f in futures:
164
- f.result()
165
-
166
- if proc.keep_stp:
167
- dest = Path(cfg.output_dir) / "stp" / f"node{index:04d}"
168
- with _step(f"run {run}: copying stp files to {dest}"):
169
- dest.mkdir(parents=True, exist_ok=True)
170
- for stp_file in stp_files:
171
- shutil.copy2(stp_file, dest)
172
- shutil.rmtree(stp_dir)
195
+ for f in futures:
196
+ f.result()
197
+
198
+ dest = cfg.stp_dir(index)
199
+ if dest is not None:
200
+ # moving is cheap if the destination is on the scratch file system
201
+ with _step(f"moving stp files to {dest}"):
202
+ dest.mkdir(parents=True, exist_ok=True)
203
+ for stp_file in stp_files:
204
+ shutil.move(stp_file, dest / stp_file.name)
173
205
 
174
206
  output = cfg.node_map(index)
175
207
  with _step(f"merging intermediate maps into {output}"):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: optmapper
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Production of LEGEND optical maps on batch systems
5
5
  Author-email: Luigi Pertoldi <gipert@pm.me>
6
6
  Maintainer: The LEGEND Collaboration
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "0.2.0",
3
+ "distance": 0,
4
+ "node": "g4dbc0aba8c0cff3ffa6fdab00c66575cd0bf6e2d",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-10-21"
8
+ }
@@ -107,6 +107,6 @@ def base_config(tmp_path, gdml_file):
107
107
  "emission": {"gaussian": {"mean": "128 nm", "sigma": "0.22 eV"}},
108
108
  "optmap": {"range_in_m": [[-1, 1], [-1, 1], [-1, 1]], "bins": [4, 4, 4]},
109
109
  "statistics": {"nodes": 1, "runs_per_node": 2, "processes_per_node": 3},
110
- "processing": {"stp_files_per_map": 2, "parallel_maps": 2, "procs_per_map": 1},
110
+ "processing": {"parallel_maps": 2, "procs_per_map": 1},
111
111
  "execution": {"site": "local"},
112
112
  }
@@ -21,7 +21,7 @@ def test_slurm_submission(tmp_path, base_config, fake_bin, monkeypatch):
21
21
  base_config["statistics"]["nodes"] = 3
22
22
  base_config["execution"] = {
23
23
  "site": "nersc",
24
- "slurm": {"node": {"mail-user": "me@example.com", "exclusive": True}},
24
+ "slurm": {"all": {"mail-user": "me@example.com"}, "node": {"exclusive": True}},
25
25
  }
26
26
  optmapper_cli([_write(tmp_path, base_config)])
27
27
 
@@ -49,6 +49,8 @@ def test_slurm_submission(tmp_path, base_config, fake_bin, monkeypatch):
49
49
  merge = calls[3]
50
50
  assert "--dependency=afterany:1000:1001:1002" in merge
51
51
  assert "--qos=shared" in merge
52
+ assert "--mail-user=me@example.com" in merge
53
+ assert "--exclusive" not in merge
52
54
  assert shlex.split(merge[-1])[-3:] == ["merge", "--config", str(resolved)]
53
55
 
54
56
  # only the resolved config and the log directory, no job scripts
@@ -98,9 +100,10 @@ def test_launcher(base_config, monkeypatch):
98
100
 
99
101
  @pytest.mark.usefixtures("fake_bin")
100
102
  @pytest.mark.parametrize("nodes", [1, 2])
101
- def test_local_production(tmp_path, base_config, nodes):
103
+ @pytest.mark.parametrize("keep_stp", [True, "scratch"])
104
+ def test_local_production(tmp_path, base_config, nodes, keep_stp):
102
105
  base_config["statistics"]["nodes"] = nodes
103
- base_config["processing"]["keep_stp"] = True
106
+ base_config["processing"]["keep_stp"] = keep_stp
104
107
  optmapper_cli([_write(tmp_path, base_config)])
105
108
 
106
109
  out = tmp_path / "out"
@@ -114,10 +117,11 @@ def test_local_production(tmp_path, base_config, nodes):
114
117
  ]
115
118
  assert (out / "test-map.mac").is_file()
116
119
  # 2 runs x 3 processes per node
117
- assert len(list((out / "stp").glob("node*/*.lh5"))) == 6 * nodes
118
- assert not (out / "scratch" / "test-map").exists() or not any(
119
- (out / "scratch" / "test-map").iterdir()
120
- )
120
+ scratch = out / "scratch" / "test-map"
121
+ stp = out / "stp" if keep_stp is True else scratch / "stp"
122
+ assert len(list(stp.glob("node*/*.lh5"))) == 6 * nodes
123
+ # only the kept stp files are left in the scratch area
124
+ assert {p.name for p in scratch.iterdir()} <= {"stp"}
121
125
  if nodes > 1:
122
126
  assert len(list((out / "nodes").glob("*.lh5"))) == nodes
123
127
  assert len(list((out / "logs").glob("test-map-merge-*.log"))) == 1
@@ -20,7 +20,7 @@ geometry:
20
20
  LEGEND_METADATA: $_/legend-metadata
21
21
  LEGEND1000_METADATA: $HOME/legend1000-metadata
22
22
  emission:
23
- spectrum: pygeomoptics.lar.g4gps_lar_emissions_spectrum
23
+ g4gps_spectrum_macro: lar-spectrum.mac
24
24
  optmap:
25
25
  range_in_m: [[-2, 2], [-2, 2], [-3, 3]]
26
26
  bins: [80, 80, 120]
@@ -32,6 +32,7 @@ statistics:
32
32
  assert cfg.output_dir == str(tmp_path / "prod" / "out")
33
33
  assert cfg.geometry.config == str(tmp_path / "prod" / "geom-config.yaml")
34
34
  assert cfg.geometry.optics_plugin == str(tmp_path / "prod" / "plugin.py")
35
+ assert cfg.emission.g4gps_spectrum_macro == str(tmp_path / "prod" / "lar-spectrum.mac")
35
36
  assert cfg.geometry.env["LEGEND_METADATA"] == str(tmp_path / "prod" / "legend-metadata")
36
37
  assert not cfg.geometry.env["LEGEND1000_METADATA"].startswith("$")
37
38
  assert cfg.gdml_file == tmp_path / "prod" / "out" / "geom.gdml"
@@ -73,6 +74,36 @@ def test_site_presets(base_config):
73
74
  load_site("nonexistent-site")
74
75
 
75
76
 
77
+ def test_slurm_all(tmp_path, base_config):
78
+ site = tmp_path / "mysite.yaml"
79
+ site.write_text(
80
+ "scheduler: slurm\n"
81
+ "slurm:\n"
82
+ " all: {account: acc, mail-type: END}\n"
83
+ " node: {time: '04:00:00', mail-type: FAIL}\n"
84
+ " merge: {time: '01:00:00'}\n"
85
+ )
86
+ base_config["execution"] = {
87
+ "site": str(site),
88
+ "slurm": {"all": {"time": "02:00:00", "qos": "debug"}, "merge": {"qos": None}},
89
+ }
90
+ slurm = Config.from_dict(base_config).execution.slurm
91
+ assert set(slurm) == {"node", "merge"}
92
+ # user options, also from slurm.all, override the site preset
93
+ assert slurm["node"] == {
94
+ "account": "acc",
95
+ "mail-type": "FAIL",
96
+ "time": "02:00:00",
97
+ "qos": "debug",
98
+ }
99
+ assert slurm["merge"] == {"account": "acc", "mail-type": "END", "time": "02:00:00", "qos": None}
100
+
101
+ base_config["execution"] = {"site": "nersc", "slurm": {"all": {"time": "08:00:00"}}}
102
+ slurm = Config.from_dict(base_config).execution.slurm
103
+ assert slurm["node"]["time"] == slurm["merge"]["time"] == "08:00:00"
104
+ assert slurm["merge"]["qos"] == "shared"
105
+
106
+
76
107
  def test_site_preset_from_file(tmp_path, base_config):
77
108
  site = tmp_path / "mysite.yaml"
78
109
  site.write_text("scheduler: slurm\nslurm:\n node:\n partition: long\n")
@@ -96,6 +127,10 @@ def test_deep_merge():
96
127
  ({"geometry": {"gdml": "a.gdml", "env": {"A": "b"}}}, "require 'executable'"),
97
128
  ({"emission": {"gaussian": {"mean": 1}}}, "both"),
98
129
  ({"emission": {}}, "exactly one of"),
130
+ (
131
+ {"emission": {"spectrum": "a.b", "g4gps_spectrum_macro": "s.mac"}},
132
+ "exactly one of",
133
+ ),
99
134
  ({"statistics": {"nodes": 0}}, "positive integer"),
100
135
  ({"optmap": {"range_in_m": [[0, 1], [0, 1]], "bins": [1, 1]}}, "three entries"),
101
136
  ({"optmap": {"range_in_m": [[0, 1], [0, 2], [0, 1]], "bins": [1, 1, 1]}}, "equal x"),
@@ -104,8 +139,10 @@ def test_deep_merge():
104
139
  "shape",
105
140
  ),
106
141
  ({"optmap": {"range_in_m": [[1, 0]] * 3, "bins": [1] * 3}}, "empty range"),
142
+ ({"processing": {"keep_stp": "home"}}, "keep_stp"),
107
143
  ({"execution": {"scheduler": "pbs"}}, "scheduler"),
108
144
  ({"execution": {"slurm": {"nodes": {}}}}, "unknown key"),
145
+ ({"execution": {"site": {"slurm": {"nodes": {}}}}}, "unknown key"),
109
146
  ({"name": "a/b"}, "invalid production name"),
110
147
  ],
111
148
  )
@@ -102,6 +102,28 @@ def test_spectrum_with_optics_plugin(tmp_path, base_config):
102
102
  assert all(a > b for a, b in itertools.pairwise(points))
103
103
 
104
104
 
105
+ def test_spectrum_macro_file(tmp_path, base_config):
106
+ from pygeomoptics.lar import g4gps_lar_emissions_spectrum # noqa: PLC0415
107
+
108
+ spectrum = tmp_path / "lar-spectrum.mac"
109
+ g4gps_lar_emissions_spectrum(str(spectrum), True)
110
+ base_config["emission"] = {"g4gps_spectrum_macro": str(spectrum)}
111
+ cfg = Config.from_dict(base_config)
112
+ assert energy_commands(cfg.emission) == spectrum.read_text().strip().splitlines()
113
+ macro = render_macro(cfg, 1)
114
+ assert macro.index("/gps/particle") < macro.index("/gps/hist/point")
115
+ assert macro.index("/gps/hist/inter") < macro.index("/gps/ang/type")
116
+
117
+ # the plain-text output of g4gps_* is not a macro
118
+ g4gps_lar_emissions_spectrum(str(spectrum), False)
119
+ with pytest.raises(ConfigError, match="output_macro=True"):
120
+ energy_commands(cfg.emission)
121
+
122
+ base_config["emission"] = {"g4gps_spectrum_macro": str(tmp_path / "missing.mac")}
123
+ with pytest.raises(ConfigError, match="cannot read"):
124
+ energy_commands(Config.from_dict(base_config).emission)
125
+
126
+
105
127
  @pytest.mark.parametrize("spec", ["pygeomoptics.lar.nonexistent", "nomodule.func", "os.getcwd"])
106
128
  def test_bad_spectrum(base_config, spec):
107
129
  base_config["emission"] = {"spectrum": spec}
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "0.1.0",
3
- "distance": 0,
4
- "node": "g44eb9ebce1da120934a68fff22e3e819804d7c30",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-10-07"
8
- }
File without changes
File without changes
File without changes
File without changes
File without changes