eclipse 0.13.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. eclipse/__init__.py +95 -0
  2. eclipse/analysis.py +853 -0
  3. eclipse/atmosphere.py +859 -0
  4. eclipse/band.py +477 -0
  5. eclipse/cli.py +138 -0
  6. eclipse/config.py +1710 -0
  7. eclipse/continuum.py +413 -0
  8. eclipse/data/eis_response/EIS_EffArea_A.004 +61 -0
  9. eclipse/data/eis_response/EIS_EffArea_B.004 +60 -0
  10. eclipse/data/eis_response/EIS_EffArea_B.005 +64 -0
  11. eclipse/data/eis_response/README_dz2025 +50 -0
  12. eclipse/data/eis_response/fit_eis_ea_2023-05-04_smooth.sav +0 -0
  13. eclipse/data/eis_response/source.txt +33 -0
  14. eclipse/data/science_cases.yaml +365 -0
  15. eclipse/data/throughput/grating_reflection_efficiency.dat +25 -0
  16. eclipse/data/throughput/primary_mirror_coating_reflectance.dat +25 -0
  17. eclipse/data/throughput/source.txt +6 -0
  18. eclipse/data/throughput/throughput_aluminium_1000_angstrom.dat +503 -0
  19. eclipse/data/throughput/throughput_aluminium_oxide_1000_angstrom.dat +503 -0
  20. eclipse/data/throughput/throughput_carbon_1000_angstrom.dat +503 -0
  21. eclipse/data_processing.py +904 -0
  22. eclipse/deprecated.py +161 -0
  23. eclipse/eis_calibration.py +573 -0
  24. eclipse/extern/__init__.py +0 -0
  25. eclipse/extern/mpfit.py +2388 -0
  26. eclipse/fitting.py +1734 -0
  27. eclipse/frame.py +594 -0
  28. eclipse/full_frame.py +944 -0
  29. eclipse/main.py +1646 -0
  30. eclipse/monte_carlo.py +660 -0
  31. eclipse/pinhole_diffraction.py +409 -0
  32. eclipse/psf.py +6 -0
  33. eclipse/radiometric.py +1328 -0
  34. eclipse/raster.py +869 -0
  35. eclipse/readout.py +1002 -0
  36. eclipse/resolution_budget.py +108 -0
  37. eclipse/results_file.py +1226 -0
  38. eclipse/sampling.py +351 -0
  39. eclipse/science_cases.py +364 -0
  40. eclipse/synthesis.py +2492 -0
  41. eclipse/synthesis_cli.py +12 -0
  42. eclipse/synthesis_file.py +882 -0
  43. eclipse/utils.py +1145 -0
  44. eclipse-0.13.0.dist-info/METADATA +70 -0
  45. eclipse-0.13.0.dist-info/RECORD +50 -0
  46. eclipse-0.13.0.dist-info/WHEEL +5 -0
  47. eclipse-0.13.0.dist-info/entry_points.txt +7 -0
  48. eclipse-0.13.0.dist-info/licenses/LICENSE +3 -0
  49. eclipse-0.13.0.dist-info/top_level.txt +2 -0
  50. euvst_response/__init__.py +129 -0
eclipse/main.py ADDED
@@ -0,0 +1,1646 @@
1
+ """
2
+ Main execution script for instrument response simulations.
3
+ """
4
+
5
+ from __future__ import annotations
6
+ import argparse
7
+ import os
8
+ import sys
9
+ import warnings
10
+ from itertools import product as itertools_product
11
+ from pathlib import Path
12
+ import yaml
13
+ import astropy.units as u
14
+ import gzip
15
+ import h5py
16
+
17
+ from .config import AluminiumFilter, Detector_SWC, Detector_EIS, Telescope_EUVST, Telescope_EIS, Simulation, check_pinhole_lists, _load_throughput_table
18
+ from .config import _FOCAL_PLANE_SETTINGS, _old_names_used, _same_setting
19
+ from .data_processing import (_whole_pixels, load_atmosphere, rebin_atmosphere,
20
+ create_uniform_intensity_cube, pad_spectral_axis, rebin_spectra)
21
+ from .raster import AtmosphereSeries, RasterSynthesiser, SynthesisRaster, SynthesisSeries
22
+ from .atmosphere import _is_pickle
23
+ from .readout import FocalPlane_SWC
24
+ from .synthesis_file import (is_synthesis_file, read_synthesis, read_synthesis_layout,
25
+ read_synthesis_products, synthesis_line_names)
26
+ from .fitting import FitConfig, FitComponent, ground_truth_summary
27
+ from .results_file import _to_json, save_results
28
+ from .monte_carlo import expected_dn_uncertainty, monte_carlo
29
+ from .pinhole_diffraction import _pinholes_on
30
+ from .radiometric import (across_slit_fwhm, spectral_optics_fwhm, spectral_psf_fwhm,
31
+ spectral_psf_margin)
32
+ from .utils import (
33
+ parse_yaml_input, ensure_list, set_debug_mode, debug_break, debug_on_error,
34
+ deduplicate_list, get_git_commit_id, _get_software_version,
35
+ _parse_section, _params_to_key, _extract_config_params, _SECTION_LIST_FIELDS,
36
+ rebin_slit_offchip, check_config_keys, load_yaml_config,
37
+ )
38
+ import dataclasses
39
+ import numpy as np
40
+
41
+
42
+ # Every top-level key main() reads. Any other key is rejected rather than
43
+ # ignored.
44
+ _TOP_LEVEL_KEYS = {
45
+ "instrument", "n_iter", "ncpu",
46
+ "uniform_intensity", "rest_wavelength", "thermal_width",
47
+ "synthesis_file", "reference_line",
48
+ "atmosphere_series", "synthesis_series", "synthesis", "raster",
49
+ "pinhole_sizes", "pinhole_positions", "pinhole_positions_spectral",
50
+ "offchip_bin_slit", "fit_signals",
51
+ "simulation", "detector", "telescope", "filter", "fitting",
52
+ }
53
+
54
+ # Keys of the 'synthesis:' section, the settings an atmosphere time series is
55
+ # synthesised with (as in synthesise-spectra), and of the 'raster:' section,
56
+ # the observing plan of an atmosphere or synthesis time series
57
+ # (eclipse.raster.RasterPlan).
58
+ _SYNTHESIS_KEYS = {"lines", "abundance", "vel_res", "vel_lim", "crop_y", "crop_z",
59
+ "precision", "mass_per_electron", "hdf5_dbase_root", "n_workers",
60
+ "goft_temperature_chunk"}
61
+ _RASTER_KEYS = {"start", "steps", "step", "repeats", "cadence", "centre", "direction"}
62
+
63
+ # Under MPI, a copy of a non-root rank's original stderr, taken before its
64
+ # output is silenced.
65
+ _STARTING_STDERR = None
66
+
67
+
68
+ def _cpu_list(text: str) -> list:
69
+ """CPU numbers in a Linux CPU list such as `0-3,8,10-11`."""
70
+ cpus = []
71
+ for part in text.strip().split(","):
72
+ if part:
73
+ first, _, last = part.partition("-")
74
+ cpus.extend(range(int(first), int(last or first) + 1))
75
+ return cpus
76
+
77
+
78
+ def _cpuset():
79
+ """CPUs allowed by this process's cgroup cpuset, or None if there is none to read."""
80
+ try:
81
+ for line in Path("/proc/self/cgroup").read_text().splitlines():
82
+ hierarchy, controllers, path = line.split(":", 2)
83
+ if hierarchy == "0" or "cpuset" in controllers.split(","):
84
+ root = Path("/sys/fs/cgroup") if hierarchy == "0" else Path("/sys/fs/cgroup/cpuset")
85
+ for name in ("cpuset.cpus.effective", "cpuset.cpus"):
86
+ listed = root / path.lstrip("/") / name
87
+ if listed.is_file() and listed.read_text().strip():
88
+ return _cpu_list(listed.read_text())
89
+ except (OSError, ValueError):
90
+ pass
91
+ return None
92
+
93
+
94
+ def _allowed_cpus(environ, cpuset, affinity) -> list:
95
+ """
96
+ CPUs this job step has on its node, for its ranks to share.
97
+
98
+ SLURM confines a step with a cpuset, which may be wider than the
99
+ `affinity` an MPI library gave a rank. The `cpuset` is used when it is
100
+ the step's: it has no more CPUs than SLURM_CPUS_ON_NODE and contains the
101
+ affinity. Otherwise the affinity is kept. Where SLURM does not confine
102
+ steps, the cpuset is the whole node and could include another job's
103
+ CPUs. The affinity is also kept when there is no cpuset or
104
+ SLURM_CPUS_ON_NODE is not set.
105
+ """
106
+ on_node = environ.get("SLURM_CPUS_ON_NODE")
107
+ if (cpuset and on_node is not None and on_node.isdigit()
108
+ and len(cpuset) <= int(on_node) and set(affinity) <= set(cpuset)):
109
+ return list(cpuset)
110
+ return list(affinity)
111
+
112
+
113
+ def _rank_cpus(environ, allowed: list) -> list:
114
+ """
115
+ CPUs for this MPI rank: its share of `allowed`, by local task id.
116
+
117
+ Each task gets SLURM_CPUS_PER_TASK of `allowed`, in order. All of
118
+ `allowed` is returned if there are too few for every task on the node
119
+ to have its own. An empty list, which leaves the affinity alone, is
120
+ returned if SLURM does not say how many CPUs a task has.
121
+ """
122
+ per_task, local = environ.get("SLURM_CPUS_PER_TASK"), environ.get("SLURM_LOCALID")
123
+ if per_task is None or local is None:
124
+ return []
125
+ per_task, local = int(per_task), int(local)
126
+ share = allowed[local * per_task:(local + 1) * per_task]
127
+ return share if len(share) == per_task else list(allowed)
128
+
129
+
130
+ # main() builds the Simulation objects itself and takes only these keys from
131
+ # the 'simulation:' section. The other Simulation fields (instrument, n_iter,
132
+ # ncpu and the pinhole lists) are top-level keys; one written here would be
133
+ # parsed and then dropped.
134
+ _SIMULATION_KEYS = {"slit_width", "expos", "vis_sl", "psf", "psf_boundary",
135
+ "spectral_psf", "slit_mask", "focal_plane_dispersion", "noise",
136
+ "readout_dark_current", "enable_pinholes", "pinhole_visible_wavelength"}
137
+
138
+ _FITTING_KEYS = {"components", "primary_component",
139
+ "constrain_positive_intensity", "backend", "max_iter",
140
+ "bessel_correction", "save_iterations", "weighted"}
141
+ _FITTING_COMPONENT_KEYS = {"wavelength", "tie_center", "tie_width",
142
+ "amplitude_greater_than", "name"}
143
+
144
+
145
+ def _dataclass_keys(cls) -> set:
146
+ """Constructor argument names of a config dataclass."""
147
+ return {f.name for f in dataclasses.fields(cls) if f.init}
148
+
149
+
150
+ def _type_name(value) -> str:
151
+ """Type of a config value for an error message; an empty YAML entry is None."""
152
+ return "nothing" if value is None else type(value).__name__
153
+
154
+
155
+ def _require_mapping(value, what: str) -> None:
156
+ """Raise unless a config section or fitting component is a mapping."""
157
+ if not isinstance(value, dict):
158
+ raise ValueError(
159
+ f"{what} must be a mapping of parameter names to values, "
160
+ f"got {_type_name(value)}."
161
+ )
162
+
163
+
164
+ def _validate_config_keys(config: dict, instrument: str) -> None:
165
+ """
166
+ Reject any config key ECLIPSE does not read.
167
+
168
+ Called before anything is loaded or computed, so a config written for an
169
+ older layout fails at once rather than after the atmosphere is loaded.
170
+
171
+ Parameters
172
+ ----------
173
+ config : dict
174
+ Whole parsed YAML config.
175
+ instrument : str
176
+ `"SWC"` or `"EIS"`, which have different detector and telescope
177
+ parameters.
178
+
179
+ Raises
180
+ ------
181
+ ValueError
182
+ If the instrument is not supported, a key is not read, or a section
183
+ that is present is not a mapping.
184
+ """
185
+ # Checked first because the valid keys depend on it, and main() treats
186
+ # anything other than SWC as EIS.
187
+ if instrument not in ("SWC", "EIS"):
188
+ raise ValueError(
189
+ f"Unknown instrument '{instrument}'. Supported values: 'SWC', 'EIS'."
190
+ )
191
+
192
+ det_class = Detector_EIS if instrument == "EIS" else Detector_SWC
193
+ # Old names, such as _dark_current_293k, still work, with a FutureWarning.
194
+ det_keys = _dataclass_keys(det_class) | set(det_class._OLD_NAMES)
195
+ tel_keys = _dataclass_keys(Telescope_EIS if instrument == "EIS"
196
+ else Telescope_EUVST)
197
+ fil_keys = _dataclass_keys(AluminiumFilter)
198
+
199
+ # 'filter' is a Telescope_EUVST field, but main() builds the filter from
200
+ # the top-level 'filter:' section and would drop a value given here.
201
+ tel_keys.discard("filter")
202
+ if instrument == "EIS":
203
+ # Allowed here; main() warns about it and ignores it.
204
+ tel_keys.add("microroughness_sigma")
205
+
206
+ sections = {
207
+ "simulation": _SIMULATION_KEYS,
208
+ "detector": det_keys,
209
+ "telescope": tel_keys,
210
+ "filter": fil_keys,
211
+ }
212
+ # Other places a key could belong, for the suggestions. The top level is
213
+ # included so a section key written at the wrong depth is reported as such.
214
+ elsewhere = {"": _TOP_LEVEL_KEYS, **sections}
215
+
216
+ check_config_keys(config, _TOP_LEVEL_KEYS, "top-level", sections)
217
+
218
+ # Checked by presence, not value, so an empty heading (which parses to
219
+ # None) gets a clear message here rather than an AttributeError later.
220
+ for name, allowed in sections.items():
221
+ if name not in config:
222
+ continue
223
+ _require_mapping(config[name], f"The '{name}:' section")
224
+ others = {k: v for k, v in elsewhere.items() if k != name}
225
+ check_config_keys(config[name], allowed, f"'{name}' section", others)
226
+
227
+ for name, allowed in (("synthesis", _SYNTHESIS_KEYS), ("raster", _RASTER_KEYS)):
228
+ if name in config:
229
+ _require_mapping(config[name], f"The '{name}:' section")
230
+ check_config_keys(config[name], allowed, f"'{name}' section")
231
+
232
+ if "fitting" in config:
233
+ fitting = config["fitting"]
234
+ _require_mapping(fitting, "The 'fitting:' section")
235
+ check_config_keys(fitting, _FITTING_KEYS, "'fitting' section")
236
+ if "components" in fitting:
237
+ components = fitting["components"]
238
+ if not isinstance(components, list):
239
+ raise ValueError(
240
+ f"'fitting.components' must be a list with one entry per "
241
+ f"Gaussian component, got {_type_name(components)}."
242
+ )
243
+ for idx, component in enumerate(components):
244
+ where = f"'fitting.components[{idx}]'"
245
+ _require_mapping(component, where)
246
+ check_config_keys(component, _FITTING_COMPONENT_KEYS, where)
247
+
248
+
249
+ def _quantity_or_none(section: dict, key: str, unit, what: str):
250
+ """Quantity in units equivalent to `unit` from a config section, or None if absent."""
251
+ if key not in section or section[key] is None:
252
+ return None
253
+ value = parse_yaml_input(section[key])
254
+ if not isinstance(value, u.Quantity) or not value.unit.is_equivalent(unit):
255
+ raise ValueError(f"'{what}.{key}' must be {unit.physical_type} with units, "
256
+ f"e.g. '{section[key]}' is not; got {section[key]!r}.")
257
+ return value
258
+
259
+
260
+ def _range_or_none(section: dict, key: str, what: str):
261
+ """`[low, high]` range of lengths from a config section, or None if absent."""
262
+ if key not in section or section[key] is None:
263
+ return None
264
+ bounds = section[key]
265
+ if not isinstance(bounds, (list, tuple)) or len(bounds) != 2:
266
+ raise ValueError(f"'{what}.{key}' must be a list of two lengths with units, "
267
+ f"e.g. ['0 Mm', '20 Mm'], got {bounds!r}.")
268
+ low, high = (parse_yaml_input(b) for b in bounds)
269
+ for value in (low, high):
270
+ if not isinstance(value, u.Quantity) or not value.unit.is_equivalent(u.Mm):
271
+ raise ValueError(f"'{what}.{key}' must hold lengths with units, got {bounds!r}.")
272
+ return (low, high)
273
+
274
+
275
+ def _parse_synthesis_settings(config: dict):
276
+ """Synthesis settings for a time series, from the 'synthesis:' section."""
277
+ from .raster import SynthesisSettings
278
+
279
+ section = config.get("synthesis")
280
+ if section is None:
281
+ raise ValueError("An 'atmosphere_series' needs a 'synthesis:' section naming at "
282
+ "least the 'lines' to synthesise.")
283
+ lines = section.get("lines")
284
+ if isinstance(lines, str):
285
+ lines = [lines]
286
+ if not isinstance(lines, list) or not lines or not all(isinstance(l, str) for l in lines):
287
+ raise ValueError("'synthesis.lines' must be a list of line names such as "
288
+ "['Fe12_195.1190'].")
289
+ settings = {"lines": tuple(lines)}
290
+ if "abundance" in section:
291
+ settings["abundance"] = str(section["abundance"])
292
+ for key in ("vel_res", "vel_lim"):
293
+ value = _quantity_or_none(section, key, u.km / u.s, "synthesis")
294
+ if value is not None:
295
+ settings[key] = value
296
+ for key in ("crop_y", "crop_z"):
297
+ settings[key] = _range_or_none(section, key, "synthesis")
298
+ if "precision" in section:
299
+ precision = str(section["precision"])
300
+ if precision not in ("float32", "float64"):
301
+ raise ValueError(f"'synthesis.precision' must be 'float32' or 'float64', "
302
+ f"got {precision!r}.")
303
+ settings["precision"] = np.float32 if precision == "float32" else np.float64
304
+ if section.get("mass_per_electron") is not None:
305
+ value = section["mass_per_electron"]
306
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
307
+ raise ValueError(f"'synthesis.mass_per_electron' must be a number of atomic mass "
308
+ f"units, such as 1.16, got {value!r}.")
309
+ settings["mass_per_electron"] = float(value)
310
+ if section.get("hdf5_dbase_root") is not None:
311
+ settings["hdf5_dbase_root"] = str(section["hdf5_dbase_root"])
312
+ # Require whole numbers rather than calling int(), which would truncate a
313
+ # fraction.
314
+ for key, least in (("n_workers", 0), ("goft_temperature_chunk", 1)):
315
+ value = section.get(key)
316
+ if value is None:
317
+ continue
318
+ if isinstance(value, bool) or not isinstance(value, int) or value < least:
319
+ raise ValueError(f"'synthesis.{key}' must be a whole number of {least} or more, "
320
+ f"got {value!r}.")
321
+ settings[key] = value
322
+ return SynthesisSettings(**settings)
323
+
324
+
325
+ def _parse_raster_plan(config: dict):
326
+ """Observing plan from the 'raster:' section."""
327
+ from .raster import RasterPlan
328
+
329
+ section = config.get("raster")
330
+ if section is None:
331
+ raise ValueError("A time series needs a 'raster:' section saying at least when "
332
+ "the observation starts ('start').")
333
+ start = _quantity_or_none(section, "start", u.s, "raster")
334
+ if start is None:
335
+ raise ValueError("'raster.start' is needed: the simulation time at which the "
336
+ "first exposure starts, with units, e.g. '3850 s'.")
337
+ plan = {"start": start}
338
+ for key in ("steps", "repeats"):
339
+ if section.get(key) is not None:
340
+ plan[key] = section[key]
341
+ plan["step"] = _quantity_or_none(section, "step", u.arcsec, "raster")
342
+ plan["cadence"] = _quantity_or_none(section, "cadence", u.s, "raster")
343
+ plan["centre"] = _quantity_or_none(section, "centre", u.Mm, "raster")
344
+ if section.get("direction") is not None:
345
+ plan["direction"] = section["direction"]
346
+ return RasterPlan(**plan)
347
+
348
+
349
+ def _series_paths(config: dict, key: str) -> list:
350
+ """Files of 'atmosphere_series' or 'synthesis_series', given as a list or a glob pattern."""
351
+ import glob
352
+
353
+ value = config[key]
354
+ if isinstance(value, str):
355
+ paths = sorted(glob.glob(value))
356
+ if not paths:
357
+ raise FileNotFoundError(f"'{key}' matches no file: {value}")
358
+ return paths
359
+ if isinstance(value, list) and value and all(isinstance(p, str) for p in value):
360
+ missing = [p for p in value if not Path(p).is_file()]
361
+ if missing:
362
+ raise FileNotFoundError(f"'{key}' names files that do not exist: {missing}")
363
+ return list(value)
364
+ kind = "atmosphere" if key == "atmosphere_series" else "synthesis"
365
+ raise ValueError(f"'{key}' must be a glob pattern or a list of {kind} files, "
366
+ f"e.g. './data/bifrost/*.h5'.")
367
+
368
+
369
+ # Line observed when the config names none and the synthesis holds several.
370
+ DEFAULT_REFERENCE_LINE = "Fe12_195.1190"
371
+
372
+ # Where synthesise-spectra writes by default, and where older versions did.
373
+ DEFAULT_SYNTHESIS_FILE = "./run/input/synthesised_spectra.h5"
374
+ LEGACY_SYNTHESIS_FILE = "./run/input/synthesised_spectra.pkl"
375
+
376
+
377
+ def _reference_line(config: dict, synthesis_path) -> str:
378
+ """
379
+ Line to observe in a synthesis file.
380
+
381
+ This is 'reference_line' if set. If the key is present but empty, it is
382
+ the file's first line, as in older versions. If the key is absent, it is
383
+ the file's only line, or else the default line, which the file must then
384
+ hold.
385
+ """
386
+ if config.get("reference_line") is not None:
387
+ return config["reference_line"]
388
+ names = synthesis_line_names(synthesis_path)
389
+ if not names:
390
+ # A file with no lines is refused when it is read.
391
+ return DEFAULT_REFERENCE_LINE
392
+ if "reference_line" in config or len(names) == 1:
393
+ return names[0]
394
+ if DEFAULT_REFERENCE_LINE not in names:
395
+ raise ValueError(f"{synthesis_path} holds the lines {names} and no 'reference_line' "
396
+ f"says which to observe; the default, {DEFAULT_REFERENCE_LINE}, is "
397
+ f"not among them.")
398
+ return DEFAULT_REFERENCE_LINE
399
+
400
+
401
+ def _parse_pinhole_config(config: dict) -> tuple:
402
+ """
403
+ Read and validate the pinhole lists from a YAML config.
404
+
405
+ The lists have one entry per pinhole, so a position list given without
406
+ the sizes describes no pinhole and is always a mistake.
407
+
408
+ Parameters
409
+ ----------
410
+ config : dict
411
+ Whole parsed YAML config.
412
+
413
+ Returns
414
+ -------
415
+ tuple
416
+ `(pinhole_sizes, pinhole_positions, pinhole_positions_spectral)`.
417
+ """
418
+ pinhole_sizes = []
419
+ pinhole_positions = []
420
+ if "pinhole_sizes" in config:
421
+ pinhole_sizes = ensure_list(parse_yaml_input(config["pinhole_sizes"]))
422
+ if "pinhole_positions" in config:
423
+ pinhole_positions = ensure_list(config["pinhole_positions"])
424
+
425
+ # Optional spectral positions, one per pinhole, as a fraction (0.0-1.0) of
426
+ # the detector's spectral width. If omitted, every pinhole is projected to
427
+ # the centre of the spectral window.
428
+ pinhole_positions_spectral = []
429
+ if "pinhole_positions_spectral" in config:
430
+ pinhole_positions_spectral = ensure_list(
431
+ config["pinhole_positions_spectral"])
432
+
433
+ pinhole_positions, pinhole_positions_spectral = check_pinhole_lists(
434
+ pinhole_sizes, pinhole_positions, pinhole_positions_spectral)
435
+
436
+ return pinhole_sizes, pinhole_positions, pinhole_positions_spectral
437
+
438
+
439
+ def _parse_fitting_config(config: dict) -> FitConfig | None:
440
+ """
441
+ Fit configuration from the `fitting:` block of a YAML config.
442
+
443
+ A block with no components gives the single-Gaussian fit. FitConfig
444
+ checks the values, including that there are no components or at least
445
+ two.
446
+
447
+ Parameters
448
+ ----------
449
+ config : dict
450
+ Whole parsed YAML config.
451
+
452
+ Returns
453
+ -------
454
+ FitConfig or None
455
+ None if the config has no `fitting:` block.
456
+ """
457
+ fitting_cfg = config.get("fitting", None)
458
+ if fitting_cfg is None:
459
+ return None
460
+
461
+ components = []
462
+ for idx, comp_dict in enumerate(fitting_cfg.get("components", [])):
463
+ if "wavelength" not in comp_dict:
464
+ raise ValueError(
465
+ f"fitting.components[{idx}] is missing required field "
466
+ f"'wavelength' (rest wavelength of this Gaussian component, "
467
+ f"e.g. 'wavelength: 195.119 angstrom')."
468
+ )
469
+ components.append(FitComponent(
470
+ wavelength=parse_yaml_input(comp_dict["wavelength"]),
471
+ tie_center=comp_dict.get("tie_center", None),
472
+ tie_width=comp_dict.get("tie_width", None),
473
+ amplitude_greater_than=comp_dict.get("amplitude_greater_than", None),
474
+ name=comp_dict.get("name", None),
475
+ ))
476
+
477
+ return FitConfig(
478
+ components=components,
479
+ primary_component=fitting_cfg.get("primary_component", 0),
480
+ constrain_positive_intensity=fitting_cfg.get(
481
+ "constrain_positive_intensity", False),
482
+ backend=fitting_cfg.get("backend", None),
483
+ max_iter=fitting_cfg.get("max_iter", FitConfig.max_iter),
484
+ bessel_correction=fitting_cfg.get("bessel_correction", False),
485
+ save_iterations=fitting_cfg.get("save_iterations", False),
486
+ weighted=fitting_cfg.get("weighted", True),
487
+ )
488
+
489
+
490
+ @debug_on_error
491
+ def main() -> None:
492
+ """
493
+ Run the `eclipse` simulation with arguments from `sys.argv`.
494
+
495
+ Reads the config file given by `--config`, runs every combination of its
496
+ settings, and writes the results to `run/result/<config name>.h5`. The
497
+ `eclipse` command checks its arguments and handles errors before calling
498
+ this.
499
+ """
500
+
501
+ # Suppress noisy astropy WCS warnings
502
+ for msg in [
503
+ "target cannot be converted to ICRS",
504
+ "target cannot be converted to ICRS, so will not be set on SpectralCoord",
505
+ "No observer defined on WCS, SpectralCoord will be converted without any velocity frame change",
506
+ ]:
507
+ warnings.filterwarnings("ignore", message=msg, category=UserWarning)
508
+ warnings.filterwarnings("ignore", module="astropy.wcs.wcsapi.fitswcs", category=UserWarning)
509
+
510
+ parser = argparse.ArgumentParser()
511
+ parser.add_argument("--config", type=str, help="YAML config file", required=True)
512
+ parser.add_argument("--debug", action="store_true", help="Enable debug mode")
513
+ args = parser.parse_args()
514
+
515
+ set_debug_mode(args.debug)
516
+ if args.debug:
517
+ print("Debug mode enabled - will break to IPython on errors")
518
+
519
+ # MPI auto-detection: when launched by srun or mpirun with several tasks,
520
+ # the Monte Carlo iterations are split across the ranks.
521
+ from .utils import _get_mpi_info
522
+ _comm, _mpi_rank, _mpi_size = _get_mpi_info()
523
+ if _mpi_size > 1:
524
+ # Intel MPI pins each rank to cores, which breaks joblib/loky. Widen
525
+ # the affinity to this rank's share of the job step's CPUs on the
526
+ # node. CPUs 0 to n would be shared by every rank and, on a node
527
+ # shared with another job, need not be this job's.
528
+ _cpus = _rank_cpus(os.environ, _allowed_cpus(os.environ, _cpuset(),
529
+ sorted(os.sched_getaffinity(0))))
530
+ if _cpus:
531
+ os.sched_setaffinity(0, _cpus)
532
+
533
+ if _mpi_rank == 0:
534
+ print(f"MPI distributed mode: {_mpi_size} processes "
535
+ f"(MC iterations will be split across ranks)")
536
+ else:
537
+ # Silence stdout and stderr on non-root ranks to avoid duplicated
538
+ # output. Redirect the OS file descriptors, not just the Python
539
+ # objects, because SLURM captures fd 1/2 directly and tqdm can
540
+ # bypass sys.stderr. The original stderr is kept for an error that
541
+ # must be reported before the run is aborted.
542
+ global _STARTING_STDERR
543
+ _STARTING_STDERR = os.dup(2)
544
+ _devnull_fd = os.open(os.devnull, os.O_WRONLY)
545
+ os.dup2(_devnull_fd, 1) # redirect fd 1 (stdout)
546
+ os.dup2(_devnull_fd, 2) # redirect fd 2 (stderr)
547
+ os.close(_devnull_fd)
548
+ # Wrap the already-redirected fds rather than opening new devnull
549
+ # handles; closefd=False prevents a ResourceWarning when these
550
+ # objects are later garbage-collected.
551
+ sys.stdout = open(1, "w", closefd=False)
552
+ sys.stderr = open(2, "w", closefd=False)
553
+
554
+ config_path = Path(args.config)
555
+ if not config_path.is_file():
556
+ raise FileNotFoundError(f"Config file not found: {args.config}")
557
+
558
+ config = load_yaml_config(config_path.read_text())
559
+
560
+ # An empty file parses to None, which would otherwise fail later with an
561
+ # AttributeError.
562
+ if config is None:
563
+ raise ValueError(f"Config file is empty: {args.config}")
564
+ if not isinstance(config, dict):
565
+ raise ValueError(
566
+ f"Config file must be a mapping of keys to values, got "
567
+ f"{type(config).__name__}: {args.config}"
568
+ )
569
+ # The config is saved with the results, so a value the results file
570
+ # cannot hold, such as a YAML alias that refers to itself, is refused now
571
+ # rather than after the run.
572
+ try:
573
+ _to_json(config)
574
+ except (TypeError, RecursionError) as error:
575
+ reason = "a value holds itself" if isinstance(error, RecursionError) else error
576
+ raise ValueError(f"{args.config} cannot be saved with the results: {reason}") from None
577
+
578
+ # Top-level scalar settings. A 'simulation:' that is not a mapping is left
579
+ # for _validate_config_keys to report.
580
+ simulation_section = config.get("simulation")
581
+ if isinstance(simulation_section, dict) and "instrument" in simulation_section:
582
+ raise ValueError(
583
+ "Set the instrument with the top-level 'instrument:' key, not "
584
+ "inside the 'simulation:' section. Both Simulation objects are "
585
+ "built from the top-level value, so one written here would be "
586
+ "read and then ignored."
587
+ )
588
+ instrument = config.get("instrument", "SWC")
589
+ if not isinstance(instrument, str):
590
+ raise ValueError(f"'instrument' must be SWC or EIS, got {instrument!r}.")
591
+ instrument = instrument.upper()
592
+
593
+ # Before anything is loaded, so a config written for an older layout fails
594
+ # here rather than after the atmosphere is loaded.
595
+ _validate_config_keys(config, instrument)
596
+
597
+ n_iter = config.get("n_iter", 25)
598
+ # Checked now: 0 would write empty results and report success, and '1e3',
599
+ # a string to PyYAML, would fail only after the ground truth was fitted.
600
+ if isinstance(n_iter, bool) or not isinstance(n_iter, int) or n_iter < 1:
601
+ raise ValueError(f"'n_iter' must be a whole number of iterations, 1 or more, got "
602
+ f"{n_iter!r}.")
603
+ ncpu = config.get("ncpu", -1)
604
+ # joblib and reproject both use ncpu and agree only on these values. 0,
605
+ # which means every CPU for the synthesis's n_workers, would fail at the
606
+ # first fit.
607
+ if isinstance(ncpu, bool) or not isinstance(ncpu, int) or (ncpu < 1 and ncpu != -1):
608
+ raise ValueError(f"'ncpu' must be a whole number of CPUs, 1 or more, or -1 for all "
609
+ f"of them, got {ncpu!r}.")
610
+
611
+ # In MPI mode, cap a given ncpu at the CPUs available to this rank so
612
+ # joblib does not oversubscribe. ncpu=-1 is left alone: joblib uses the
613
+ # OS affinity mask, which I_MPI_PIN_DOMAIN=auto sets correctly.
614
+ if _mpi_size > 1:
615
+ if ncpu != -1:
616
+ slurm_cpus = os.environ.get("SLURM_CPUS_PER_TASK")
617
+ if slurm_cpus is not None:
618
+ available_cpus = int(slurm_cpus)
619
+ else:
620
+ available_cpus = os.cpu_count() or 1
621
+ if ncpu > available_cpus:
622
+ ncpu = available_cpus
623
+ if _mpi_rank == 0:
624
+ print(f"MPI: ncpu={ncpu} per rank")
625
+
626
+ # Simulation mode. A time series is either atmosphere files, synthesised
627
+ # in this run, or synthesis files made beforehand.
628
+ uniform_intensity_mode = "uniform_intensity" in config
629
+ if uniform_intensity_mode and "synthesis_file" in config:
630
+ warnings.warn("'synthesis_file' is ignored: 'uniform_intensity' says what is observed.",
631
+ UserWarning, stacklevel=2)
632
+ # These describe the line of a uniform intensity; other modes observe the
633
+ # spectra they are given.
634
+ for key in ("rest_wavelength", "thermal_width"):
635
+ if key in config and not uniform_intensity_mode:
636
+ warnings.warn(f"'{key}' is ignored: it describes the line of a 'uniform_intensity', "
637
+ f"and this run observes the spectra it is given.",
638
+ UserWarning, stacklevel=2)
639
+ atmosphere_series_mode = "atmosphere_series" in config
640
+ synthesis_series_mode = "synthesis_series" in config
641
+ raster_mode = atmosphere_series_mode or synthesis_series_mode
642
+ given = [key for key in ("uniform_intensity", "synthesis_file", "atmosphere_series",
643
+ "synthesis_series") if key in config]
644
+ if raster_mode and len(given) > 1:
645
+ raise ValueError(f"Give only one of {given}: each says what is observed.")
646
+ if not atmosphere_series_mode and "synthesis" in config:
647
+ raise ValueError("The 'synthesis:' section belongs to an 'atmosphere_series' run "
648
+ "and would not be read here.")
649
+ if not raster_mode and "raster" in config:
650
+ raise ValueError("The 'raster:' section belongs to an 'atmosphere_series' or "
651
+ "'synthesis_series' run and would not be read here.")
652
+
653
+ if atmosphere_series_mode:
654
+ series_paths = _series_paths(config, "atmosphere_series")
655
+ synthesis_settings = _parse_synthesis_settings(config)
656
+ raster_plan = _parse_raster_plan(config)
657
+ # Empty or absent: the first line.
658
+ reference_line = config.get("reference_line") or synthesis_settings.lines[0]
659
+ if reference_line not in synthesis_settings.lines:
660
+ raise ValueError(f"'reference_line' {reference_line!r} is not one of "
661
+ f"'synthesis.lines' {list(synthesis_settings.lines)}.")
662
+ print("TIME SERIES MODE")
663
+ print(f" Atmosphere files: {len(series_paths)}")
664
+ print(f" Lines: {list(synthesis_settings.lines)}, reference {reference_line}")
665
+ print(f" Raster: {raster_plan.steps} step(s), {raster_plan.repeats} repeat(s), "
666
+ f"starting at {raster_plan.start}")
667
+ elif synthesis_series_mode:
668
+ series_paths = _series_paths(config, "synthesis_series")
669
+ synthesis_settings = None
670
+ raster_plan = _parse_raster_plan(config)
671
+ reference_line = _reference_line(config, series_paths[0])
672
+ print("TIME SERIES MODE")
673
+ print(f" Synthesis files: {len(series_paths)}")
674
+ print(f" Reference line: {reference_line}")
675
+ print(f" Raster: {raster_plan.steps} step(s), {raster_plan.repeats} repeat(s), "
676
+ f"starting at {raster_plan.start}")
677
+ elif uniform_intensity_mode:
678
+ uniform_intensity = parse_yaml_input(config["uniform_intensity"])
679
+ if not hasattr(uniform_intensity, "unit"):
680
+ raise ValueError(
681
+ "uniform_intensity must include units, e.g. '5000 erg / (s cm2 sr)'"
682
+ )
683
+ uniform_rest_wavelength = parse_yaml_input(config.get("rest_wavelength", "195.119 AA"))
684
+ uniform_thermal_width = parse_yaml_input(config.get("thermal_width", "20 km/s"))
685
+ # Each must be positive, in units of the right kind: a zero or negative
686
+ # width gives a nonsense fit, which would be saved as a result.
687
+ for key, value, kind, example in (
688
+ ("uniform_intensity", uniform_intensity, u.erg / (u.s * u.cm**2 * u.sr),
689
+ "5000 erg / (s cm2 sr)"),
690
+ ("rest_wavelength", uniform_rest_wavelength, u.AA, "195.119 AA"),
691
+ ("thermal_width", uniform_thermal_width, u.km / u.s, "20 km/s")):
692
+ if not (isinstance(value, u.Quantity) and value.unit.is_equivalent(kind)
693
+ and np.isfinite(value.value) and value.value > 0):
694
+ raise ValueError(f"'{key}' must be a positive {kind.physical_type}, such as "
695
+ f"'{example}', got {config.get(key)!r}.")
696
+ print("UNIFORM INTENSITY MODE")
697
+ print(f" Intensity: {uniform_intensity}")
698
+ print(f" Rest wavelength: {uniform_rest_wavelength}")
699
+ print(f" Thermal width (1-sigma): {uniform_thermal_width}")
700
+ else:
701
+ synthesis_file = config.get("synthesis_file", DEFAULT_SYNTHESIS_FILE)
702
+ # Fall back to the pickle older versions wrote, for a run that leaves
703
+ # the synthesis file at its default.
704
+ if "synthesis_file" not in config and Path(LEGACY_SYNTHESIS_FILE).is_file():
705
+ if not Path(synthesis_file).is_file():
706
+ synthesis_file = LEGACY_SYNTHESIS_FILE
707
+ else:
708
+ warnings.warn(f"Both {DEFAULT_SYNTHESIS_FILE} and {LEGACY_SYNTHESIS_FILE} exist "
709
+ f"and 'synthesis_file' names neither; observing "
710
+ f"{DEFAULT_SYNTHESIS_FILE}, where the synthesis now writes. "
711
+ f"Name the one to observe with 'synthesis_file'.",
712
+ UserWarning, stacklevel=2)
713
+ if not Path(synthesis_file).is_file():
714
+ raise FileNotFoundError(
715
+ f"Synthesis file not found: {synthesis_file}. "
716
+ "Please check the 'synthesis_file' path in your config file."
717
+ )
718
+ synthesis_is_hdf5 = is_synthesis_file(synthesis_file)
719
+ if synthesis_is_hdf5:
720
+ reference_line = _reference_line(config, synthesis_file)
721
+ elif not _is_pickle(Path(synthesis_file)):
722
+ raise ValueError(
723
+ f"{synthesis_file} is neither a synthesis file, which is HDF5, nor a synthesis "
724
+ f"pickle as older versions of ECLIPSE wrote. A synthesis from another code is "
725
+ f"written as a synthesis file with eclipse.write_synthesis.")
726
+ else:
727
+ warnings.warn(
728
+ f"{synthesis_file} is a synthesis pickle, as older versions of ECLIPSE "
729
+ f"wrote them. Pickles are deprecated and will not be read in a future "
730
+ f"release: re-run the synthesis, or convert the file with "
731
+ f"eclipse.convert_synthesis_pickle.", FutureWarning, stacklevel=2)
732
+ reference_line = config.get("reference_line", DEFAULT_REFERENCE_LINE)
733
+
734
+ # Pinhole config (fixed paired lists, not swept)
735
+ (pinhole_sizes, pinhole_positions,
736
+ pinhole_positions_spectral) = _parse_pinhole_config(config)
737
+
738
+ # Parse config sections
739
+ sim_fixed, sim_sweep = _parse_section(config.get("simulation", {}), "simulation")
740
+ det_fixed, det_sweep = _parse_section(config.get("detector", {}), "detector")
741
+ # An old name, such as _dark_current_293k, is recorded as the config gives
742
+ # it and passed to the detector under its new one.
743
+ det_renamed = _old_names_used(Detector_EIS if instrument == "EIS" else Detector_SWC,
744
+ {**det_fixed, **det_sweep}, "detector.", error=ValueError)
745
+ tel_fixed, tel_sweep = _parse_section(config.get("telescope", {}), "telescope")
746
+ fil_fixed, fil_sweep = _parse_section(config.get("filter", {}), "filter")
747
+
748
+ # The width of the pixels is taken where the observed line lands, which a
749
+ # run reads from a synthesis file or the uniform-intensity settings.
750
+ focal_plane_dispersion = (sim_fixed.get("focal_plane_dispersion")
751
+ or True in sim_sweep.get("focal_plane_dispersion", []))
752
+ if focal_plane_dispersion and instrument == "SWC" and (atmosphere_series_mode or not (
753
+ uniform_intensity_mode or synthesis_series_mode or synthesis_is_hdf5)):
754
+ raise ValueError(
755
+ "simulation.focal_plane_dispersion needs the observed line's rest wavelength, which "
756
+ "ECLIPSE reads from a synthesis file or the uniform-intensity settings, not yet from "
757
+ "an atmosphere series or a synthesis pickle.")
758
+
759
+ # A run models frames taken with the shutter closed outside the exposure.
760
+ # Without a shutter, light falls on the CCDs while a frame is cleared and
761
+ # read, and every line in the band smears into the window. The run
762
+ # synthesises only the window, so it cannot model this, and refuses the
763
+ # setting rather than ignoring it.
764
+ if "shutter" in det_sweep or det_fixed.get("shutter", True) is False:
765
+ raise ValueError(
766
+ "detector.shutter must be true for an instrument run, which models frames taken "
767
+ "with the shutter closed outside the exposure. Frames taken without it are "
768
+ "modelled by eclipse.readout, which takes the detector's settings.")
769
+ # A run does not model charge transfer either, so a CTE below 1 would be ignored.
770
+ for name in ("cte_parallel", "cte_serial"):
771
+ if name in det_sweep or det_fixed.get(name, 1.0) != 1.0:
772
+ raise ValueError(
773
+ f"detector.{name} must be 1 for an instrument run, which does not model "
774
+ f"charge transfer. Charge left behind as a frame is read is modelled by "
775
+ f"eclipse.readout, which takes the detector's settings.")
776
+ # Nor does it take its wavelengths from the SW focal plane. It uses the focal
777
+ # plane only to find the row recording each wavelength, for the dark current,
778
+ # and, with focal_plane_dispersion, the width of its pixels. Other values
779
+ # would make these disagree with its wavelengths.
780
+ if instrument == "SWC":
781
+ focal_plane = {name: det_fixed[name] for name in _FOCAL_PLANE_SETTINGS
782
+ if name in det_fixed}
783
+ default, given = Detector_SWC(), Detector_SWC(**focal_plane)
784
+ for name in _FOCAL_PLANE_SETTINGS:
785
+ same = _same_setting(getattr(given, name), getattr(default, name))
786
+ if name in det_sweep or not same:
787
+ raise ValueError(
788
+ f"detector.{name} must be left at its default for an instrument run, "
789
+ f"which takes its wavelengths from detector.wvl_res, not from the SW focal "
790
+ f"plane, and needs the two to agree. Full-CCD frames, made by "
791
+ f"eclipse.full_frame, do.")
792
+
793
+ # The wavelength grids are cached by slit width and detector sampling, and
794
+ # sized for the slit's spectral PSF, which also depends on the slit
795
+ # psf_params was measured with. Like psf_params, that is a fixed property
796
+ # of the telescope, so it takes one value.
797
+ if "psf_slit_width" in tel_sweep:
798
+ raise ValueError(
799
+ "telescope.psf_slit_width is the slit psf_params was measured with, "
800
+ "so like psf_params it takes a single value and cannot be swept.")
801
+
802
+ # Instrument-specific validation
803
+ if instrument == "EIS":
804
+ if config.get("filter"):
805
+ warnings.warn(
806
+ "The 'filter:' section describes SWC's filter and is ignored for EIS, whose "
807
+ "filters are folded into its effective area tables.",
808
+ UserWarning,
809
+ )
810
+ fil_fixed, fil_sweep = {}, {}
811
+ vis_sl = [sim_fixed["vis_sl"]] if "vis_sl" in sim_fixed else sim_sweep.get("vis_sl", [])
812
+ if any(np.any(u.Quantity(value).value > 0) for value in vis_sl):
813
+ warnings.warn(
814
+ "EIS's filters are not modelled for visible light, so vis_sl reaches its CCD "
815
+ "as given, where for SWC it is the flux before the filter. Give EIS the flux "
816
+ "after its filters.",
817
+ UserWarning,
818
+ )
819
+ for key in ("microroughness_sigma",):
820
+ if key in tel_fixed or key in tel_sweep:
821
+ warnings.warn(
822
+ f"EIS does not support '{key}'. This parameter will be ignored.",
823
+ UserWarning,
824
+ )
825
+ tel_fixed.pop(key, None)
826
+ tel_sweep.pop(key, None)
827
+ # Any pinhole key, not just the sizes: positions given alone would
828
+ # otherwise be accepted and silently ignored.
829
+ if (pinhole_sizes or pinhole_positions or pinhole_positions_spectral
830
+ or sim_fixed.get("enable_pinholes")
831
+ or any(sim_sweep.get("enable_pinholes", []))
832
+ or "pinhole_visible_wavelength" in sim_fixed
833
+ or "pinhole_visible_wavelength" in sim_sweep):
834
+ raise ValueError(
835
+ "Pinhole effects are not supported for EIS. Remove "
836
+ "enable_pinholes, pinhole_sizes, pinhole_positions, "
837
+ "pinhole_positions_spectral and pinhole_visible_wavelength, or run this "
838
+ "config against SWC."
839
+ )
840
+
841
+ # Parse fitting configuration
842
+ fit_config = _parse_fitting_config(config)
843
+ if fit_config is not None:
844
+ if fit_config.backend == "mpfit":
845
+ backend_label = "mpfit (forced)"
846
+ elif fit_config.backend == "scipy":
847
+ backend_label = "scipy (forced)"
848
+ else:
849
+ backend_label = "scipy (auto)"
850
+ if fit_config.is_single:
851
+ print(f"Single-Gaussian fitting: max_iter={fit_config.max_iter}, "
852
+ f"backend={backend_label}")
853
+ else:
854
+ print(f"Multi-component fitting enabled: {fit_config.n_components} components "
855
+ f"(primary={fit_config.primary_component}, "
856
+ f"{fit_config.n_full_params} params, "
857
+ f"backend={backend_label})")
858
+
859
+ # Parse off-chip slit binning (ground-based spatial binning along the slit)
860
+ offchip_bin_slits = ensure_list(config.get("offchip_bin_slit", [1]))
861
+
862
+ # Parse which signals to fit (default: both DN and photon)
863
+ fit_signals = config.get("fit_signals", "both")
864
+ if fit_signals not in ("both", "dn", "photon"):
865
+ raise ValueError(
866
+ f"Unknown fit_signals value '{fit_signals}'. "
867
+ f"Supported values: 'both', 'dn', 'photon'."
868
+ )
869
+ if fit_signals != "both":
870
+ skipped = "photon" if fit_signals == "dn" else "dn"
871
+ print(f"Fitting only '{fit_signals}' signal (skipping '{skipped}')")
872
+ # Unless the fitting block turns weighting off, the DN fits are weighted
873
+ # by each pixel's uncertainty and the photon fits maximise the Poisson
874
+ # likelihood. The ground truth is weighted like the DN fits, except when
875
+ # only photons are fitted, as no DN uncertainty is then worked out.
876
+ weighted_fits = fit_config is None or fit_config.weighted
877
+ weighted_truth = weighted_fits and fit_signals != "photon"
878
+ print("DN fits weighted by each pixel's uncertainty, photon fits by their Poisson "
879
+ "likelihood" if weighted_fits else "Fits unweighted")
880
+
881
+ def _fit_ground_truth(cube_binned, offchip_bin_slit, uncertainty=None):
882
+ """Fit to the noise-free spectra, reporting where it failed."""
883
+ print(f"Fitting ground truth cube (offchip_bin_slit={offchip_bin_slit})...")
884
+ truth = ground_truth_summary(cube_binned, fit_config, n_jobs=ncpu,
885
+ uncertainty=uncertainty)
886
+ truth_failed = truth["failed"]
887
+ if truth_failed.any():
888
+ print(f" Ground truth fit failed in {np.count_nonzero(truth_failed)} "
889
+ f"of {truth_failed.size} pixels; their true velocity and "
890
+ f"width are NaN")
891
+ return truth
892
+ # ensure_list wraps scalars in a list. Values must be plain ints (no
893
+ # units), not passed through int(), which would turn a fraction or a
894
+ # numeric string into one.
895
+ if not offchip_bin_slits or any(isinstance(v, bool) or not isinstance(v, int) or v < 1
896
+ for v in offchip_bin_slits):
897
+ raise ValueError(f"'offchip_bin_slit' must be whole numbers of rows, 1 or more, got "
898
+ f"{config.get('offchip_bin_slit')!r}.")
899
+ offchip_bin_slits = deduplicate_list(offchip_bin_slits, "offchip_bin_slit")
900
+ if any(b > 1 for b in offchip_bin_slits):
901
+ print(f"Off-chip slit binning values: {offchip_bin_slits} "
902
+ f"(ground-based, all noise per pixel before summation)")
903
+
904
+ # Apply defaults for required params not specified anywhere
905
+ _sim_defaults = {
906
+ "slit_width": 0.2 * u.arcsec,
907
+ "expos": 1.0 * u.s,
908
+ "vis_sl": 0.0 * u.photon / (u.s * u.cm**2),
909
+ "psf": False,
910
+ "psf_boundary": "replicate",
911
+ "spectral_psf": "quadrature",
912
+ "slit_mask": False,
913
+ "focal_plane_dispersion": False,
914
+ "noise": True,
915
+ "readout_dark_current": True,
916
+ "enable_pinholes": False,
917
+ }
918
+ _det_defaults = {
919
+ "ccd_temperature": -60 * u.Celsius,
920
+ }
921
+ for attr, default in _sim_defaults.items():
922
+ if attr not in sim_fixed and attr not in sim_sweep:
923
+ sim_fixed[attr] = default
924
+ for attr, default in _det_defaults.items():
925
+ if attr not in det_fixed and attr not in det_sweep:
926
+ det_fixed[attr] = default
927
+
928
+ def _swept(*attrs):
929
+ """List of values for each simulation attribute in `attrs`: swept, or the fixed one."""
930
+ return [list(sim_sweep[attr]) if attr in sim_sweep else [sim_fixed[attr]]
931
+ for attr in attrs]
932
+
933
+ # Warnings
934
+ psf_vals = list(sim_sweep.get("psf", [sim_fixed.get("psf", False)]))
935
+ if any(psf_vals):
936
+ if instrument == "SWC":
937
+ warnings.warn(
938
+ "The SWC PSF is the modelled PSF including simulations and some microroughness "
939
+ "measurements. Final PSF will be measured before launch.",
940
+ UserWarning,
941
+ )
942
+ elif instrument == "EIS":
943
+ warnings.warn(
944
+ "The EIS PSF is not well understood. We use a symmetrical Gaussian kernel with "
945
+ "a FWHM of 3 pixels from Ugarte-Urra (2016) EIS Software Note 2.",
946
+ UserWarning,
947
+ )
948
+
949
+ noise_vals = list(sim_sweep.get("noise", [sim_fixed.get("noise", True)]))
950
+ if not all(noise_vals) and n_iter > 1:
951
+ warnings.warn(
952
+ f"noise is False, so every Monte Carlo iteration is identical. "
953
+ f"n_iter is {n_iter}; set it to 1 to avoid repeating the same "
954
+ f"deterministic run.",
955
+ UserWarning,
956
+ )
957
+
958
+ enable_ph_vals = list(sim_sweep.get("enable_pinholes", [sim_fixed.get("enable_pinholes", False)]))
959
+ if any(enable_ph_vals):
960
+ warnings.warn(
961
+ "Pinhole effects are only intended for use by the instrument team. "
962
+ "Please contact MSSL for more information.",
963
+ UserWarning,
964
+ )
965
+ if not pinhole_sizes:
966
+ warnings.warn(
967
+ "enable_pinholes is True but no pinhole_sizes specified. "
968
+ "Pinhole effects will be disabled.",
969
+ UserWarning,
970
+ )
971
+ if "enable_pinholes" in sim_sweep:
972
+ sim_sweep["enable_pinholes"] = [False]
973
+ else:
974
+ sim_fixed["enable_pinholes"] = False
975
+ # The wavelength sets only the light through the pinholes.
976
+ if "pinhole_visible_wavelength" in sim_sweep and not (pinhole_sizes and any(enable_ph_vals)):
977
+ raise ValueError(
978
+ "simulation.pinhole_visible_wavelength sets the light through the pinholes, which "
979
+ "are off, so a sweep of it would repeat one run. Give it one value, or turn the "
980
+ "pinholes on with enable_pinholes and pinhole_sizes.")
981
+
982
+ # Build sweep dimensions: every list-valued entry in a section becomes a named sweep dimension.
983
+ # Keys use "section.attribute" notation.
984
+ sweep_dims = {}
985
+ for attr, vals in sim_sweep.items():
986
+ sweep_dims[f"simulation.{attr}"] = vals
987
+ for attr, vals in det_sweep.items():
988
+ sweep_dims[f"detector.{attr}"] = vals
989
+ for attr, vals in tel_sweep.items():
990
+ sweep_dims[f"telescope.{attr}"] = vals
991
+ for attr, vals in fil_sweep.items():
992
+ if instrument != "EIS":
993
+ sweep_dims[f"filter.{attr}"] = vals
994
+
995
+ dim_names = list(sweep_dims.keys())
996
+ dim_values = [sweep_dims[n] for n in dim_names]
997
+ total_combinations = 1
998
+ for v in dim_values:
999
+ total_combinations *= len(v)
1000
+
1001
+ print(f"\nRunning {total_combinations} parameter combination(s).")
1002
+ if sweep_dims:
1003
+ print("Swept dimensions:")
1004
+ for dim, vals in sweep_dims.items():
1005
+ print(f" {dim}: {vals}")
1006
+
1007
+ # Load or create input cube
1008
+ raster = None
1009
+ raster_summed = {}
1010
+ raster_cubes = {}
1011
+ synthesis = None
1012
+ # Metadata kept with the cubes from a single synthesis file, as with a
1013
+ # pickle: its dynamic mode.
1014
+ file_meta = {}
1015
+ if uniform_intensity_mode:
1016
+ # A uniform intensity has no position along the slit for the slit's
1017
+ # fiducials to block, so the option is refused rather than ignored.
1018
+ if sim_fixed.get("slit_mask") or True in sim_sweep.get("slit_mask", []):
1019
+ raise ValueError("simulation.slit_mask needs a scene along the slit, from a "
1020
+ "synthesis or an atmosphere, not a uniform intensity.")
1021
+ cube_sim = None
1022
+ print("\nSkipping atmosphere loading (uniform intensity mode).")
1023
+ elif atmosphere_series_mode:
1024
+ # The cubes are synthesised per combination in the loop, since the
1025
+ # slit width and exposure time decide what the slit sees.
1026
+ cube_sim = None
1027
+ print("\nReading the atmosphere series...")
1028
+ series = AtmosphereSeries(series_paths)
1029
+ print(f" {len(series)} snapshots from {series.times[0]:.3f} to {series.times[-1]:.3f}")
1030
+ # Check the plan before computing the contribution functions, which
1031
+ # take minutes and gigabytes.
1032
+ RasterSynthesiser.check_plan(series, raster_plan, *_swept("slit_width", "expos"))
1033
+ raster = RasterSynthesiser(series, synthesis_settings)
1034
+ elif synthesis_series_mode:
1035
+ # The spectra are read per combination in the loop, a strip of
1036
+ # columns at a time, each column once.
1037
+ cube_sim = None
1038
+ print("\nReading the synthesis series...")
1039
+ series = SynthesisSeries(series_paths, reference_line)
1040
+ print(f" {len(series)} snapshots from {series.times[0]:.3f} to {series.times[-1]:.3f}")
1041
+ print(f" Lines in the window of {reference_line}: {', '.join(series.lines)}")
1042
+ SynthesisRaster.check_plan(series, raster_plan, *_swept("slit_width", "expos"))
1043
+ raster = SynthesisRaster(series)
1044
+ else:
1045
+ print(f"\nLoading the synthesis from {synthesis_file}...")
1046
+ print(f"Using '{reference_line}' as reference line for wavelength grid and metadata...")
1047
+ if synthesis_is_hdf5:
1048
+ # Read only the lines that reach the reference line's window and
1049
+ # sum them on its wavelengths. The sum is resampled onto the
1050
+ # detector grid per slit width in the loop.
1051
+ synthesis = read_synthesis(synthesis_file, reference_line)
1052
+ print(f" Lines in its window: {', '.join(synthesis.lines)}")
1053
+ if synthesis.source:
1054
+ print(f" Source: {synthesis.source}")
1055
+ products = read_synthesis_products(synthesis_file, keys=("dynamic_mode",))
1056
+ dynamic_mode_info = products.get("dynamic_mode", {"enabled": False})
1057
+ summed_input = synthesis.summed(reference_line)
1058
+ # Saved in the results as the spectra the instrument observed, if
1059
+ # their wavelengths are evenly spaced so a WCS can describe them.
1060
+ file_meta = {"dynamic_mode": dynamic_mode_info}
1061
+ cube_sim = (synthesis.summed_cube(reference_line, summed_input, file_meta)
1062
+ if synthesis.evenly_spaced(reference_line) else None)
1063
+ else:
1064
+ cube_sim, dynamic_mode_info = load_atmosphere(synthesis_file, reference_line)
1065
+
1066
+ is_dynamic_mode = dynamic_mode_info.get("enabled", False)
1067
+ if is_dynamic_mode:
1068
+ print("Synthesis was done in DYNAMIC MODE (time-varying atmosphere)")
1069
+ print(f" Slit width: {dynamic_mode_info['slit_width']}")
1070
+ print(f" Slit rest time: {dynamic_mode_info['slit_rest_time']}")
1071
+ print(f" Timesteps used: {len(dynamic_mode_info['available_timesteps'])}")
1072
+
1073
+ synth_slit_width = dynamic_mode_info["slit_width"]
1074
+ synth_rest_time = dynamic_mode_info["slit_rest_time"]
1075
+
1076
+ slit_width_vals = sweep_dims.get(
1077
+ "simulation.slit_width", [sim_fixed["slit_width"]]
1078
+ )
1079
+ expos_vals = sweep_dims.get(
1080
+ "simulation.expos", [sim_fixed["expos"]]
1081
+ )
1082
+
1083
+ if len(slit_width_vals) != 1:
1084
+ raise ValueError(
1085
+ f"Dynamic mode synthesis requires exactly one slit width. "
1086
+ f"Config specifies {len(slit_width_vals)}: {slit_width_vals}. "
1087
+ f"Please provide only the synthesis slit width: {synth_slit_width}"
1088
+ )
1089
+ if not np.isclose(
1090
+ slit_width_vals[0].to_value(u.arcsec),
1091
+ synth_slit_width.to_value(u.arcsec),
1092
+ rtol=1e-6,
1093
+ ):
1094
+ raise ValueError(
1095
+ f"Slit width mismatch: synthesis was done with {synth_slit_width}, "
1096
+ f"but config specifies {slit_width_vals[0]}."
1097
+ )
1098
+ if len(expos_vals) != 1:
1099
+ raise ValueError(
1100
+ f"Dynamic mode synthesis requires exactly one exposure time. "
1101
+ f"Config specifies {len(expos_vals)}: {expos_vals}. "
1102
+ f"Please provide only the synthesis slit rest time: {synth_rest_time}"
1103
+ )
1104
+ if not np.isclose(
1105
+ expos_vals[0].to_value(u.s),
1106
+ synth_rest_time.to_value(u.s),
1107
+ rtol=1e-6,
1108
+ ):
1109
+ raise ValueError(
1110
+ f"Exposure time mismatch: synthesis was done with {synth_rest_time}, "
1111
+ f"but config specifies {expos_vals[0]}."
1112
+ )
1113
+ print(" Dynamic mode parameters validated successfully!")
1114
+
1115
+ # Main sweep loop
1116
+ all_results = {}
1117
+ # cube_reb_cache: keyed by (slit_width_arcsec, plate_scale, wvl_res)
1118
+ cube_reb_cache = {}
1119
+ # rebin_cache: keyed by (slit_width_arcsec, plate_scale, wvl_res, offchip_bin_slit)
1120
+ rebin_cache = {}
1121
+ # observed_cache: cubes the Monte Carlo observes through the PSF, keyed by
1122
+ # sampling and PSF.
1123
+ observed_cache = {}
1124
+ # Synthesis-series raster, its metadata and its summed spectra, by sampling.
1125
+ raster_syntheses = {}
1126
+ # Keyed by slit_width_arcsec (first match) for convenient downstream access
1127
+ cube_reb_dict = {}
1128
+
1129
+ # Add offchip_bin_slit as a sweep dimension if needed
1130
+ if len(offchip_bin_slits) > 1:
1131
+ sweep_dims["offchip_bin_slit"] = offchip_bin_slits
1132
+ dim_names = list(sweep_dims.keys())
1133
+ dim_values = [sweep_dims[n] for n in dim_names]
1134
+ total_combinations = 1
1135
+ for v in dim_values:
1136
+ total_combinations *= len(v)
1137
+ print(f"\nUpdated to {total_combinations} parameter combination(s) (including offchip_bin_slit sweep).")
1138
+
1139
+ # Each raster of a time series is observed on its own, so repeated
1140
+ # rasters, or the exposures of a sit-and-stare, are swept over.
1141
+ if raster_mode and raster_plan.repeats > 1:
1142
+ sweep_dims["raster.repeat"] = list(range(raster_plan.repeats))
1143
+ dim_names = list(sweep_dims.keys())
1144
+ dim_values = [sweep_dims[n] for n in dim_names]
1145
+ total_combinations = 1
1146
+ for v in dim_values:
1147
+ total_combinations *= len(v)
1148
+ print(f"\nUpdated to {total_combinations} parameter combination(s) (one per raster repeat).")
1149
+
1150
+ def _combinations():
1151
+ """New iterator over the sweep's combinations, yielding one at a time."""
1152
+ return itertools_product(*dim_values) if dim_names else iter([()])
1153
+
1154
+ def _combination(combo_values):
1155
+ """Sweep values and config objects of one combination; the objects check the values."""
1156
+ combo = dict(zip(dim_names, combo_values)) if dim_names else {}
1157
+ offchip = combo.pop("offchip_bin_slit", offchip_bin_slits[0])
1158
+ repeat = combo.pop("raster.repeat", 0)
1159
+
1160
+ # Merge sweep values with fixed values for this combination
1161
+ all_sim = {
1162
+ **sim_fixed,
1163
+ **{k[len("simulation."):]: v for k, v in combo.items() if k.startswith("simulation.")},
1164
+ }
1165
+ all_det = {
1166
+ **det_fixed,
1167
+ **{k[len("detector."):]: v for k, v in combo.items() if k.startswith("detector.")},
1168
+ }
1169
+ all_det = {det_renamed.get(k, k): v for k, v in all_det.items()}
1170
+ all_tel = {
1171
+ **tel_fixed,
1172
+ **{k[len("telescope."):]: v for k, v in combo.items() if k.startswith("telescope.")},
1173
+ }
1174
+ all_fil = {
1175
+ **fil_fixed,
1176
+ **{k[len("filter."):]: v for k, v in combo.items() if k.startswith("filter.")},
1177
+ }
1178
+
1179
+ # Build config objects
1180
+ if instrument == "SWC":
1181
+ fil = AluminiumFilter(**all_fil) if all_fil else AluminiumFilter()
1182
+ tel_kwargs = {k: v for k, v in all_tel.items() if k != "filter"}
1183
+ tel_kwargs["filter"] = fil
1184
+ tel = Telescope_EUVST(**tel_kwargs)
1185
+ det = Detector_SWC(**all_det) if all_det else Detector_SWC()
1186
+ else:
1187
+ fil = None
1188
+ tel_kwargs = {k: v for k, v in all_tel.items() if k != "filter"}
1189
+ tel = Telescope_EIS(**tel_kwargs) if tel_kwargs else Telescope_EIS()
1190
+ det = Detector_EIS(**all_det) if all_det else Detector_EIS()
1191
+
1192
+ enable = all_sim.get("enable_pinholes", False)
1193
+ sim = Simulation(
1194
+ expos=all_sim["expos"],
1195
+ n_iter=n_iter,
1196
+ slit_width=all_sim["slit_width"],
1197
+ ncpu=ncpu,
1198
+ instrument=instrument,
1199
+ vis_sl=all_sim.get("vis_sl", 0.0 * u.photon / (u.s * u.cm**2)),
1200
+ psf=all_sim.get("psf", False),
1201
+ psf_boundary=all_sim.get("psf_boundary", "replicate"),
1202
+ spectral_psf=all_sim.get("spectral_psf", "quadrature"),
1203
+ slit_mask=all_sim.get("slit_mask", False),
1204
+ focal_plane_dispersion=all_sim.get("focal_plane_dispersion", False),
1205
+ noise=all_sim.get("noise", True),
1206
+ readout_dark_current=all_sim.get("readout_dark_current", True),
1207
+ enable_pinholes=enable,
1208
+ pinhole_sizes=pinhole_sizes if enable else [],
1209
+ pinhole_positions=pinhole_positions if enable else [],
1210
+ pinhole_positions_spectral=pinhole_positions_spectral if enable else [],
1211
+ pinhole_visible_wavelength=all_sim.get("pinhole_visible_wavelength",
1212
+ Simulation.pinhole_visible_wavelength),
1213
+ )
1214
+ # The optics' spectral FWHM depends on the telescope and the detector
1215
+ # together. Work it out here, not only once the scene is laid onto
1216
+ # the detector. It is needed when psf_params is for a given slit, and
1217
+ # always for the PSF convolved with the slit.
1218
+ if sim.focal_plane_dispersion:
1219
+ det = _with_focal_plane_width(det, all_det)
1220
+ if tel.psf_slit_width is not None or (sim.psf and sim.spectral_psf == "convolution"):
1221
+ spectral_optics_fwhm(tel, det)
1222
+ return combo, offchip, repeat, fil, tel, det, sim
1223
+
1224
+ def _with_focal_plane_width(det, all_det):
1225
+ """`det` with the focal plane's pixel width where the observed line lands."""
1226
+ if "wvl_res" in all_det:
1227
+ raise ValueError("simulation.focal_plane_dispersion takes the pixels' width from "
1228
+ "the focal plane, so detector.wvl_res would be ignored; give one "
1229
+ "or the other.")
1230
+ wavelength = _observed_wavelength()
1231
+ low, high = det.lit_band
1232
+ if not low <= wavelength <= high:
1233
+ raise ValueError(f"simulation.focal_plane_dispersion needs the observed line on the "
1234
+ f"focal plane, but {wavelength} is outside the lit band, "
1235
+ f"{low} to {high}.")
1236
+ try:
1237
+ width = FocalPlane_SWC.from_detector(det).pixel_width(wavelength)
1238
+ except ValueError as error:
1239
+ raise ValueError(f"simulation.focal_plane_dispersion needs the observed line on the "
1240
+ f"focal plane: {error}") from None
1241
+ return dataclasses.replace(det, wvl_res=width.to(det.wvl_res.unit))
1242
+
1243
+ observed_wavelength = []
1244
+
1245
+ def _observed_wavelength():
1246
+ """Rest wavelength of the line the run observes, read once."""
1247
+ if not observed_wavelength:
1248
+ if uniform_intensity_mode:
1249
+ found = uniform_rest_wavelength
1250
+ elif synthesis_series_mode or (not raster_mode and synthesis_is_hdf5):
1251
+ path = series_paths[0] if synthesis_series_mode else synthesis_file
1252
+ layout = read_synthesis_layout(path, reference_line)
1253
+ found = layout["lines"][reference_line]["rest_wavelength"]
1254
+ else:
1255
+ raise ValueError("simulation.focal_plane_dispersion needs a synthesis file or a "
1256
+ "uniform intensity.")
1257
+ observed_wavelength.append(u.Quantity(found).to(u.AA))
1258
+ return observed_wavelength[0]
1259
+
1260
+ # Check every combination before running any, so a bad value does not
1261
+ # stop the sweep part-way and lose the earlier results, which are saved
1262
+ # only at the end.
1263
+ for combo_values in _combinations():
1264
+ _, _, _, _, tel, _, _ = _combination(combo_values)
1265
+ # Read the throughput tables now too, so a missing or faulty table is
1266
+ # found before the first photons are counted. They cannot be swept, so
1267
+ # the last combination's tables are every combination's.
1268
+ if instrument == "SWC":
1269
+ for table in (tel.pm_table, tel.grating_table, tel.filter.al_table,
1270
+ tel.filter.oxide_table, tel.filter.c_table):
1271
+ _load_throughput_table(table)
1272
+
1273
+ def _check_offchip_bins(fov_along_slit):
1274
+ """Refuse an offchip_bin_slit that bins more rows than a combination's scene covers."""
1275
+ for combo_values in _combinations():
1276
+ _, n_bin, _, _, _, det, _ = _combination(combo_values)
1277
+ n_slit = _whole_pixels(fov_along_slit, det.plate_scale_angle)
1278
+ if n_bin > n_slit:
1279
+ raise ValueError(
1280
+ f"offchip_bin_slit {n_bin} bins more rows than the {n_slit} along the "
1281
+ f"slit, so there would be nothing left; bin at most {n_slit}.")
1282
+
1283
+ # The off-chip binning is checked against the scene when it is first
1284
+ # rebinned. A uniform intensity is made with the rows it bins.
1285
+ offchip_checked = uniform_intensity_mode
1286
+
1287
+ for combination_idx, combo_values in enumerate(_combinations(), start=1):
1288
+ (combo, offchip_bin_slit, raster_repeat, filter_obj, TEL, DET,
1289
+ SIM) = _combination(combo_values)
1290
+
1291
+ # Extract core simulation params
1292
+ slit_width = SIM.slit_width
1293
+ expos = SIM.expos
1294
+ vis_sl = SIM.vis_sl
1295
+ psf = SIM.psf
1296
+ slit_mask = SIM.slit_mask
1297
+ psf_boundary = SIM.psf_boundary
1298
+ spectral_psf = SIM.spectral_psf
1299
+ noise = SIM.noise
1300
+ enable_pinholes = SIM.enable_pinholes
1301
+
1302
+ # Two-level rebinning cache: rebin_atmosphere does not depend on offchip_bin_slit,
1303
+ # so cube_reb_cache is keyed by the sampling alone to avoid redundant rebin calls
1304
+ # when sweeping multiple binning values at fixed spatial/spectral sampling.
1305
+ # The spectral PSF's width sets how far the rebinned cube's wavelength
1306
+ # grid reaches, and it changes with the telescope's resolution budget,
1307
+ # which may be swept.
1308
+ sampling_key = (
1309
+ slit_width.to_value(u.arcsec),
1310
+ DET.plate_scale_angle.to_value(u.arcsec / u.pixel),
1311
+ DET.wvl_res.to_value(u.cm / u.pixel),
1312
+ spectral_psf_fwhm(TEL, DET, slit_width),
1313
+ )
1314
+ # In uniform-intensity mode the cube has offchip_bin_slit pixels along the
1315
+ # slit, giving rebin_slit_offchip independent noise realisations to sum, so
1316
+ # the key includes offchip_bin_slit. A time series is synthesised per
1317
+ # exposure time as well as per slit width.
1318
+ if uniform_intensity_mode:
1319
+ cube_reb_key = (*sampling_key, offchip_bin_slit)
1320
+ elif raster_mode:
1321
+ cube_reb_key = (*sampling_key, expos.to_value(u.s), raster_repeat)
1322
+ else:
1323
+ cube_reb_key = sampling_key
1324
+ rebin_cache_key = (*cube_reb_key, offchip_bin_slit)
1325
+
1326
+ if atmosphere_series_mode and cube_reb_key not in cube_reb_cache:
1327
+ print(f"\nSynthesising the time series as observed "
1328
+ f"(slit_width={slit_width}, expos={expos})...")
1329
+ cube_sim = raster.summed_cube(raster_plan, slit_width, expos, reference_line,
1330
+ repeat=raster_repeat)
1331
+ raster_summed[cube_reb_key] = cube_sim
1332
+ print(f" {cube_sim.data.shape[1]} exposures, {raster.strips_synthesised} "
1333
+ f"strips synthesised so far")
1334
+ if synthesis_series_mode and cube_reb_key not in cube_reb_cache:
1335
+ print(f"\nReading the time series as observed "
1336
+ f"(slit_width={slit_width}, expos={expos})...")
1337
+ # The exposures form a synthesis, one column each, which goes onto
1338
+ # the detector like a single snapshot.
1339
+ synthesis, raster_meta = raster.synthesis(raster_plan, slit_width, expos,
1340
+ repeat=raster_repeat)
1341
+ summed_input = synthesis.summed(reference_line)
1342
+ cube_sim = None
1343
+ if synthesis.evenly_spaced(reference_line):
1344
+ cube_sim = synthesis.summed_cube(reference_line, summed_input, raster_meta)
1345
+ raster_summed[cube_reb_key] = cube_sim
1346
+ raster_syntheses[cube_reb_key] = (synthesis, raster_meta, summed_input)
1347
+ print(f" {len(raster_meta['positions'])} exposures, {raster.strips_read} "
1348
+ f"strips read so far")
1349
+
1350
+ if cube_reb_key not in cube_reb_cache:
1351
+ print(
1352
+ f"\nRebinning atmosphere "
1353
+ f"(slit_width={slit_width}, "
1354
+ f"plate_scale={DET.plate_scale_angle}, "
1355
+ f"wvl_res={DET.wvl_res})..."
1356
+ )
1357
+ SIM_rebin = Simulation(
1358
+ expos=1.0 * u.s,
1359
+ n_iter=n_iter,
1360
+ slit_width=slit_width,
1361
+ ncpu=ncpu,
1362
+ instrument=instrument,
1363
+ psf=False,
1364
+ )
1365
+ if uniform_intensity_mode:
1366
+ cube_reb_cache[cube_reb_key] = create_uniform_intensity_cube(
1367
+ total_intensity=uniform_intensity,
1368
+ rest_wavelength=uniform_rest_wavelength,
1369
+ thermal_width=uniform_thermal_width,
1370
+ det=DET,
1371
+ sim=SIM_rebin,
1372
+ n_slit_pixels=offchip_bin_slit,
1373
+ tel=TEL,
1374
+ )
1375
+ else:
1376
+ # A slit wider than the one psf_params is for spreads each line
1377
+ # beyond the synthesis window's margin, so the window is padded
1378
+ # to hold it. The reference slit's window is unchanged.
1379
+ if synthesis is None:
1380
+ rebinned = rebin_atmosphere(cube_sim, DET, SIM_rebin)
1381
+ else:
1382
+ rebinned = rebin_spectra(
1383
+ synthesis, reference_line, DET, SIM_rebin, summed=summed_input,
1384
+ meta=raster_meta if synthesis_series_mode else file_meta)
1385
+ cube_reb_cache[cube_reb_key] = pad_spectral_axis(
1386
+ rebinned, spectral_psf_margin(TEL, DET, slit_width))
1387
+ if not offchip_checked:
1388
+ # The scene has the same length along the slit for every
1389
+ # combination, so each one's number of rows is known now,
1390
+ # before any has run.
1391
+ _check_offchip_bins(cube_reb_cache[cube_reb_key].meta["fov_along_slit"])
1392
+ offchip_checked = True
1393
+
1394
+ cube_reb = cube_reb_cache[cube_reb_key]
1395
+
1396
+ if rebin_cache_key not in rebin_cache:
1397
+ # Apply off-chip binning
1398
+ cube_reb_binned = rebin_slit_offchip(cube_reb, offchip_bin_slit)
1399
+
1400
+ # A weighted ground-truth fit uses the expected DN uncertainties,
1401
+ # which depend on the exposure and the instrument, so it is
1402
+ # fitted for each combination below.
1403
+ ground_truth = (None if weighted_truth
1404
+ else _fit_ground_truth(cube_reb_binned, offchip_bin_slit))
1405
+ rebin_cache[rebin_cache_key] = (cube_reb_binned, ground_truth)
1406
+ # Key by (slit_width_arcsec, offchip_bin_slit) so sweeps over
1407
+ # several binning factors at a fixed slit width keep all their
1408
+ # cubes; a single key would silently keep only the first. A time
1409
+ # series adds the exposure time, which changes the cube too.
1410
+ cube_key = ((sampling_key[0], expos.to_value(u.s), raster_repeat, offchip_bin_slit)
1411
+ if raster_mode else (sampling_key[0], offchip_bin_slit))
1412
+ cube_reb_dict.setdefault(cube_key, cube_reb_binned)
1413
+ if raster_mode:
1414
+ raster_cubes[cube_key] = raster_summed[cube_reb_key]
1415
+
1416
+ cube_reb_binned, ground_truth = rebin_cache[rebin_cache_key]
1417
+
1418
+ # The cube the Monte Carlo observes: the scene laid onto the pixels
1419
+ # through the telescope, with each photon counted at its own
1420
+ # wavelength. With the PSF on, the scene is also blurred by the PSF
1421
+ # before it is laid onto the pixels. This is exact, whereas blurring
1422
+ # the pixels afterwards moves a narrow line within its pixel. With
1423
+ # slit_mask on, the slit's ends and fiducials are cut out in the same
1424
+ # step. The ground truth is the scene without any of these. A uniform
1425
+ # intensity is made again through the spectral response, which
1426
+ # depends on the slit, the telescope and spectral_psf.
1427
+ cube_obs = cube_reb
1428
+ # The observed cube includes the telescope's throughput as well as its
1429
+ # PSF, so every telescope and filter setting is in the key, except the
1430
+ # one for visible light, which the Monte Carlo adds. With the pinholes
1431
+ # on, it also includes the light they let past the filter.
1432
+ filter_params = (_extract_config_params(filter_obj, "filter")
1433
+ if filter_obj is not None else {})
1434
+ filter_params.pop("filter.visible_decade_thickness", None)
1435
+ telescope_key = _params_to_key({**_extract_config_params(TEL, "telescope"),
1436
+ **filter_params})
1437
+ pinholes = _pinholes_on(SIM)
1438
+ if uniform_intensity_mode:
1439
+ if psf:
1440
+ psf_key = (*cube_reb_key, spectral_psf, telescope_key, pinholes)
1441
+ if psf_key not in observed_cache:
1442
+ observed_cache[psf_key] = create_uniform_intensity_cube(
1443
+ total_intensity=uniform_intensity,
1444
+ rest_wavelength=uniform_rest_wavelength,
1445
+ thermal_width=uniform_thermal_width,
1446
+ det=DET,
1447
+ sim=SIM,
1448
+ n_slit_pixels=offchip_bin_slit,
1449
+ tel=TEL,
1450
+ )
1451
+ cube_obs = observed_cache[psf_key]
1452
+ else:
1453
+ psf_key = (*cube_reb_key, psf, slit_mask, spectral_psf, psf_boundary, telescope_key,
1454
+ pinholes)
1455
+ if psf_key not in observed_cache:
1456
+ print("Laying the scene onto the detector through the telescope...")
1457
+ SIM_obs = Simulation(expos=1.0 * u.s, n_iter=n_iter, slit_width=slit_width,
1458
+ ncpu=ncpu, instrument=instrument, psf=psf,
1459
+ psf_boundary=psf_boundary, spectral_psf=spectral_psf,
1460
+ slit_mask=slit_mask,
1461
+ enable_pinholes=SIM.enable_pinholes,
1462
+ pinhole_sizes=list(SIM.pinhole_sizes),
1463
+ pinhole_positions=list(SIM.pinhole_positions),
1464
+ pinhole_positions_spectral=list(
1465
+ SIM.pinhole_positions_spectral))
1466
+ across = across_slit_fwhm(TEL) if psf else None
1467
+ extend = psf_boundary == "replicate"
1468
+ if atmosphere_series_mode:
1469
+ scene = raster_summed[cube_reb_key] if across is None else raster.summed_cube(
1470
+ raster_plan, slit_width, expos, reference_line, repeat=raster_repeat,
1471
+ across_slit=across, extend=extend)
1472
+ observed = rebin_atmosphere(scene, DET, SIM_obs, tel=TEL)
1473
+ elif synthesis_series_mode:
1474
+ series_synthesis, series_meta, series_summed = raster_syntheses[cube_reb_key]
1475
+ if across is not None:
1476
+ series_synthesis, series_meta = raster.synthesis(
1477
+ raster_plan, slit_width, expos, repeat=raster_repeat,
1478
+ across_slit=across, extend=extend)
1479
+ series_summed = None
1480
+ observed = rebin_spectra(series_synthesis, reference_line, DET, SIM_obs,
1481
+ summed=series_summed, meta=series_meta, tel=TEL)
1482
+ elif synthesis is None:
1483
+ observed = rebin_atmosphere(cube_sim, DET, SIM_obs, tel=TEL)
1484
+ else:
1485
+ observed = rebin_spectra(synthesis, reference_line, DET, SIM_obs,
1486
+ summed=summed_input, meta=file_meta, tel=TEL)
1487
+ observed_cache[psf_key] = observed
1488
+ cube_obs = observed_cache[psf_key]
1489
+
1490
+ # Progress output
1491
+ print(f"\n--- Combination {combination_idx}/{total_combinations} ---")
1492
+ for k, v in combo.items():
1493
+ print(f" {k}: {v}")
1494
+ if offchip_bin_slit > 1:
1495
+ print(f" offchip_bin_slit: {offchip_bin_slit}")
1496
+ if raster_mode and raster_plan.repeats > 1:
1497
+ print(f" raster.repeat: {raster_repeat}")
1498
+ if not combo and offchip_bin_slit == 1 and not (raster_mode and raster_plan.repeats > 1):
1499
+ print(" (single combination - all parameters fixed)")
1500
+ print(f" Calculated dark current: {DET.dark_current:.2e}")
1501
+ if instrument == "SWC":
1502
+ print(f" Microroughness sigma: {TEL.microroughness_sigma}")
1503
+ if enable_pinholes and pinhole_sizes:
1504
+ print(f" Pinhole sizes: {pinhole_sizes}")
1505
+ print(f" Pinhole positions: {pinhole_positions}")
1506
+
1507
+ if weighted_truth:
1508
+ # Weight the ground truth by the DN uncertainty expected with no
1509
+ # noise, so it is the same fit, of the same line, as for the
1510
+ # measured spectra.
1511
+ expected = expected_dn_uncertainty(cube_obs, expos, DET, TEL, SIM, offchip_bin_slit,
1512
+ uniform_mode=uniform_intensity_mode)
1513
+ if expected.shape != cube_reb_binned.data.shape:
1514
+ raise RuntimeError(
1515
+ f"The detector's pixels, shaped {expected.shape}, do not match the "
1516
+ f"spectra the ground truth is fitted to, shaped "
1517
+ f"{cube_reb_binned.data.shape}.")
1518
+ ground_truth = _fit_ground_truth(cube_reb_binned, offchip_bin_slit, expected)
1519
+
1520
+ # Run Monte Carlo
1521
+ first_dn_signal, dn_fit_stats, first_photon_signal, photon_fit_stats = monte_carlo(
1522
+ cube_obs, expos, DET, TEL, SIM,
1523
+ n_iter=SIM.n_iter,
1524
+ fit_config=fit_config,
1525
+ offchip_bin_slit=offchip_bin_slit,
1526
+ fit_signals=fit_signals,
1527
+ uniform_mode=uniform_intensity_mode,
1528
+ )
1529
+
1530
+ # Only store results on rank 0 (MPI-aware)
1531
+ if _mpi_rank == 0 and first_dn_signal is not None:
1532
+ # Build parameters dict from actual config objects so all fields
1533
+ # (including those using class defaults) are recorded.
1534
+ parameters = {}
1535
+ parameters.update(_extract_config_params(SIM, "simulation"))
1536
+ parameters.update(_extract_config_params(DET, "detector"))
1537
+ if instrument == "SWC":
1538
+ parameters.update(_extract_config_params(filter_obj, "filter"))
1539
+ parameters.update(_extract_config_params(TEL, "telescope"))
1540
+ # Add offchip_bin_slit to the parameters dict
1541
+ parameters["offchip_bin_slit"] = offchip_bin_slit
1542
+ if raster_mode:
1543
+ parameters["raster.repeat"] = raster_repeat
1544
+
1545
+ param_key = _params_to_key(parameters)
1546
+
1547
+ all_results[param_key] = {
1548
+ "parameters": parameters,
1549
+ "config_objects": {
1550
+ "detector": DET,
1551
+ "telescope": TEL,
1552
+ "simulation": SIM,
1553
+ },
1554
+ "first_dn_signal_data": first_dn_signal.data,
1555
+ "first_dn_signal_unit": first_dn_signal.unit,
1556
+ "first_photon_signal_data": first_photon_signal.data,
1557
+ "first_photon_signal_unit": first_photon_signal.unit,
1558
+ "first_signal_wcs": first_dn_signal.wcs,
1559
+ "dn_fit_stats": dn_fit_stats,
1560
+ "photon_fit_stats": photon_fit_stats,
1561
+ "ground_truth": ground_truth,
1562
+ }
1563
+
1564
+ del first_dn_signal, first_photon_signal, dn_fit_stats, photon_fit_stats
1565
+
1566
+ # Package results (rank 0 only in MPI mode)
1567
+ if _mpi_rank == 0:
1568
+ results = {
1569
+ "all_combinations": all_results,
1570
+ # All sweep dimensions and their value lists
1571
+ "sweep_dimensions": sweep_dims,
1572
+ # All fixed (non-swept) parameter values
1573
+ "fixed_params": {
1574
+ **{f"simulation.{k}": v for k, v in sim_fixed.items()},
1575
+ **{f"detector.{k}": v for k, v in det_fixed.items()},
1576
+ **{f"telescope.{k}": v for k, v in tel_fixed.items()},
1577
+ **(
1578
+ {f"filter.{k}": v for k, v in fil_fixed.items()}
1579
+ if instrument == "SWC"
1580
+ else {}
1581
+ ),
1582
+ "offchip_bin_slit": offchip_bin_slits[0] if len(offchip_bin_slits) == 1 else None,
1583
+ **({"raster.repeat": 0} if raster_mode and raster_plan.repeats == 1 else {}),
1584
+ },
1585
+ "fit_config": fit_config,
1586
+ "fit_signals": fit_signals,
1587
+ # Whether the fits were weighted: DN by each pixel's uncertainty,
1588
+ # photons by their Poisson likelihood. Results made before this was
1589
+ # recorded were unweighted.
1590
+ "fit_weighted": weighted_fits,
1591
+ }
1592
+
1593
+ # Save
1594
+ git_commit_id = get_git_commit_id()
1595
+ software_version = _get_software_version()
1596
+
1597
+ output_file = Path(f"run/result/{Path(args.config).stem}.h5")
1598
+
1599
+ print(f"\nSaving results to {output_file}")
1600
+ save_data = {
1601
+ "results": results,
1602
+ "config": config,
1603
+ "instrument": instrument,
1604
+ "cube_sim": cube_sim,
1605
+ "cube_reb_dict": cube_reb_dict,
1606
+ "git_commit_id": git_commit_id,
1607
+ "software_version": software_version,
1608
+ }
1609
+ if raster_mode:
1610
+ # How the series was observed, and the cube each combination saw;
1611
+ # cube_sim is the last combination's.
1612
+ save_data["raster"] = {
1613
+ "plan": raster_plan,
1614
+ "settings": synthesis_settings,
1615
+ "series": [str(p) for p in series.paths],
1616
+ "times": series.times,
1617
+ "hdf5_dbase_root": raster.goft_dbase_root if atmosphere_series_mode else None,
1618
+ "cubes": raster_cubes,
1619
+ }
1620
+
1621
+ output_file = save_results(output_file, save_data)
1622
+
1623
+ print(f"Saved results to {output_file} ({os.path.getsize(output_file) / 1e6:.1f} MB)")
1624
+ # Move aside a results pickle of the same name left by an older
1625
+ # version, so it is not read in place of these results.
1626
+ stale = output_file.with_suffix(".pkl")
1627
+ if stale.is_file():
1628
+ # Do not overwrite a file moved aside before, which may be the
1629
+ # only copy of even older results.
1630
+ aside = stale.with_name(stale.name + ".old")
1631
+ number = 0
1632
+ while aside.exists():
1633
+ number += 1
1634
+ aside = stale.with_name(f"{stale.name}.old.{number}")
1635
+ try:
1636
+ os.replace(stale, aside)
1637
+ print(f"Moved {stale}, from an older version, to {aside.name}")
1638
+ except OSError as error:
1639
+ print(f"Could not move {stale}, from an older version, aside ({error}); "
1640
+ f"the results are {output_file}, not it")
1641
+ print(f"Software version: {software_version} | Git commit: {git_commit_id}")
1642
+ print(f"Instrument response simulation complete! Total combinations: {total_combinations}")
1643
+
1644
+
1645
+ if __name__ == "__main__":
1646
+ main()