eclipse 0.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eclipse/__init__.py +95 -0
- eclipse/analysis.py +853 -0
- eclipse/atmosphere.py +859 -0
- eclipse/band.py +477 -0
- eclipse/cli.py +138 -0
- eclipse/config.py +1710 -0
- eclipse/continuum.py +413 -0
- eclipse/data/eis_response/EIS_EffArea_A.004 +61 -0
- eclipse/data/eis_response/EIS_EffArea_B.004 +60 -0
- eclipse/data/eis_response/EIS_EffArea_B.005 +64 -0
- eclipse/data/eis_response/README_dz2025 +50 -0
- eclipse/data/eis_response/fit_eis_ea_2023-05-04_smooth.sav +0 -0
- eclipse/data/eis_response/source.txt +33 -0
- eclipse/data/science_cases.yaml +365 -0
- eclipse/data/throughput/grating_reflection_efficiency.dat +25 -0
- eclipse/data/throughput/primary_mirror_coating_reflectance.dat +25 -0
- eclipse/data/throughput/source.txt +6 -0
- eclipse/data/throughput/throughput_aluminium_1000_angstrom.dat +503 -0
- eclipse/data/throughput/throughput_aluminium_oxide_1000_angstrom.dat +503 -0
- eclipse/data/throughput/throughput_carbon_1000_angstrom.dat +503 -0
- eclipse/data_processing.py +904 -0
- eclipse/deprecated.py +161 -0
- eclipse/eis_calibration.py +573 -0
- eclipse/extern/__init__.py +0 -0
- eclipse/extern/mpfit.py +2388 -0
- eclipse/fitting.py +1734 -0
- eclipse/frame.py +594 -0
- eclipse/full_frame.py +944 -0
- eclipse/main.py +1646 -0
- eclipse/monte_carlo.py +660 -0
- eclipse/pinhole_diffraction.py +409 -0
- eclipse/psf.py +6 -0
- eclipse/radiometric.py +1328 -0
- eclipse/raster.py +869 -0
- eclipse/readout.py +1002 -0
- eclipse/resolution_budget.py +108 -0
- eclipse/results_file.py +1226 -0
- eclipse/sampling.py +351 -0
- eclipse/science_cases.py +364 -0
- eclipse/synthesis.py +2492 -0
- eclipse/synthesis_cli.py +12 -0
- eclipse/synthesis_file.py +882 -0
- eclipse/utils.py +1145 -0
- eclipse-0.13.0.dist-info/METADATA +70 -0
- eclipse-0.13.0.dist-info/RECORD +50 -0
- eclipse-0.13.0.dist-info/WHEEL +5 -0
- eclipse-0.13.0.dist-info/entry_points.txt +7 -0
- eclipse-0.13.0.dist-info/licenses/LICENSE +3 -0
- eclipse-0.13.0.dist-info/top_level.txt +2 -0
- euvst_response/__init__.py +129 -0
eclipse/main.py
ADDED
|
@@ -0,0 +1,1646 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Main execution script for instrument response simulations.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
import argparse
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import warnings
|
|
10
|
+
from itertools import product as itertools_product
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import yaml
|
|
13
|
+
import astropy.units as u
|
|
14
|
+
import gzip
|
|
15
|
+
import h5py
|
|
16
|
+
|
|
17
|
+
from .config import AluminiumFilter, Detector_SWC, Detector_EIS, Telescope_EUVST, Telescope_EIS, Simulation, check_pinhole_lists, _load_throughput_table
|
|
18
|
+
from .config import _FOCAL_PLANE_SETTINGS, _old_names_used, _same_setting
|
|
19
|
+
from .data_processing import (_whole_pixels, load_atmosphere, rebin_atmosphere,
|
|
20
|
+
create_uniform_intensity_cube, pad_spectral_axis, rebin_spectra)
|
|
21
|
+
from .raster import AtmosphereSeries, RasterSynthesiser, SynthesisRaster, SynthesisSeries
|
|
22
|
+
from .atmosphere import _is_pickle
|
|
23
|
+
from .readout import FocalPlane_SWC
|
|
24
|
+
from .synthesis_file import (is_synthesis_file, read_synthesis, read_synthesis_layout,
|
|
25
|
+
read_synthesis_products, synthesis_line_names)
|
|
26
|
+
from .fitting import FitConfig, FitComponent, ground_truth_summary
|
|
27
|
+
from .results_file import _to_json, save_results
|
|
28
|
+
from .monte_carlo import expected_dn_uncertainty, monte_carlo
|
|
29
|
+
from .pinhole_diffraction import _pinholes_on
|
|
30
|
+
from .radiometric import (across_slit_fwhm, spectral_optics_fwhm, spectral_psf_fwhm,
|
|
31
|
+
spectral_psf_margin)
|
|
32
|
+
from .utils import (
|
|
33
|
+
parse_yaml_input, ensure_list, set_debug_mode, debug_break, debug_on_error,
|
|
34
|
+
deduplicate_list, get_git_commit_id, _get_software_version,
|
|
35
|
+
_parse_section, _params_to_key, _extract_config_params, _SECTION_LIST_FIELDS,
|
|
36
|
+
rebin_slit_offchip, check_config_keys, load_yaml_config,
|
|
37
|
+
)
|
|
38
|
+
import dataclasses
|
|
39
|
+
import numpy as np
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# Every top-level key main() reads. Any other key is rejected rather than
|
|
43
|
+
# ignored.
|
|
44
|
+
_TOP_LEVEL_KEYS = {
|
|
45
|
+
"instrument", "n_iter", "ncpu",
|
|
46
|
+
"uniform_intensity", "rest_wavelength", "thermal_width",
|
|
47
|
+
"synthesis_file", "reference_line",
|
|
48
|
+
"atmosphere_series", "synthesis_series", "synthesis", "raster",
|
|
49
|
+
"pinhole_sizes", "pinhole_positions", "pinhole_positions_spectral",
|
|
50
|
+
"offchip_bin_slit", "fit_signals",
|
|
51
|
+
"simulation", "detector", "telescope", "filter", "fitting",
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
# Keys of the 'synthesis:' section, the settings an atmosphere time series is
|
|
55
|
+
# synthesised with (as in synthesise-spectra), and of the 'raster:' section,
|
|
56
|
+
# the observing plan of an atmosphere or synthesis time series
|
|
57
|
+
# (eclipse.raster.RasterPlan).
|
|
58
|
+
_SYNTHESIS_KEYS = {"lines", "abundance", "vel_res", "vel_lim", "crop_y", "crop_z",
|
|
59
|
+
"precision", "mass_per_electron", "hdf5_dbase_root", "n_workers",
|
|
60
|
+
"goft_temperature_chunk"}
|
|
61
|
+
_RASTER_KEYS = {"start", "steps", "step", "repeats", "cadence", "centre", "direction"}
|
|
62
|
+
|
|
63
|
+
# Under MPI, a copy of a non-root rank's original stderr, taken before its
|
|
64
|
+
# output is silenced.
|
|
65
|
+
_STARTING_STDERR = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _cpu_list(text: str) -> list:
|
|
69
|
+
"""CPU numbers in a Linux CPU list such as `0-3,8,10-11`."""
|
|
70
|
+
cpus = []
|
|
71
|
+
for part in text.strip().split(","):
|
|
72
|
+
if part:
|
|
73
|
+
first, _, last = part.partition("-")
|
|
74
|
+
cpus.extend(range(int(first), int(last or first) + 1))
|
|
75
|
+
return cpus
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _cpuset():
|
|
79
|
+
"""CPUs allowed by this process's cgroup cpuset, or None if there is none to read."""
|
|
80
|
+
try:
|
|
81
|
+
for line in Path("/proc/self/cgroup").read_text().splitlines():
|
|
82
|
+
hierarchy, controllers, path = line.split(":", 2)
|
|
83
|
+
if hierarchy == "0" or "cpuset" in controllers.split(","):
|
|
84
|
+
root = Path("/sys/fs/cgroup") if hierarchy == "0" else Path("/sys/fs/cgroup/cpuset")
|
|
85
|
+
for name in ("cpuset.cpus.effective", "cpuset.cpus"):
|
|
86
|
+
listed = root / path.lstrip("/") / name
|
|
87
|
+
if listed.is_file() and listed.read_text().strip():
|
|
88
|
+
return _cpu_list(listed.read_text())
|
|
89
|
+
except (OSError, ValueError):
|
|
90
|
+
pass
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _allowed_cpus(environ, cpuset, affinity) -> list:
|
|
95
|
+
"""
|
|
96
|
+
CPUs this job step has on its node, for its ranks to share.
|
|
97
|
+
|
|
98
|
+
SLURM confines a step with a cpuset, which may be wider than the
|
|
99
|
+
`affinity` an MPI library gave a rank. The `cpuset` is used when it is
|
|
100
|
+
the step's: it has no more CPUs than SLURM_CPUS_ON_NODE and contains the
|
|
101
|
+
affinity. Otherwise the affinity is kept. Where SLURM does not confine
|
|
102
|
+
steps, the cpuset is the whole node and could include another job's
|
|
103
|
+
CPUs. The affinity is also kept when there is no cpuset or
|
|
104
|
+
SLURM_CPUS_ON_NODE is not set.
|
|
105
|
+
"""
|
|
106
|
+
on_node = environ.get("SLURM_CPUS_ON_NODE")
|
|
107
|
+
if (cpuset and on_node is not None and on_node.isdigit()
|
|
108
|
+
and len(cpuset) <= int(on_node) and set(affinity) <= set(cpuset)):
|
|
109
|
+
return list(cpuset)
|
|
110
|
+
return list(affinity)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _rank_cpus(environ, allowed: list) -> list:
|
|
114
|
+
"""
|
|
115
|
+
CPUs for this MPI rank: its share of `allowed`, by local task id.
|
|
116
|
+
|
|
117
|
+
Each task gets SLURM_CPUS_PER_TASK of `allowed`, in order. All of
|
|
118
|
+
`allowed` is returned if there are too few for every task on the node
|
|
119
|
+
to have its own. An empty list, which leaves the affinity alone, is
|
|
120
|
+
returned if SLURM does not say how many CPUs a task has.
|
|
121
|
+
"""
|
|
122
|
+
per_task, local = environ.get("SLURM_CPUS_PER_TASK"), environ.get("SLURM_LOCALID")
|
|
123
|
+
if per_task is None or local is None:
|
|
124
|
+
return []
|
|
125
|
+
per_task, local = int(per_task), int(local)
|
|
126
|
+
share = allowed[local * per_task:(local + 1) * per_task]
|
|
127
|
+
return share if len(share) == per_task else list(allowed)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# main() builds the Simulation objects itself and takes only these keys from
|
|
131
|
+
# the 'simulation:' section. The other Simulation fields (instrument, n_iter,
|
|
132
|
+
# ncpu and the pinhole lists) are top-level keys; one written here would be
|
|
133
|
+
# parsed and then dropped.
|
|
134
|
+
_SIMULATION_KEYS = {"slit_width", "expos", "vis_sl", "psf", "psf_boundary",
|
|
135
|
+
"spectral_psf", "slit_mask", "focal_plane_dispersion", "noise",
|
|
136
|
+
"readout_dark_current", "enable_pinholes", "pinhole_visible_wavelength"}
|
|
137
|
+
|
|
138
|
+
_FITTING_KEYS = {"components", "primary_component",
|
|
139
|
+
"constrain_positive_intensity", "backend", "max_iter",
|
|
140
|
+
"bessel_correction", "save_iterations", "weighted"}
|
|
141
|
+
_FITTING_COMPONENT_KEYS = {"wavelength", "tie_center", "tie_width",
|
|
142
|
+
"amplitude_greater_than", "name"}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _dataclass_keys(cls) -> set:
|
|
146
|
+
"""Constructor argument names of a config dataclass."""
|
|
147
|
+
return {f.name for f in dataclasses.fields(cls) if f.init}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _type_name(value) -> str:
|
|
151
|
+
"""Type of a config value for an error message; an empty YAML entry is None."""
|
|
152
|
+
return "nothing" if value is None else type(value).__name__
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _require_mapping(value, what: str) -> None:
|
|
156
|
+
"""Raise unless a config section or fitting component is a mapping."""
|
|
157
|
+
if not isinstance(value, dict):
|
|
158
|
+
raise ValueError(
|
|
159
|
+
f"{what} must be a mapping of parameter names to values, "
|
|
160
|
+
f"got {_type_name(value)}."
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _validate_config_keys(config: dict, instrument: str) -> None:
|
|
165
|
+
"""
|
|
166
|
+
Reject any config key ECLIPSE does not read.
|
|
167
|
+
|
|
168
|
+
Called before anything is loaded or computed, so a config written for an
|
|
169
|
+
older layout fails at once rather than after the atmosphere is loaded.
|
|
170
|
+
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
config : dict
|
|
174
|
+
Whole parsed YAML config.
|
|
175
|
+
instrument : str
|
|
176
|
+
`"SWC"` or `"EIS"`, which have different detector and telescope
|
|
177
|
+
parameters.
|
|
178
|
+
|
|
179
|
+
Raises
|
|
180
|
+
------
|
|
181
|
+
ValueError
|
|
182
|
+
If the instrument is not supported, a key is not read, or a section
|
|
183
|
+
that is present is not a mapping.
|
|
184
|
+
"""
|
|
185
|
+
# Checked first because the valid keys depend on it, and main() treats
|
|
186
|
+
# anything other than SWC as EIS.
|
|
187
|
+
if instrument not in ("SWC", "EIS"):
|
|
188
|
+
raise ValueError(
|
|
189
|
+
f"Unknown instrument '{instrument}'. Supported values: 'SWC', 'EIS'."
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
det_class = Detector_EIS if instrument == "EIS" else Detector_SWC
|
|
193
|
+
# Old names, such as _dark_current_293k, still work, with a FutureWarning.
|
|
194
|
+
det_keys = _dataclass_keys(det_class) | set(det_class._OLD_NAMES)
|
|
195
|
+
tel_keys = _dataclass_keys(Telescope_EIS if instrument == "EIS"
|
|
196
|
+
else Telescope_EUVST)
|
|
197
|
+
fil_keys = _dataclass_keys(AluminiumFilter)
|
|
198
|
+
|
|
199
|
+
# 'filter' is a Telescope_EUVST field, but main() builds the filter from
|
|
200
|
+
# the top-level 'filter:' section and would drop a value given here.
|
|
201
|
+
tel_keys.discard("filter")
|
|
202
|
+
if instrument == "EIS":
|
|
203
|
+
# Allowed here; main() warns about it and ignores it.
|
|
204
|
+
tel_keys.add("microroughness_sigma")
|
|
205
|
+
|
|
206
|
+
sections = {
|
|
207
|
+
"simulation": _SIMULATION_KEYS,
|
|
208
|
+
"detector": det_keys,
|
|
209
|
+
"telescope": tel_keys,
|
|
210
|
+
"filter": fil_keys,
|
|
211
|
+
}
|
|
212
|
+
# Other places a key could belong, for the suggestions. The top level is
|
|
213
|
+
# included so a section key written at the wrong depth is reported as such.
|
|
214
|
+
elsewhere = {"": _TOP_LEVEL_KEYS, **sections}
|
|
215
|
+
|
|
216
|
+
check_config_keys(config, _TOP_LEVEL_KEYS, "top-level", sections)
|
|
217
|
+
|
|
218
|
+
# Checked by presence, not value, so an empty heading (which parses to
|
|
219
|
+
# None) gets a clear message here rather than an AttributeError later.
|
|
220
|
+
for name, allowed in sections.items():
|
|
221
|
+
if name not in config:
|
|
222
|
+
continue
|
|
223
|
+
_require_mapping(config[name], f"The '{name}:' section")
|
|
224
|
+
others = {k: v for k, v in elsewhere.items() if k != name}
|
|
225
|
+
check_config_keys(config[name], allowed, f"'{name}' section", others)
|
|
226
|
+
|
|
227
|
+
for name, allowed in (("synthesis", _SYNTHESIS_KEYS), ("raster", _RASTER_KEYS)):
|
|
228
|
+
if name in config:
|
|
229
|
+
_require_mapping(config[name], f"The '{name}:' section")
|
|
230
|
+
check_config_keys(config[name], allowed, f"'{name}' section")
|
|
231
|
+
|
|
232
|
+
if "fitting" in config:
|
|
233
|
+
fitting = config["fitting"]
|
|
234
|
+
_require_mapping(fitting, "The 'fitting:' section")
|
|
235
|
+
check_config_keys(fitting, _FITTING_KEYS, "'fitting' section")
|
|
236
|
+
if "components" in fitting:
|
|
237
|
+
components = fitting["components"]
|
|
238
|
+
if not isinstance(components, list):
|
|
239
|
+
raise ValueError(
|
|
240
|
+
f"'fitting.components' must be a list with one entry per "
|
|
241
|
+
f"Gaussian component, got {_type_name(components)}."
|
|
242
|
+
)
|
|
243
|
+
for idx, component in enumerate(components):
|
|
244
|
+
where = f"'fitting.components[{idx}]'"
|
|
245
|
+
_require_mapping(component, where)
|
|
246
|
+
check_config_keys(component, _FITTING_COMPONENT_KEYS, where)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _quantity_or_none(section: dict, key: str, unit, what: str):
|
|
250
|
+
"""Quantity in units equivalent to `unit` from a config section, or None if absent."""
|
|
251
|
+
if key not in section or section[key] is None:
|
|
252
|
+
return None
|
|
253
|
+
value = parse_yaml_input(section[key])
|
|
254
|
+
if not isinstance(value, u.Quantity) or not value.unit.is_equivalent(unit):
|
|
255
|
+
raise ValueError(f"'{what}.{key}' must be {unit.physical_type} with units, "
|
|
256
|
+
f"e.g. '{section[key]}' is not; got {section[key]!r}.")
|
|
257
|
+
return value
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _range_or_none(section: dict, key: str, what: str):
|
|
261
|
+
"""`[low, high]` range of lengths from a config section, or None if absent."""
|
|
262
|
+
if key not in section or section[key] is None:
|
|
263
|
+
return None
|
|
264
|
+
bounds = section[key]
|
|
265
|
+
if not isinstance(bounds, (list, tuple)) or len(bounds) != 2:
|
|
266
|
+
raise ValueError(f"'{what}.{key}' must be a list of two lengths with units, "
|
|
267
|
+
f"e.g. ['0 Mm', '20 Mm'], got {bounds!r}.")
|
|
268
|
+
low, high = (parse_yaml_input(b) for b in bounds)
|
|
269
|
+
for value in (low, high):
|
|
270
|
+
if not isinstance(value, u.Quantity) or not value.unit.is_equivalent(u.Mm):
|
|
271
|
+
raise ValueError(f"'{what}.{key}' must hold lengths with units, got {bounds!r}.")
|
|
272
|
+
return (low, high)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _parse_synthesis_settings(config: dict):
|
|
276
|
+
"""Synthesis settings for a time series, from the 'synthesis:' section."""
|
|
277
|
+
from .raster import SynthesisSettings
|
|
278
|
+
|
|
279
|
+
section = config.get("synthesis")
|
|
280
|
+
if section is None:
|
|
281
|
+
raise ValueError("An 'atmosphere_series' needs a 'synthesis:' section naming at "
|
|
282
|
+
"least the 'lines' to synthesise.")
|
|
283
|
+
lines = section.get("lines")
|
|
284
|
+
if isinstance(lines, str):
|
|
285
|
+
lines = [lines]
|
|
286
|
+
if not isinstance(lines, list) or not lines or not all(isinstance(l, str) for l in lines):
|
|
287
|
+
raise ValueError("'synthesis.lines' must be a list of line names such as "
|
|
288
|
+
"['Fe12_195.1190'].")
|
|
289
|
+
settings = {"lines": tuple(lines)}
|
|
290
|
+
if "abundance" in section:
|
|
291
|
+
settings["abundance"] = str(section["abundance"])
|
|
292
|
+
for key in ("vel_res", "vel_lim"):
|
|
293
|
+
value = _quantity_or_none(section, key, u.km / u.s, "synthesis")
|
|
294
|
+
if value is not None:
|
|
295
|
+
settings[key] = value
|
|
296
|
+
for key in ("crop_y", "crop_z"):
|
|
297
|
+
settings[key] = _range_or_none(section, key, "synthesis")
|
|
298
|
+
if "precision" in section:
|
|
299
|
+
precision = str(section["precision"])
|
|
300
|
+
if precision not in ("float32", "float64"):
|
|
301
|
+
raise ValueError(f"'synthesis.precision' must be 'float32' or 'float64', "
|
|
302
|
+
f"got {precision!r}.")
|
|
303
|
+
settings["precision"] = np.float32 if precision == "float32" else np.float64
|
|
304
|
+
if section.get("mass_per_electron") is not None:
|
|
305
|
+
value = section["mass_per_electron"]
|
|
306
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
307
|
+
raise ValueError(f"'synthesis.mass_per_electron' must be a number of atomic mass "
|
|
308
|
+
f"units, such as 1.16, got {value!r}.")
|
|
309
|
+
settings["mass_per_electron"] = float(value)
|
|
310
|
+
if section.get("hdf5_dbase_root") is not None:
|
|
311
|
+
settings["hdf5_dbase_root"] = str(section["hdf5_dbase_root"])
|
|
312
|
+
# Require whole numbers rather than calling int(), which would truncate a
|
|
313
|
+
# fraction.
|
|
314
|
+
for key, least in (("n_workers", 0), ("goft_temperature_chunk", 1)):
|
|
315
|
+
value = section.get(key)
|
|
316
|
+
if value is None:
|
|
317
|
+
continue
|
|
318
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < least:
|
|
319
|
+
raise ValueError(f"'synthesis.{key}' must be a whole number of {least} or more, "
|
|
320
|
+
f"got {value!r}.")
|
|
321
|
+
settings[key] = value
|
|
322
|
+
return SynthesisSettings(**settings)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _parse_raster_plan(config: dict):
|
|
326
|
+
"""Observing plan from the 'raster:' section."""
|
|
327
|
+
from .raster import RasterPlan
|
|
328
|
+
|
|
329
|
+
section = config.get("raster")
|
|
330
|
+
if section is None:
|
|
331
|
+
raise ValueError("A time series needs a 'raster:' section saying at least when "
|
|
332
|
+
"the observation starts ('start').")
|
|
333
|
+
start = _quantity_or_none(section, "start", u.s, "raster")
|
|
334
|
+
if start is None:
|
|
335
|
+
raise ValueError("'raster.start' is needed: the simulation time at which the "
|
|
336
|
+
"first exposure starts, with units, e.g. '3850 s'.")
|
|
337
|
+
plan = {"start": start}
|
|
338
|
+
for key in ("steps", "repeats"):
|
|
339
|
+
if section.get(key) is not None:
|
|
340
|
+
plan[key] = section[key]
|
|
341
|
+
plan["step"] = _quantity_or_none(section, "step", u.arcsec, "raster")
|
|
342
|
+
plan["cadence"] = _quantity_or_none(section, "cadence", u.s, "raster")
|
|
343
|
+
plan["centre"] = _quantity_or_none(section, "centre", u.Mm, "raster")
|
|
344
|
+
if section.get("direction") is not None:
|
|
345
|
+
plan["direction"] = section["direction"]
|
|
346
|
+
return RasterPlan(**plan)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _series_paths(config: dict, key: str) -> list:
|
|
350
|
+
"""Files of 'atmosphere_series' or 'synthesis_series', given as a list or a glob pattern."""
|
|
351
|
+
import glob
|
|
352
|
+
|
|
353
|
+
value = config[key]
|
|
354
|
+
if isinstance(value, str):
|
|
355
|
+
paths = sorted(glob.glob(value))
|
|
356
|
+
if not paths:
|
|
357
|
+
raise FileNotFoundError(f"'{key}' matches no file: {value}")
|
|
358
|
+
return paths
|
|
359
|
+
if isinstance(value, list) and value and all(isinstance(p, str) for p in value):
|
|
360
|
+
missing = [p for p in value if not Path(p).is_file()]
|
|
361
|
+
if missing:
|
|
362
|
+
raise FileNotFoundError(f"'{key}' names files that do not exist: {missing}")
|
|
363
|
+
return list(value)
|
|
364
|
+
kind = "atmosphere" if key == "atmosphere_series" else "synthesis"
|
|
365
|
+
raise ValueError(f"'{key}' must be a glob pattern or a list of {kind} files, "
|
|
366
|
+
f"e.g. './data/bifrost/*.h5'.")
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
# Line observed when the config names none and the synthesis holds several.
|
|
370
|
+
DEFAULT_REFERENCE_LINE = "Fe12_195.1190"
|
|
371
|
+
|
|
372
|
+
# Where synthesise-spectra writes by default, and where older versions did.
|
|
373
|
+
DEFAULT_SYNTHESIS_FILE = "./run/input/synthesised_spectra.h5"
|
|
374
|
+
LEGACY_SYNTHESIS_FILE = "./run/input/synthesised_spectra.pkl"
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _reference_line(config: dict, synthesis_path) -> str:
|
|
378
|
+
"""
|
|
379
|
+
Line to observe in a synthesis file.
|
|
380
|
+
|
|
381
|
+
This is 'reference_line' if set. If the key is present but empty, it is
|
|
382
|
+
the file's first line, as in older versions. If the key is absent, it is
|
|
383
|
+
the file's only line, or else the default line, which the file must then
|
|
384
|
+
hold.
|
|
385
|
+
"""
|
|
386
|
+
if config.get("reference_line") is not None:
|
|
387
|
+
return config["reference_line"]
|
|
388
|
+
names = synthesis_line_names(synthesis_path)
|
|
389
|
+
if not names:
|
|
390
|
+
# A file with no lines is refused when it is read.
|
|
391
|
+
return DEFAULT_REFERENCE_LINE
|
|
392
|
+
if "reference_line" in config or len(names) == 1:
|
|
393
|
+
return names[0]
|
|
394
|
+
if DEFAULT_REFERENCE_LINE not in names:
|
|
395
|
+
raise ValueError(f"{synthesis_path} holds the lines {names} and no 'reference_line' "
|
|
396
|
+
f"says which to observe; the default, {DEFAULT_REFERENCE_LINE}, is "
|
|
397
|
+
f"not among them.")
|
|
398
|
+
return DEFAULT_REFERENCE_LINE
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _parse_pinhole_config(config: dict) -> tuple:
|
|
402
|
+
"""
|
|
403
|
+
Read and validate the pinhole lists from a YAML config.
|
|
404
|
+
|
|
405
|
+
The lists have one entry per pinhole, so a position list given without
|
|
406
|
+
the sizes describes no pinhole and is always a mistake.
|
|
407
|
+
|
|
408
|
+
Parameters
|
|
409
|
+
----------
|
|
410
|
+
config : dict
|
|
411
|
+
Whole parsed YAML config.
|
|
412
|
+
|
|
413
|
+
Returns
|
|
414
|
+
-------
|
|
415
|
+
tuple
|
|
416
|
+
`(pinhole_sizes, pinhole_positions, pinhole_positions_spectral)`.
|
|
417
|
+
"""
|
|
418
|
+
pinhole_sizes = []
|
|
419
|
+
pinhole_positions = []
|
|
420
|
+
if "pinhole_sizes" in config:
|
|
421
|
+
pinhole_sizes = ensure_list(parse_yaml_input(config["pinhole_sizes"]))
|
|
422
|
+
if "pinhole_positions" in config:
|
|
423
|
+
pinhole_positions = ensure_list(config["pinhole_positions"])
|
|
424
|
+
|
|
425
|
+
# Optional spectral positions, one per pinhole, as a fraction (0.0-1.0) of
|
|
426
|
+
# the detector's spectral width. If omitted, every pinhole is projected to
|
|
427
|
+
# the centre of the spectral window.
|
|
428
|
+
pinhole_positions_spectral = []
|
|
429
|
+
if "pinhole_positions_spectral" in config:
|
|
430
|
+
pinhole_positions_spectral = ensure_list(
|
|
431
|
+
config["pinhole_positions_spectral"])
|
|
432
|
+
|
|
433
|
+
pinhole_positions, pinhole_positions_spectral = check_pinhole_lists(
|
|
434
|
+
pinhole_sizes, pinhole_positions, pinhole_positions_spectral)
|
|
435
|
+
|
|
436
|
+
return pinhole_sizes, pinhole_positions, pinhole_positions_spectral
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _parse_fitting_config(config: dict) -> FitConfig | None:
|
|
440
|
+
"""
|
|
441
|
+
Fit configuration from the `fitting:` block of a YAML config.
|
|
442
|
+
|
|
443
|
+
A block with no components gives the single-Gaussian fit. FitConfig
|
|
444
|
+
checks the values, including that there are no components or at least
|
|
445
|
+
two.
|
|
446
|
+
|
|
447
|
+
Parameters
|
|
448
|
+
----------
|
|
449
|
+
config : dict
|
|
450
|
+
Whole parsed YAML config.
|
|
451
|
+
|
|
452
|
+
Returns
|
|
453
|
+
-------
|
|
454
|
+
FitConfig or None
|
|
455
|
+
None if the config has no `fitting:` block.
|
|
456
|
+
"""
|
|
457
|
+
fitting_cfg = config.get("fitting", None)
|
|
458
|
+
if fitting_cfg is None:
|
|
459
|
+
return None
|
|
460
|
+
|
|
461
|
+
components = []
|
|
462
|
+
for idx, comp_dict in enumerate(fitting_cfg.get("components", [])):
|
|
463
|
+
if "wavelength" not in comp_dict:
|
|
464
|
+
raise ValueError(
|
|
465
|
+
f"fitting.components[{idx}] is missing required field "
|
|
466
|
+
f"'wavelength' (rest wavelength of this Gaussian component, "
|
|
467
|
+
f"e.g. 'wavelength: 195.119 angstrom')."
|
|
468
|
+
)
|
|
469
|
+
components.append(FitComponent(
|
|
470
|
+
wavelength=parse_yaml_input(comp_dict["wavelength"]),
|
|
471
|
+
tie_center=comp_dict.get("tie_center", None),
|
|
472
|
+
tie_width=comp_dict.get("tie_width", None),
|
|
473
|
+
amplitude_greater_than=comp_dict.get("amplitude_greater_than", None),
|
|
474
|
+
name=comp_dict.get("name", None),
|
|
475
|
+
))
|
|
476
|
+
|
|
477
|
+
return FitConfig(
|
|
478
|
+
components=components,
|
|
479
|
+
primary_component=fitting_cfg.get("primary_component", 0),
|
|
480
|
+
constrain_positive_intensity=fitting_cfg.get(
|
|
481
|
+
"constrain_positive_intensity", False),
|
|
482
|
+
backend=fitting_cfg.get("backend", None),
|
|
483
|
+
max_iter=fitting_cfg.get("max_iter", FitConfig.max_iter),
|
|
484
|
+
bessel_correction=fitting_cfg.get("bessel_correction", False),
|
|
485
|
+
save_iterations=fitting_cfg.get("save_iterations", False),
|
|
486
|
+
weighted=fitting_cfg.get("weighted", True),
|
|
487
|
+
)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
@debug_on_error
|
|
491
|
+
def main() -> None:
|
|
492
|
+
"""
|
|
493
|
+
Run the `eclipse` simulation with arguments from `sys.argv`.
|
|
494
|
+
|
|
495
|
+
Reads the config file given by `--config`, runs every combination of its
|
|
496
|
+
settings, and writes the results to `run/result/<config name>.h5`. The
|
|
497
|
+
`eclipse` command checks its arguments and handles errors before calling
|
|
498
|
+
this.
|
|
499
|
+
"""
|
|
500
|
+
|
|
501
|
+
# Suppress noisy astropy WCS warnings
|
|
502
|
+
for msg in [
|
|
503
|
+
"target cannot be converted to ICRS",
|
|
504
|
+
"target cannot be converted to ICRS, so will not be set on SpectralCoord",
|
|
505
|
+
"No observer defined on WCS, SpectralCoord will be converted without any velocity frame change",
|
|
506
|
+
]:
|
|
507
|
+
warnings.filterwarnings("ignore", message=msg, category=UserWarning)
|
|
508
|
+
warnings.filterwarnings("ignore", module="astropy.wcs.wcsapi.fitswcs", category=UserWarning)
|
|
509
|
+
|
|
510
|
+
parser = argparse.ArgumentParser()
|
|
511
|
+
parser.add_argument("--config", type=str, help="YAML config file", required=True)
|
|
512
|
+
parser.add_argument("--debug", action="store_true", help="Enable debug mode")
|
|
513
|
+
args = parser.parse_args()
|
|
514
|
+
|
|
515
|
+
set_debug_mode(args.debug)
|
|
516
|
+
if args.debug:
|
|
517
|
+
print("Debug mode enabled - will break to IPython on errors")
|
|
518
|
+
|
|
519
|
+
# MPI auto-detection: when launched by srun or mpirun with several tasks,
|
|
520
|
+
# the Monte Carlo iterations are split across the ranks.
|
|
521
|
+
from .utils import _get_mpi_info
|
|
522
|
+
_comm, _mpi_rank, _mpi_size = _get_mpi_info()
|
|
523
|
+
if _mpi_size > 1:
|
|
524
|
+
# Intel MPI pins each rank to cores, which breaks joblib/loky. Widen
|
|
525
|
+
# the affinity to this rank's share of the job step's CPUs on the
|
|
526
|
+
# node. CPUs 0 to n would be shared by every rank and, on a node
|
|
527
|
+
# shared with another job, need not be this job's.
|
|
528
|
+
_cpus = _rank_cpus(os.environ, _allowed_cpus(os.environ, _cpuset(),
|
|
529
|
+
sorted(os.sched_getaffinity(0))))
|
|
530
|
+
if _cpus:
|
|
531
|
+
os.sched_setaffinity(0, _cpus)
|
|
532
|
+
|
|
533
|
+
if _mpi_rank == 0:
|
|
534
|
+
print(f"MPI distributed mode: {_mpi_size} processes "
|
|
535
|
+
f"(MC iterations will be split across ranks)")
|
|
536
|
+
else:
|
|
537
|
+
# Silence stdout and stderr on non-root ranks to avoid duplicated
|
|
538
|
+
# output. Redirect the OS file descriptors, not just the Python
|
|
539
|
+
# objects, because SLURM captures fd 1/2 directly and tqdm can
|
|
540
|
+
# bypass sys.stderr. The original stderr is kept for an error that
|
|
541
|
+
# must be reported before the run is aborted.
|
|
542
|
+
global _STARTING_STDERR
|
|
543
|
+
_STARTING_STDERR = os.dup(2)
|
|
544
|
+
_devnull_fd = os.open(os.devnull, os.O_WRONLY)
|
|
545
|
+
os.dup2(_devnull_fd, 1) # redirect fd 1 (stdout)
|
|
546
|
+
os.dup2(_devnull_fd, 2) # redirect fd 2 (stderr)
|
|
547
|
+
os.close(_devnull_fd)
|
|
548
|
+
# Wrap the already-redirected fds rather than opening new devnull
|
|
549
|
+
# handles; closefd=False prevents a ResourceWarning when these
|
|
550
|
+
# objects are later garbage-collected.
|
|
551
|
+
sys.stdout = open(1, "w", closefd=False)
|
|
552
|
+
sys.stderr = open(2, "w", closefd=False)
|
|
553
|
+
|
|
554
|
+
config_path = Path(args.config)
|
|
555
|
+
if not config_path.is_file():
|
|
556
|
+
raise FileNotFoundError(f"Config file not found: {args.config}")
|
|
557
|
+
|
|
558
|
+
config = load_yaml_config(config_path.read_text())
|
|
559
|
+
|
|
560
|
+
# An empty file parses to None, which would otherwise fail later with an
|
|
561
|
+
# AttributeError.
|
|
562
|
+
if config is None:
|
|
563
|
+
raise ValueError(f"Config file is empty: {args.config}")
|
|
564
|
+
if not isinstance(config, dict):
|
|
565
|
+
raise ValueError(
|
|
566
|
+
f"Config file must be a mapping of keys to values, got "
|
|
567
|
+
f"{type(config).__name__}: {args.config}"
|
|
568
|
+
)
|
|
569
|
+
# The config is saved with the results, so a value the results file
|
|
570
|
+
# cannot hold, such as a YAML alias that refers to itself, is refused now
|
|
571
|
+
# rather than after the run.
|
|
572
|
+
try:
|
|
573
|
+
_to_json(config)
|
|
574
|
+
except (TypeError, RecursionError) as error:
|
|
575
|
+
reason = "a value holds itself" if isinstance(error, RecursionError) else error
|
|
576
|
+
raise ValueError(f"{args.config} cannot be saved with the results: {reason}") from None
|
|
577
|
+
|
|
578
|
+
# Top-level scalar settings. A 'simulation:' that is not a mapping is left
|
|
579
|
+
# for _validate_config_keys to report.
|
|
580
|
+
simulation_section = config.get("simulation")
|
|
581
|
+
if isinstance(simulation_section, dict) and "instrument" in simulation_section:
|
|
582
|
+
raise ValueError(
|
|
583
|
+
"Set the instrument with the top-level 'instrument:' key, not "
|
|
584
|
+
"inside the 'simulation:' section. Both Simulation objects are "
|
|
585
|
+
"built from the top-level value, so one written here would be "
|
|
586
|
+
"read and then ignored."
|
|
587
|
+
)
|
|
588
|
+
instrument = config.get("instrument", "SWC")
|
|
589
|
+
if not isinstance(instrument, str):
|
|
590
|
+
raise ValueError(f"'instrument' must be SWC or EIS, got {instrument!r}.")
|
|
591
|
+
instrument = instrument.upper()
|
|
592
|
+
|
|
593
|
+
# Before anything is loaded, so a config written for an older layout fails
|
|
594
|
+
# here rather than after the atmosphere is loaded.
|
|
595
|
+
_validate_config_keys(config, instrument)
|
|
596
|
+
|
|
597
|
+
n_iter = config.get("n_iter", 25)
|
|
598
|
+
# Checked now: 0 would write empty results and report success, and '1e3',
|
|
599
|
+
# a string to PyYAML, would fail only after the ground truth was fitted.
|
|
600
|
+
if isinstance(n_iter, bool) or not isinstance(n_iter, int) or n_iter < 1:
|
|
601
|
+
raise ValueError(f"'n_iter' must be a whole number of iterations, 1 or more, got "
|
|
602
|
+
f"{n_iter!r}.")
|
|
603
|
+
ncpu = config.get("ncpu", -1)
|
|
604
|
+
# joblib and reproject both use ncpu and agree only on these values. 0,
|
|
605
|
+
# which means every CPU for the synthesis's n_workers, would fail at the
|
|
606
|
+
# first fit.
|
|
607
|
+
if isinstance(ncpu, bool) or not isinstance(ncpu, int) or (ncpu < 1 and ncpu != -1):
|
|
608
|
+
raise ValueError(f"'ncpu' must be a whole number of CPUs, 1 or more, or -1 for all "
|
|
609
|
+
f"of them, got {ncpu!r}.")
|
|
610
|
+
|
|
611
|
+
# In MPI mode, cap a given ncpu at the CPUs available to this rank so
|
|
612
|
+
# joblib does not oversubscribe. ncpu=-1 is left alone: joblib uses the
|
|
613
|
+
# OS affinity mask, which I_MPI_PIN_DOMAIN=auto sets correctly.
|
|
614
|
+
if _mpi_size > 1:
|
|
615
|
+
if ncpu != -1:
|
|
616
|
+
slurm_cpus = os.environ.get("SLURM_CPUS_PER_TASK")
|
|
617
|
+
if slurm_cpus is not None:
|
|
618
|
+
available_cpus = int(slurm_cpus)
|
|
619
|
+
else:
|
|
620
|
+
available_cpus = os.cpu_count() or 1
|
|
621
|
+
if ncpu > available_cpus:
|
|
622
|
+
ncpu = available_cpus
|
|
623
|
+
if _mpi_rank == 0:
|
|
624
|
+
print(f"MPI: ncpu={ncpu} per rank")
|
|
625
|
+
|
|
626
|
+
# Simulation mode. A time series is either atmosphere files, synthesised
|
|
627
|
+
# in this run, or synthesis files made beforehand.
|
|
628
|
+
uniform_intensity_mode = "uniform_intensity" in config
|
|
629
|
+
if uniform_intensity_mode and "synthesis_file" in config:
|
|
630
|
+
warnings.warn("'synthesis_file' is ignored: 'uniform_intensity' says what is observed.",
|
|
631
|
+
UserWarning, stacklevel=2)
|
|
632
|
+
# These describe the line of a uniform intensity; other modes observe the
|
|
633
|
+
# spectra they are given.
|
|
634
|
+
for key in ("rest_wavelength", "thermal_width"):
|
|
635
|
+
if key in config and not uniform_intensity_mode:
|
|
636
|
+
warnings.warn(f"'{key}' is ignored: it describes the line of a 'uniform_intensity', "
|
|
637
|
+
f"and this run observes the spectra it is given.",
|
|
638
|
+
UserWarning, stacklevel=2)
|
|
639
|
+
atmosphere_series_mode = "atmosphere_series" in config
|
|
640
|
+
synthesis_series_mode = "synthesis_series" in config
|
|
641
|
+
raster_mode = atmosphere_series_mode or synthesis_series_mode
|
|
642
|
+
given = [key for key in ("uniform_intensity", "synthesis_file", "atmosphere_series",
|
|
643
|
+
"synthesis_series") if key in config]
|
|
644
|
+
if raster_mode and len(given) > 1:
|
|
645
|
+
raise ValueError(f"Give only one of {given}: each says what is observed.")
|
|
646
|
+
if not atmosphere_series_mode and "synthesis" in config:
|
|
647
|
+
raise ValueError("The 'synthesis:' section belongs to an 'atmosphere_series' run "
|
|
648
|
+
"and would not be read here.")
|
|
649
|
+
if not raster_mode and "raster" in config:
|
|
650
|
+
raise ValueError("The 'raster:' section belongs to an 'atmosphere_series' or "
|
|
651
|
+
"'synthesis_series' run and would not be read here.")
|
|
652
|
+
|
|
653
|
+
if atmosphere_series_mode:
|
|
654
|
+
series_paths = _series_paths(config, "atmosphere_series")
|
|
655
|
+
synthesis_settings = _parse_synthesis_settings(config)
|
|
656
|
+
raster_plan = _parse_raster_plan(config)
|
|
657
|
+
# Empty or absent: the first line.
|
|
658
|
+
reference_line = config.get("reference_line") or synthesis_settings.lines[0]
|
|
659
|
+
if reference_line not in synthesis_settings.lines:
|
|
660
|
+
raise ValueError(f"'reference_line' {reference_line!r} is not one of "
|
|
661
|
+
f"'synthesis.lines' {list(synthesis_settings.lines)}.")
|
|
662
|
+
print("TIME SERIES MODE")
|
|
663
|
+
print(f" Atmosphere files: {len(series_paths)}")
|
|
664
|
+
print(f" Lines: {list(synthesis_settings.lines)}, reference {reference_line}")
|
|
665
|
+
print(f" Raster: {raster_plan.steps} step(s), {raster_plan.repeats} repeat(s), "
|
|
666
|
+
f"starting at {raster_plan.start}")
|
|
667
|
+
elif synthesis_series_mode:
|
|
668
|
+
series_paths = _series_paths(config, "synthesis_series")
|
|
669
|
+
synthesis_settings = None
|
|
670
|
+
raster_plan = _parse_raster_plan(config)
|
|
671
|
+
reference_line = _reference_line(config, series_paths[0])
|
|
672
|
+
print("TIME SERIES MODE")
|
|
673
|
+
print(f" Synthesis files: {len(series_paths)}")
|
|
674
|
+
print(f" Reference line: {reference_line}")
|
|
675
|
+
print(f" Raster: {raster_plan.steps} step(s), {raster_plan.repeats} repeat(s), "
|
|
676
|
+
f"starting at {raster_plan.start}")
|
|
677
|
+
elif uniform_intensity_mode:
|
|
678
|
+
uniform_intensity = parse_yaml_input(config["uniform_intensity"])
|
|
679
|
+
if not hasattr(uniform_intensity, "unit"):
|
|
680
|
+
raise ValueError(
|
|
681
|
+
"uniform_intensity must include units, e.g. '5000 erg / (s cm2 sr)'"
|
|
682
|
+
)
|
|
683
|
+
uniform_rest_wavelength = parse_yaml_input(config.get("rest_wavelength", "195.119 AA"))
|
|
684
|
+
uniform_thermal_width = parse_yaml_input(config.get("thermal_width", "20 km/s"))
|
|
685
|
+
# Each must be positive, in units of the right kind: a zero or negative
|
|
686
|
+
# width gives a nonsense fit, which would be saved as a result.
|
|
687
|
+
for key, value, kind, example in (
|
|
688
|
+
("uniform_intensity", uniform_intensity, u.erg / (u.s * u.cm**2 * u.sr),
|
|
689
|
+
"5000 erg / (s cm2 sr)"),
|
|
690
|
+
("rest_wavelength", uniform_rest_wavelength, u.AA, "195.119 AA"),
|
|
691
|
+
("thermal_width", uniform_thermal_width, u.km / u.s, "20 km/s")):
|
|
692
|
+
if not (isinstance(value, u.Quantity) and value.unit.is_equivalent(kind)
|
|
693
|
+
and np.isfinite(value.value) and value.value > 0):
|
|
694
|
+
raise ValueError(f"'{key}' must be a positive {kind.physical_type}, such as "
|
|
695
|
+
f"'{example}', got {config.get(key)!r}.")
|
|
696
|
+
print("UNIFORM INTENSITY MODE")
|
|
697
|
+
print(f" Intensity: {uniform_intensity}")
|
|
698
|
+
print(f" Rest wavelength: {uniform_rest_wavelength}")
|
|
699
|
+
print(f" Thermal width (1-sigma): {uniform_thermal_width}")
|
|
700
|
+
else:
|
|
701
|
+
synthesis_file = config.get("synthesis_file", DEFAULT_SYNTHESIS_FILE)
|
|
702
|
+
# Fall back to the pickle older versions wrote, for a run that leaves
|
|
703
|
+
# the synthesis file at its default.
|
|
704
|
+
if "synthesis_file" not in config and Path(LEGACY_SYNTHESIS_FILE).is_file():
|
|
705
|
+
if not Path(synthesis_file).is_file():
|
|
706
|
+
synthesis_file = LEGACY_SYNTHESIS_FILE
|
|
707
|
+
else:
|
|
708
|
+
warnings.warn(f"Both {DEFAULT_SYNTHESIS_FILE} and {LEGACY_SYNTHESIS_FILE} exist "
|
|
709
|
+
f"and 'synthesis_file' names neither; observing "
|
|
710
|
+
f"{DEFAULT_SYNTHESIS_FILE}, where the synthesis now writes. "
|
|
711
|
+
f"Name the one to observe with 'synthesis_file'.",
|
|
712
|
+
UserWarning, stacklevel=2)
|
|
713
|
+
if not Path(synthesis_file).is_file():
|
|
714
|
+
raise FileNotFoundError(
|
|
715
|
+
f"Synthesis file not found: {synthesis_file}. "
|
|
716
|
+
"Please check the 'synthesis_file' path in your config file."
|
|
717
|
+
)
|
|
718
|
+
synthesis_is_hdf5 = is_synthesis_file(synthesis_file)
|
|
719
|
+
if synthesis_is_hdf5:
|
|
720
|
+
reference_line = _reference_line(config, synthesis_file)
|
|
721
|
+
elif not _is_pickle(Path(synthesis_file)):
|
|
722
|
+
raise ValueError(
|
|
723
|
+
f"{synthesis_file} is neither a synthesis file, which is HDF5, nor a synthesis "
|
|
724
|
+
f"pickle as older versions of ECLIPSE wrote. A synthesis from another code is "
|
|
725
|
+
f"written as a synthesis file with eclipse.write_synthesis.")
|
|
726
|
+
else:
|
|
727
|
+
warnings.warn(
|
|
728
|
+
f"{synthesis_file} is a synthesis pickle, as older versions of ECLIPSE "
|
|
729
|
+
f"wrote them. Pickles are deprecated and will not be read in a future "
|
|
730
|
+
f"release: re-run the synthesis, or convert the file with "
|
|
731
|
+
f"eclipse.convert_synthesis_pickle.", FutureWarning, stacklevel=2)
|
|
732
|
+
reference_line = config.get("reference_line", DEFAULT_REFERENCE_LINE)
|
|
733
|
+
|
|
734
|
+
# Pinhole config (fixed paired lists, not swept)
|
|
735
|
+
(pinhole_sizes, pinhole_positions,
|
|
736
|
+
pinhole_positions_spectral) = _parse_pinhole_config(config)
|
|
737
|
+
|
|
738
|
+
# Parse config sections
|
|
739
|
+
sim_fixed, sim_sweep = _parse_section(config.get("simulation", {}), "simulation")
|
|
740
|
+
det_fixed, det_sweep = _parse_section(config.get("detector", {}), "detector")
|
|
741
|
+
# An old name, such as _dark_current_293k, is recorded as the config gives
|
|
742
|
+
# it and passed to the detector under its new one.
|
|
743
|
+
det_renamed = _old_names_used(Detector_EIS if instrument == "EIS" else Detector_SWC,
|
|
744
|
+
{**det_fixed, **det_sweep}, "detector.", error=ValueError)
|
|
745
|
+
tel_fixed, tel_sweep = _parse_section(config.get("telescope", {}), "telescope")
|
|
746
|
+
fil_fixed, fil_sweep = _parse_section(config.get("filter", {}), "filter")
|
|
747
|
+
|
|
748
|
+
# The width of the pixels is taken where the observed line lands, which a
|
|
749
|
+
# run reads from a synthesis file or the uniform-intensity settings.
|
|
750
|
+
focal_plane_dispersion = (sim_fixed.get("focal_plane_dispersion")
|
|
751
|
+
or True in sim_sweep.get("focal_plane_dispersion", []))
|
|
752
|
+
if focal_plane_dispersion and instrument == "SWC" and (atmosphere_series_mode or not (
|
|
753
|
+
uniform_intensity_mode or synthesis_series_mode or synthesis_is_hdf5)):
|
|
754
|
+
raise ValueError(
|
|
755
|
+
"simulation.focal_plane_dispersion needs the observed line's rest wavelength, which "
|
|
756
|
+
"ECLIPSE reads from a synthesis file or the uniform-intensity settings, not yet from "
|
|
757
|
+
"an atmosphere series or a synthesis pickle.")
|
|
758
|
+
|
|
759
|
+
# A run models frames taken with the shutter closed outside the exposure.
|
|
760
|
+
# Without a shutter, light falls on the CCDs while a frame is cleared and
|
|
761
|
+
# read, and every line in the band smears into the window. The run
|
|
762
|
+
# synthesises only the window, so it cannot model this, and refuses the
|
|
763
|
+
# setting rather than ignoring it.
|
|
764
|
+
if "shutter" in det_sweep or det_fixed.get("shutter", True) is False:
|
|
765
|
+
raise ValueError(
|
|
766
|
+
"detector.shutter must be true for an instrument run, which models frames taken "
|
|
767
|
+
"with the shutter closed outside the exposure. Frames taken without it are "
|
|
768
|
+
"modelled by eclipse.readout, which takes the detector's settings.")
|
|
769
|
+
# A run does not model charge transfer either, so a CTE below 1 would be ignored.
|
|
770
|
+
for name in ("cte_parallel", "cte_serial"):
|
|
771
|
+
if name in det_sweep or det_fixed.get(name, 1.0) != 1.0:
|
|
772
|
+
raise ValueError(
|
|
773
|
+
f"detector.{name} must be 1 for an instrument run, which does not model "
|
|
774
|
+
f"charge transfer. Charge left behind as a frame is read is modelled by "
|
|
775
|
+
f"eclipse.readout, which takes the detector's settings.")
|
|
776
|
+
# Nor does it take its wavelengths from the SW focal plane. It uses the focal
|
|
777
|
+
# plane only to find the row recording each wavelength, for the dark current,
|
|
778
|
+
# and, with focal_plane_dispersion, the width of its pixels. Other values
|
|
779
|
+
# would make these disagree with its wavelengths.
|
|
780
|
+
if instrument == "SWC":
|
|
781
|
+
focal_plane = {name: det_fixed[name] for name in _FOCAL_PLANE_SETTINGS
|
|
782
|
+
if name in det_fixed}
|
|
783
|
+
default, given = Detector_SWC(), Detector_SWC(**focal_plane)
|
|
784
|
+
for name in _FOCAL_PLANE_SETTINGS:
|
|
785
|
+
same = _same_setting(getattr(given, name), getattr(default, name))
|
|
786
|
+
if name in det_sweep or not same:
|
|
787
|
+
raise ValueError(
|
|
788
|
+
f"detector.{name} must be left at its default for an instrument run, "
|
|
789
|
+
f"which takes its wavelengths from detector.wvl_res, not from the SW focal "
|
|
790
|
+
f"plane, and needs the two to agree. Full-CCD frames, made by "
|
|
791
|
+
f"eclipse.full_frame, do.")
|
|
792
|
+
|
|
793
|
+
# The wavelength grids are cached by slit width and detector sampling, and
|
|
794
|
+
# sized for the slit's spectral PSF, which also depends on the slit
|
|
795
|
+
# psf_params was measured with. Like psf_params, that is a fixed property
|
|
796
|
+
# of the telescope, so it takes one value.
|
|
797
|
+
if "psf_slit_width" in tel_sweep:
|
|
798
|
+
raise ValueError(
|
|
799
|
+
"telescope.psf_slit_width is the slit psf_params was measured with, "
|
|
800
|
+
"so like psf_params it takes a single value and cannot be swept.")
|
|
801
|
+
|
|
802
|
+
# Instrument-specific validation
|
|
803
|
+
if instrument == "EIS":
|
|
804
|
+
if config.get("filter"):
|
|
805
|
+
warnings.warn(
|
|
806
|
+
"The 'filter:' section describes SWC's filter and is ignored for EIS, whose "
|
|
807
|
+
"filters are folded into its effective area tables.",
|
|
808
|
+
UserWarning,
|
|
809
|
+
)
|
|
810
|
+
fil_fixed, fil_sweep = {}, {}
|
|
811
|
+
vis_sl = [sim_fixed["vis_sl"]] if "vis_sl" in sim_fixed else sim_sweep.get("vis_sl", [])
|
|
812
|
+
if any(np.any(u.Quantity(value).value > 0) for value in vis_sl):
|
|
813
|
+
warnings.warn(
|
|
814
|
+
"EIS's filters are not modelled for visible light, so vis_sl reaches its CCD "
|
|
815
|
+
"as given, where for SWC it is the flux before the filter. Give EIS the flux "
|
|
816
|
+
"after its filters.",
|
|
817
|
+
UserWarning,
|
|
818
|
+
)
|
|
819
|
+
for key in ("microroughness_sigma",):
|
|
820
|
+
if key in tel_fixed or key in tel_sweep:
|
|
821
|
+
warnings.warn(
|
|
822
|
+
f"EIS does not support '{key}'. This parameter will be ignored.",
|
|
823
|
+
UserWarning,
|
|
824
|
+
)
|
|
825
|
+
tel_fixed.pop(key, None)
|
|
826
|
+
tel_sweep.pop(key, None)
|
|
827
|
+
# Any pinhole key, not just the sizes: positions given alone would
|
|
828
|
+
# otherwise be accepted and silently ignored.
|
|
829
|
+
if (pinhole_sizes or pinhole_positions or pinhole_positions_spectral
|
|
830
|
+
or sim_fixed.get("enable_pinholes")
|
|
831
|
+
or any(sim_sweep.get("enable_pinholes", []))
|
|
832
|
+
or "pinhole_visible_wavelength" in sim_fixed
|
|
833
|
+
or "pinhole_visible_wavelength" in sim_sweep):
|
|
834
|
+
raise ValueError(
|
|
835
|
+
"Pinhole effects are not supported for EIS. Remove "
|
|
836
|
+
"enable_pinholes, pinhole_sizes, pinhole_positions, "
|
|
837
|
+
"pinhole_positions_spectral and pinhole_visible_wavelength, or run this "
|
|
838
|
+
"config against SWC."
|
|
839
|
+
)
|
|
840
|
+
|
|
841
|
+
# Parse fitting configuration
|
|
842
|
+
fit_config = _parse_fitting_config(config)
|
|
843
|
+
if fit_config is not None:
|
|
844
|
+
if fit_config.backend == "mpfit":
|
|
845
|
+
backend_label = "mpfit (forced)"
|
|
846
|
+
elif fit_config.backend == "scipy":
|
|
847
|
+
backend_label = "scipy (forced)"
|
|
848
|
+
else:
|
|
849
|
+
backend_label = "scipy (auto)"
|
|
850
|
+
if fit_config.is_single:
|
|
851
|
+
print(f"Single-Gaussian fitting: max_iter={fit_config.max_iter}, "
|
|
852
|
+
f"backend={backend_label}")
|
|
853
|
+
else:
|
|
854
|
+
print(f"Multi-component fitting enabled: {fit_config.n_components} components "
|
|
855
|
+
f"(primary={fit_config.primary_component}, "
|
|
856
|
+
f"{fit_config.n_full_params} params, "
|
|
857
|
+
f"backend={backend_label})")
|
|
858
|
+
|
|
859
|
+
# Parse off-chip slit binning (ground-based spatial binning along the slit)
|
|
860
|
+
offchip_bin_slits = ensure_list(config.get("offchip_bin_slit", [1]))
|
|
861
|
+
|
|
862
|
+
# Parse which signals to fit (default: both DN and photon)
|
|
863
|
+
fit_signals = config.get("fit_signals", "both")
|
|
864
|
+
if fit_signals not in ("both", "dn", "photon"):
|
|
865
|
+
raise ValueError(
|
|
866
|
+
f"Unknown fit_signals value '{fit_signals}'. "
|
|
867
|
+
f"Supported values: 'both', 'dn', 'photon'."
|
|
868
|
+
)
|
|
869
|
+
if fit_signals != "both":
|
|
870
|
+
skipped = "photon" if fit_signals == "dn" else "dn"
|
|
871
|
+
print(f"Fitting only '{fit_signals}' signal (skipping '{skipped}')")
|
|
872
|
+
# Unless the fitting block turns weighting off, the DN fits are weighted
|
|
873
|
+
# by each pixel's uncertainty and the photon fits maximise the Poisson
|
|
874
|
+
# likelihood. The ground truth is weighted like the DN fits, except when
|
|
875
|
+
# only photons are fitted, as no DN uncertainty is then worked out.
|
|
876
|
+
weighted_fits = fit_config is None or fit_config.weighted
|
|
877
|
+
weighted_truth = weighted_fits and fit_signals != "photon"
|
|
878
|
+
print("DN fits weighted by each pixel's uncertainty, photon fits by their Poisson "
|
|
879
|
+
"likelihood" if weighted_fits else "Fits unweighted")
|
|
880
|
+
|
|
881
|
+
def _fit_ground_truth(cube_binned, offchip_bin_slit, uncertainty=None):
|
|
882
|
+
"""Fit to the noise-free spectra, reporting where it failed."""
|
|
883
|
+
print(f"Fitting ground truth cube (offchip_bin_slit={offchip_bin_slit})...")
|
|
884
|
+
truth = ground_truth_summary(cube_binned, fit_config, n_jobs=ncpu,
|
|
885
|
+
uncertainty=uncertainty)
|
|
886
|
+
truth_failed = truth["failed"]
|
|
887
|
+
if truth_failed.any():
|
|
888
|
+
print(f" Ground truth fit failed in {np.count_nonzero(truth_failed)} "
|
|
889
|
+
f"of {truth_failed.size} pixels; their true velocity and "
|
|
890
|
+
f"width are NaN")
|
|
891
|
+
return truth
|
|
892
|
+
# ensure_list wraps scalars in a list. Values must be plain ints (no
|
|
893
|
+
# units), not passed through int(), which would turn a fraction or a
|
|
894
|
+
# numeric string into one.
|
|
895
|
+
if not offchip_bin_slits or any(isinstance(v, bool) or not isinstance(v, int) or v < 1
|
|
896
|
+
for v in offchip_bin_slits):
|
|
897
|
+
raise ValueError(f"'offchip_bin_slit' must be whole numbers of rows, 1 or more, got "
|
|
898
|
+
f"{config.get('offchip_bin_slit')!r}.")
|
|
899
|
+
offchip_bin_slits = deduplicate_list(offchip_bin_slits, "offchip_bin_slit")
|
|
900
|
+
if any(b > 1 for b in offchip_bin_slits):
|
|
901
|
+
print(f"Off-chip slit binning values: {offchip_bin_slits} "
|
|
902
|
+
f"(ground-based, all noise per pixel before summation)")
|
|
903
|
+
|
|
904
|
+
# Apply defaults for required params not specified anywhere
|
|
905
|
+
_sim_defaults = {
|
|
906
|
+
"slit_width": 0.2 * u.arcsec,
|
|
907
|
+
"expos": 1.0 * u.s,
|
|
908
|
+
"vis_sl": 0.0 * u.photon / (u.s * u.cm**2),
|
|
909
|
+
"psf": False,
|
|
910
|
+
"psf_boundary": "replicate",
|
|
911
|
+
"spectral_psf": "quadrature",
|
|
912
|
+
"slit_mask": False,
|
|
913
|
+
"focal_plane_dispersion": False,
|
|
914
|
+
"noise": True,
|
|
915
|
+
"readout_dark_current": True,
|
|
916
|
+
"enable_pinholes": False,
|
|
917
|
+
}
|
|
918
|
+
_det_defaults = {
|
|
919
|
+
"ccd_temperature": -60 * u.Celsius,
|
|
920
|
+
}
|
|
921
|
+
for attr, default in _sim_defaults.items():
|
|
922
|
+
if attr not in sim_fixed and attr not in sim_sweep:
|
|
923
|
+
sim_fixed[attr] = default
|
|
924
|
+
for attr, default in _det_defaults.items():
|
|
925
|
+
if attr not in det_fixed and attr not in det_sweep:
|
|
926
|
+
det_fixed[attr] = default
|
|
927
|
+
|
|
928
|
+
def _swept(*attrs):
|
|
929
|
+
"""List of values for each simulation attribute in `attrs`: swept, or the fixed one."""
|
|
930
|
+
return [list(sim_sweep[attr]) if attr in sim_sweep else [sim_fixed[attr]]
|
|
931
|
+
for attr in attrs]
|
|
932
|
+
|
|
933
|
+
# Warnings
|
|
934
|
+
psf_vals = list(sim_sweep.get("psf", [sim_fixed.get("psf", False)]))
|
|
935
|
+
if any(psf_vals):
|
|
936
|
+
if instrument == "SWC":
|
|
937
|
+
warnings.warn(
|
|
938
|
+
"The SWC PSF is the modelled PSF including simulations and some microroughness "
|
|
939
|
+
"measurements. Final PSF will be measured before launch.",
|
|
940
|
+
UserWarning,
|
|
941
|
+
)
|
|
942
|
+
elif instrument == "EIS":
|
|
943
|
+
warnings.warn(
|
|
944
|
+
"The EIS PSF is not well understood. We use a symmetrical Gaussian kernel with "
|
|
945
|
+
"a FWHM of 3 pixels from Ugarte-Urra (2016) EIS Software Note 2.",
|
|
946
|
+
UserWarning,
|
|
947
|
+
)
|
|
948
|
+
|
|
949
|
+
noise_vals = list(sim_sweep.get("noise", [sim_fixed.get("noise", True)]))
|
|
950
|
+
if not all(noise_vals) and n_iter > 1:
|
|
951
|
+
warnings.warn(
|
|
952
|
+
f"noise is False, so every Monte Carlo iteration is identical. "
|
|
953
|
+
f"n_iter is {n_iter}; set it to 1 to avoid repeating the same "
|
|
954
|
+
f"deterministic run.",
|
|
955
|
+
UserWarning,
|
|
956
|
+
)
|
|
957
|
+
|
|
958
|
+
enable_ph_vals = list(sim_sweep.get("enable_pinholes", [sim_fixed.get("enable_pinholes", False)]))
|
|
959
|
+
if any(enable_ph_vals):
|
|
960
|
+
warnings.warn(
|
|
961
|
+
"Pinhole effects are only intended for use by the instrument team. "
|
|
962
|
+
"Please contact MSSL for more information.",
|
|
963
|
+
UserWarning,
|
|
964
|
+
)
|
|
965
|
+
if not pinhole_sizes:
|
|
966
|
+
warnings.warn(
|
|
967
|
+
"enable_pinholes is True but no pinhole_sizes specified. "
|
|
968
|
+
"Pinhole effects will be disabled.",
|
|
969
|
+
UserWarning,
|
|
970
|
+
)
|
|
971
|
+
if "enable_pinholes" in sim_sweep:
|
|
972
|
+
sim_sweep["enable_pinholes"] = [False]
|
|
973
|
+
else:
|
|
974
|
+
sim_fixed["enable_pinholes"] = False
|
|
975
|
+
# The wavelength sets only the light through the pinholes.
|
|
976
|
+
if "pinhole_visible_wavelength" in sim_sweep and not (pinhole_sizes and any(enable_ph_vals)):
|
|
977
|
+
raise ValueError(
|
|
978
|
+
"simulation.pinhole_visible_wavelength sets the light through the pinholes, which "
|
|
979
|
+
"are off, so a sweep of it would repeat one run. Give it one value, or turn the "
|
|
980
|
+
"pinholes on with enable_pinholes and pinhole_sizes.")
|
|
981
|
+
|
|
982
|
+
# Build sweep dimensions: every list-valued entry in a section becomes a named sweep dimension.
|
|
983
|
+
# Keys use "section.attribute" notation.
|
|
984
|
+
sweep_dims = {}
|
|
985
|
+
for attr, vals in sim_sweep.items():
|
|
986
|
+
sweep_dims[f"simulation.{attr}"] = vals
|
|
987
|
+
for attr, vals in det_sweep.items():
|
|
988
|
+
sweep_dims[f"detector.{attr}"] = vals
|
|
989
|
+
for attr, vals in tel_sweep.items():
|
|
990
|
+
sweep_dims[f"telescope.{attr}"] = vals
|
|
991
|
+
for attr, vals in fil_sweep.items():
|
|
992
|
+
if instrument != "EIS":
|
|
993
|
+
sweep_dims[f"filter.{attr}"] = vals
|
|
994
|
+
|
|
995
|
+
dim_names = list(sweep_dims.keys())
|
|
996
|
+
dim_values = [sweep_dims[n] for n in dim_names]
|
|
997
|
+
total_combinations = 1
|
|
998
|
+
for v in dim_values:
|
|
999
|
+
total_combinations *= len(v)
|
|
1000
|
+
|
|
1001
|
+
print(f"\nRunning {total_combinations} parameter combination(s).")
|
|
1002
|
+
if sweep_dims:
|
|
1003
|
+
print("Swept dimensions:")
|
|
1004
|
+
for dim, vals in sweep_dims.items():
|
|
1005
|
+
print(f" {dim}: {vals}")
|
|
1006
|
+
|
|
1007
|
+
# Load or create input cube
|
|
1008
|
+
raster = None
|
|
1009
|
+
raster_summed = {}
|
|
1010
|
+
raster_cubes = {}
|
|
1011
|
+
synthesis = None
|
|
1012
|
+
# Metadata kept with the cubes from a single synthesis file, as with a
|
|
1013
|
+
# pickle: its dynamic mode.
|
|
1014
|
+
file_meta = {}
|
|
1015
|
+
if uniform_intensity_mode:
|
|
1016
|
+
# A uniform intensity has no position along the slit for the slit's
|
|
1017
|
+
# fiducials to block, so the option is refused rather than ignored.
|
|
1018
|
+
if sim_fixed.get("slit_mask") or True in sim_sweep.get("slit_mask", []):
|
|
1019
|
+
raise ValueError("simulation.slit_mask needs a scene along the slit, from a "
|
|
1020
|
+
"synthesis or an atmosphere, not a uniform intensity.")
|
|
1021
|
+
cube_sim = None
|
|
1022
|
+
print("\nSkipping atmosphere loading (uniform intensity mode).")
|
|
1023
|
+
elif atmosphere_series_mode:
|
|
1024
|
+
# The cubes are synthesised per combination in the loop, since the
|
|
1025
|
+
# slit width and exposure time decide what the slit sees.
|
|
1026
|
+
cube_sim = None
|
|
1027
|
+
print("\nReading the atmosphere series...")
|
|
1028
|
+
series = AtmosphereSeries(series_paths)
|
|
1029
|
+
print(f" {len(series)} snapshots from {series.times[0]:.3f} to {series.times[-1]:.3f}")
|
|
1030
|
+
# Check the plan before computing the contribution functions, which
|
|
1031
|
+
# take minutes and gigabytes.
|
|
1032
|
+
RasterSynthesiser.check_plan(series, raster_plan, *_swept("slit_width", "expos"))
|
|
1033
|
+
raster = RasterSynthesiser(series, synthesis_settings)
|
|
1034
|
+
elif synthesis_series_mode:
|
|
1035
|
+
# The spectra are read per combination in the loop, a strip of
|
|
1036
|
+
# columns at a time, each column once.
|
|
1037
|
+
cube_sim = None
|
|
1038
|
+
print("\nReading the synthesis series...")
|
|
1039
|
+
series = SynthesisSeries(series_paths, reference_line)
|
|
1040
|
+
print(f" {len(series)} snapshots from {series.times[0]:.3f} to {series.times[-1]:.3f}")
|
|
1041
|
+
print(f" Lines in the window of {reference_line}: {', '.join(series.lines)}")
|
|
1042
|
+
SynthesisRaster.check_plan(series, raster_plan, *_swept("slit_width", "expos"))
|
|
1043
|
+
raster = SynthesisRaster(series)
|
|
1044
|
+
else:
|
|
1045
|
+
print(f"\nLoading the synthesis from {synthesis_file}...")
|
|
1046
|
+
print(f"Using '{reference_line}' as reference line for wavelength grid and metadata...")
|
|
1047
|
+
if synthesis_is_hdf5:
|
|
1048
|
+
# Read only the lines that reach the reference line's window and
|
|
1049
|
+
# sum them on its wavelengths. The sum is resampled onto the
|
|
1050
|
+
# detector grid per slit width in the loop.
|
|
1051
|
+
synthesis = read_synthesis(synthesis_file, reference_line)
|
|
1052
|
+
print(f" Lines in its window: {', '.join(synthesis.lines)}")
|
|
1053
|
+
if synthesis.source:
|
|
1054
|
+
print(f" Source: {synthesis.source}")
|
|
1055
|
+
products = read_synthesis_products(synthesis_file, keys=("dynamic_mode",))
|
|
1056
|
+
dynamic_mode_info = products.get("dynamic_mode", {"enabled": False})
|
|
1057
|
+
summed_input = synthesis.summed(reference_line)
|
|
1058
|
+
# Saved in the results as the spectra the instrument observed, if
|
|
1059
|
+
# their wavelengths are evenly spaced so a WCS can describe them.
|
|
1060
|
+
file_meta = {"dynamic_mode": dynamic_mode_info}
|
|
1061
|
+
cube_sim = (synthesis.summed_cube(reference_line, summed_input, file_meta)
|
|
1062
|
+
if synthesis.evenly_spaced(reference_line) else None)
|
|
1063
|
+
else:
|
|
1064
|
+
cube_sim, dynamic_mode_info = load_atmosphere(synthesis_file, reference_line)
|
|
1065
|
+
|
|
1066
|
+
is_dynamic_mode = dynamic_mode_info.get("enabled", False)
|
|
1067
|
+
if is_dynamic_mode:
|
|
1068
|
+
print("Synthesis was done in DYNAMIC MODE (time-varying atmosphere)")
|
|
1069
|
+
print(f" Slit width: {dynamic_mode_info['slit_width']}")
|
|
1070
|
+
print(f" Slit rest time: {dynamic_mode_info['slit_rest_time']}")
|
|
1071
|
+
print(f" Timesteps used: {len(dynamic_mode_info['available_timesteps'])}")
|
|
1072
|
+
|
|
1073
|
+
synth_slit_width = dynamic_mode_info["slit_width"]
|
|
1074
|
+
synth_rest_time = dynamic_mode_info["slit_rest_time"]
|
|
1075
|
+
|
|
1076
|
+
slit_width_vals = sweep_dims.get(
|
|
1077
|
+
"simulation.slit_width", [sim_fixed["slit_width"]]
|
|
1078
|
+
)
|
|
1079
|
+
expos_vals = sweep_dims.get(
|
|
1080
|
+
"simulation.expos", [sim_fixed["expos"]]
|
|
1081
|
+
)
|
|
1082
|
+
|
|
1083
|
+
if len(slit_width_vals) != 1:
|
|
1084
|
+
raise ValueError(
|
|
1085
|
+
f"Dynamic mode synthesis requires exactly one slit width. "
|
|
1086
|
+
f"Config specifies {len(slit_width_vals)}: {slit_width_vals}. "
|
|
1087
|
+
f"Please provide only the synthesis slit width: {synth_slit_width}"
|
|
1088
|
+
)
|
|
1089
|
+
if not np.isclose(
|
|
1090
|
+
slit_width_vals[0].to_value(u.arcsec),
|
|
1091
|
+
synth_slit_width.to_value(u.arcsec),
|
|
1092
|
+
rtol=1e-6,
|
|
1093
|
+
):
|
|
1094
|
+
raise ValueError(
|
|
1095
|
+
f"Slit width mismatch: synthesis was done with {synth_slit_width}, "
|
|
1096
|
+
f"but config specifies {slit_width_vals[0]}."
|
|
1097
|
+
)
|
|
1098
|
+
if len(expos_vals) != 1:
|
|
1099
|
+
raise ValueError(
|
|
1100
|
+
f"Dynamic mode synthesis requires exactly one exposure time. "
|
|
1101
|
+
f"Config specifies {len(expos_vals)}: {expos_vals}. "
|
|
1102
|
+
f"Please provide only the synthesis slit rest time: {synth_rest_time}"
|
|
1103
|
+
)
|
|
1104
|
+
if not np.isclose(
|
|
1105
|
+
expos_vals[0].to_value(u.s),
|
|
1106
|
+
synth_rest_time.to_value(u.s),
|
|
1107
|
+
rtol=1e-6,
|
|
1108
|
+
):
|
|
1109
|
+
raise ValueError(
|
|
1110
|
+
f"Exposure time mismatch: synthesis was done with {synth_rest_time}, "
|
|
1111
|
+
f"but config specifies {expos_vals[0]}."
|
|
1112
|
+
)
|
|
1113
|
+
print(" Dynamic mode parameters validated successfully!")
|
|
1114
|
+
|
|
1115
|
+
# Main sweep loop
|
|
1116
|
+
all_results = {}
|
|
1117
|
+
# cube_reb_cache: keyed by (slit_width_arcsec, plate_scale, wvl_res)
|
|
1118
|
+
cube_reb_cache = {}
|
|
1119
|
+
# rebin_cache: keyed by (slit_width_arcsec, plate_scale, wvl_res, offchip_bin_slit)
|
|
1120
|
+
rebin_cache = {}
|
|
1121
|
+
# observed_cache: cubes the Monte Carlo observes through the PSF, keyed by
|
|
1122
|
+
# sampling and PSF.
|
|
1123
|
+
observed_cache = {}
|
|
1124
|
+
# Synthesis-series raster, its metadata and its summed spectra, by sampling.
|
|
1125
|
+
raster_syntheses = {}
|
|
1126
|
+
# Keyed by slit_width_arcsec (first match) for convenient downstream access
|
|
1127
|
+
cube_reb_dict = {}
|
|
1128
|
+
|
|
1129
|
+
# Add offchip_bin_slit as a sweep dimension if needed
|
|
1130
|
+
if len(offchip_bin_slits) > 1:
|
|
1131
|
+
sweep_dims["offchip_bin_slit"] = offchip_bin_slits
|
|
1132
|
+
dim_names = list(sweep_dims.keys())
|
|
1133
|
+
dim_values = [sweep_dims[n] for n in dim_names]
|
|
1134
|
+
total_combinations = 1
|
|
1135
|
+
for v in dim_values:
|
|
1136
|
+
total_combinations *= len(v)
|
|
1137
|
+
print(f"\nUpdated to {total_combinations} parameter combination(s) (including offchip_bin_slit sweep).")
|
|
1138
|
+
|
|
1139
|
+
# Each raster of a time series is observed on its own, so repeated
|
|
1140
|
+
# rasters, or the exposures of a sit-and-stare, are swept over.
|
|
1141
|
+
if raster_mode and raster_plan.repeats > 1:
|
|
1142
|
+
sweep_dims["raster.repeat"] = list(range(raster_plan.repeats))
|
|
1143
|
+
dim_names = list(sweep_dims.keys())
|
|
1144
|
+
dim_values = [sweep_dims[n] for n in dim_names]
|
|
1145
|
+
total_combinations = 1
|
|
1146
|
+
for v in dim_values:
|
|
1147
|
+
total_combinations *= len(v)
|
|
1148
|
+
print(f"\nUpdated to {total_combinations} parameter combination(s) (one per raster repeat).")
|
|
1149
|
+
|
|
1150
|
+
def _combinations():
|
|
1151
|
+
"""New iterator over the sweep's combinations, yielding one at a time."""
|
|
1152
|
+
return itertools_product(*dim_values) if dim_names else iter([()])
|
|
1153
|
+
|
|
1154
|
+
def _combination(combo_values):
|
|
1155
|
+
"""Sweep values and config objects of one combination; the objects check the values."""
|
|
1156
|
+
combo = dict(zip(dim_names, combo_values)) if dim_names else {}
|
|
1157
|
+
offchip = combo.pop("offchip_bin_slit", offchip_bin_slits[0])
|
|
1158
|
+
repeat = combo.pop("raster.repeat", 0)
|
|
1159
|
+
|
|
1160
|
+
# Merge sweep values with fixed values for this combination
|
|
1161
|
+
all_sim = {
|
|
1162
|
+
**sim_fixed,
|
|
1163
|
+
**{k[len("simulation."):]: v for k, v in combo.items() if k.startswith("simulation.")},
|
|
1164
|
+
}
|
|
1165
|
+
all_det = {
|
|
1166
|
+
**det_fixed,
|
|
1167
|
+
**{k[len("detector."):]: v for k, v in combo.items() if k.startswith("detector.")},
|
|
1168
|
+
}
|
|
1169
|
+
all_det = {det_renamed.get(k, k): v for k, v in all_det.items()}
|
|
1170
|
+
all_tel = {
|
|
1171
|
+
**tel_fixed,
|
|
1172
|
+
**{k[len("telescope."):]: v for k, v in combo.items() if k.startswith("telescope.")},
|
|
1173
|
+
}
|
|
1174
|
+
all_fil = {
|
|
1175
|
+
**fil_fixed,
|
|
1176
|
+
**{k[len("filter."):]: v for k, v in combo.items() if k.startswith("filter.")},
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
# Build config objects
|
|
1180
|
+
if instrument == "SWC":
|
|
1181
|
+
fil = AluminiumFilter(**all_fil) if all_fil else AluminiumFilter()
|
|
1182
|
+
tel_kwargs = {k: v for k, v in all_tel.items() if k != "filter"}
|
|
1183
|
+
tel_kwargs["filter"] = fil
|
|
1184
|
+
tel = Telescope_EUVST(**tel_kwargs)
|
|
1185
|
+
det = Detector_SWC(**all_det) if all_det else Detector_SWC()
|
|
1186
|
+
else:
|
|
1187
|
+
fil = None
|
|
1188
|
+
tel_kwargs = {k: v for k, v in all_tel.items() if k != "filter"}
|
|
1189
|
+
tel = Telescope_EIS(**tel_kwargs) if tel_kwargs else Telescope_EIS()
|
|
1190
|
+
det = Detector_EIS(**all_det) if all_det else Detector_EIS()
|
|
1191
|
+
|
|
1192
|
+
enable = all_sim.get("enable_pinholes", False)
|
|
1193
|
+
sim = Simulation(
|
|
1194
|
+
expos=all_sim["expos"],
|
|
1195
|
+
n_iter=n_iter,
|
|
1196
|
+
slit_width=all_sim["slit_width"],
|
|
1197
|
+
ncpu=ncpu,
|
|
1198
|
+
instrument=instrument,
|
|
1199
|
+
vis_sl=all_sim.get("vis_sl", 0.0 * u.photon / (u.s * u.cm**2)),
|
|
1200
|
+
psf=all_sim.get("psf", False),
|
|
1201
|
+
psf_boundary=all_sim.get("psf_boundary", "replicate"),
|
|
1202
|
+
spectral_psf=all_sim.get("spectral_psf", "quadrature"),
|
|
1203
|
+
slit_mask=all_sim.get("slit_mask", False),
|
|
1204
|
+
focal_plane_dispersion=all_sim.get("focal_plane_dispersion", False),
|
|
1205
|
+
noise=all_sim.get("noise", True),
|
|
1206
|
+
readout_dark_current=all_sim.get("readout_dark_current", True),
|
|
1207
|
+
enable_pinholes=enable,
|
|
1208
|
+
pinhole_sizes=pinhole_sizes if enable else [],
|
|
1209
|
+
pinhole_positions=pinhole_positions if enable else [],
|
|
1210
|
+
pinhole_positions_spectral=pinhole_positions_spectral if enable else [],
|
|
1211
|
+
pinhole_visible_wavelength=all_sim.get("pinhole_visible_wavelength",
|
|
1212
|
+
Simulation.pinhole_visible_wavelength),
|
|
1213
|
+
)
|
|
1214
|
+
# The optics' spectral FWHM depends on the telescope and the detector
|
|
1215
|
+
# together. Work it out here, not only once the scene is laid onto
|
|
1216
|
+
# the detector. It is needed when psf_params is for a given slit, and
|
|
1217
|
+
# always for the PSF convolved with the slit.
|
|
1218
|
+
if sim.focal_plane_dispersion:
|
|
1219
|
+
det = _with_focal_plane_width(det, all_det)
|
|
1220
|
+
if tel.psf_slit_width is not None or (sim.psf and sim.spectral_psf == "convolution"):
|
|
1221
|
+
spectral_optics_fwhm(tel, det)
|
|
1222
|
+
return combo, offchip, repeat, fil, tel, det, sim
|
|
1223
|
+
|
|
1224
|
+
def _with_focal_plane_width(det, all_det):
|
|
1225
|
+
"""`det` with the focal plane's pixel width where the observed line lands."""
|
|
1226
|
+
if "wvl_res" in all_det:
|
|
1227
|
+
raise ValueError("simulation.focal_plane_dispersion takes the pixels' width from "
|
|
1228
|
+
"the focal plane, so detector.wvl_res would be ignored; give one "
|
|
1229
|
+
"or the other.")
|
|
1230
|
+
wavelength = _observed_wavelength()
|
|
1231
|
+
low, high = det.lit_band
|
|
1232
|
+
if not low <= wavelength <= high:
|
|
1233
|
+
raise ValueError(f"simulation.focal_plane_dispersion needs the observed line on the "
|
|
1234
|
+
f"focal plane, but {wavelength} is outside the lit band, "
|
|
1235
|
+
f"{low} to {high}.")
|
|
1236
|
+
try:
|
|
1237
|
+
width = FocalPlane_SWC.from_detector(det).pixel_width(wavelength)
|
|
1238
|
+
except ValueError as error:
|
|
1239
|
+
raise ValueError(f"simulation.focal_plane_dispersion needs the observed line on the "
|
|
1240
|
+
f"focal plane: {error}") from None
|
|
1241
|
+
return dataclasses.replace(det, wvl_res=width.to(det.wvl_res.unit))
|
|
1242
|
+
|
|
1243
|
+
observed_wavelength = []
|
|
1244
|
+
|
|
1245
|
+
def _observed_wavelength():
|
|
1246
|
+
"""Rest wavelength of the line the run observes, read once."""
|
|
1247
|
+
if not observed_wavelength:
|
|
1248
|
+
if uniform_intensity_mode:
|
|
1249
|
+
found = uniform_rest_wavelength
|
|
1250
|
+
elif synthesis_series_mode or (not raster_mode and synthesis_is_hdf5):
|
|
1251
|
+
path = series_paths[0] if synthesis_series_mode else synthesis_file
|
|
1252
|
+
layout = read_synthesis_layout(path, reference_line)
|
|
1253
|
+
found = layout["lines"][reference_line]["rest_wavelength"]
|
|
1254
|
+
else:
|
|
1255
|
+
raise ValueError("simulation.focal_plane_dispersion needs a synthesis file or a "
|
|
1256
|
+
"uniform intensity.")
|
|
1257
|
+
observed_wavelength.append(u.Quantity(found).to(u.AA))
|
|
1258
|
+
return observed_wavelength[0]
|
|
1259
|
+
|
|
1260
|
+
# Check every combination before running any, so a bad value does not
|
|
1261
|
+
# stop the sweep part-way and lose the earlier results, which are saved
|
|
1262
|
+
# only at the end.
|
|
1263
|
+
for combo_values in _combinations():
|
|
1264
|
+
_, _, _, _, tel, _, _ = _combination(combo_values)
|
|
1265
|
+
# Read the throughput tables now too, so a missing or faulty table is
|
|
1266
|
+
# found before the first photons are counted. They cannot be swept, so
|
|
1267
|
+
# the last combination's tables are every combination's.
|
|
1268
|
+
if instrument == "SWC":
|
|
1269
|
+
for table in (tel.pm_table, tel.grating_table, tel.filter.al_table,
|
|
1270
|
+
tel.filter.oxide_table, tel.filter.c_table):
|
|
1271
|
+
_load_throughput_table(table)
|
|
1272
|
+
|
|
1273
|
+
def _check_offchip_bins(fov_along_slit):
|
|
1274
|
+
"""Refuse an offchip_bin_slit that bins more rows than a combination's scene covers."""
|
|
1275
|
+
for combo_values in _combinations():
|
|
1276
|
+
_, n_bin, _, _, _, det, _ = _combination(combo_values)
|
|
1277
|
+
n_slit = _whole_pixels(fov_along_slit, det.plate_scale_angle)
|
|
1278
|
+
if n_bin > n_slit:
|
|
1279
|
+
raise ValueError(
|
|
1280
|
+
f"offchip_bin_slit {n_bin} bins more rows than the {n_slit} along the "
|
|
1281
|
+
f"slit, so there would be nothing left; bin at most {n_slit}.")
|
|
1282
|
+
|
|
1283
|
+
# The off-chip binning is checked against the scene when it is first
|
|
1284
|
+
# rebinned. A uniform intensity is made with the rows it bins.
|
|
1285
|
+
offchip_checked = uniform_intensity_mode
|
|
1286
|
+
|
|
1287
|
+
for combination_idx, combo_values in enumerate(_combinations(), start=1):
|
|
1288
|
+
(combo, offchip_bin_slit, raster_repeat, filter_obj, TEL, DET,
|
|
1289
|
+
SIM) = _combination(combo_values)
|
|
1290
|
+
|
|
1291
|
+
# Extract core simulation params
|
|
1292
|
+
slit_width = SIM.slit_width
|
|
1293
|
+
expos = SIM.expos
|
|
1294
|
+
vis_sl = SIM.vis_sl
|
|
1295
|
+
psf = SIM.psf
|
|
1296
|
+
slit_mask = SIM.slit_mask
|
|
1297
|
+
psf_boundary = SIM.psf_boundary
|
|
1298
|
+
spectral_psf = SIM.spectral_psf
|
|
1299
|
+
noise = SIM.noise
|
|
1300
|
+
enable_pinholes = SIM.enable_pinholes
|
|
1301
|
+
|
|
1302
|
+
# Two-level rebinning cache: rebin_atmosphere does not depend on offchip_bin_slit,
|
|
1303
|
+
# so cube_reb_cache is keyed by the sampling alone to avoid redundant rebin calls
|
|
1304
|
+
# when sweeping multiple binning values at fixed spatial/spectral sampling.
|
|
1305
|
+
# The spectral PSF's width sets how far the rebinned cube's wavelength
|
|
1306
|
+
# grid reaches, and it changes with the telescope's resolution budget,
|
|
1307
|
+
# which may be swept.
|
|
1308
|
+
sampling_key = (
|
|
1309
|
+
slit_width.to_value(u.arcsec),
|
|
1310
|
+
DET.plate_scale_angle.to_value(u.arcsec / u.pixel),
|
|
1311
|
+
DET.wvl_res.to_value(u.cm / u.pixel),
|
|
1312
|
+
spectral_psf_fwhm(TEL, DET, slit_width),
|
|
1313
|
+
)
|
|
1314
|
+
# In uniform-intensity mode the cube has offchip_bin_slit pixels along the
|
|
1315
|
+
# slit, giving rebin_slit_offchip independent noise realisations to sum, so
|
|
1316
|
+
# the key includes offchip_bin_slit. A time series is synthesised per
|
|
1317
|
+
# exposure time as well as per slit width.
|
|
1318
|
+
if uniform_intensity_mode:
|
|
1319
|
+
cube_reb_key = (*sampling_key, offchip_bin_slit)
|
|
1320
|
+
elif raster_mode:
|
|
1321
|
+
cube_reb_key = (*sampling_key, expos.to_value(u.s), raster_repeat)
|
|
1322
|
+
else:
|
|
1323
|
+
cube_reb_key = sampling_key
|
|
1324
|
+
rebin_cache_key = (*cube_reb_key, offchip_bin_slit)
|
|
1325
|
+
|
|
1326
|
+
if atmosphere_series_mode and cube_reb_key not in cube_reb_cache:
|
|
1327
|
+
print(f"\nSynthesising the time series as observed "
|
|
1328
|
+
f"(slit_width={slit_width}, expos={expos})...")
|
|
1329
|
+
cube_sim = raster.summed_cube(raster_plan, slit_width, expos, reference_line,
|
|
1330
|
+
repeat=raster_repeat)
|
|
1331
|
+
raster_summed[cube_reb_key] = cube_sim
|
|
1332
|
+
print(f" {cube_sim.data.shape[1]} exposures, {raster.strips_synthesised} "
|
|
1333
|
+
f"strips synthesised so far")
|
|
1334
|
+
if synthesis_series_mode and cube_reb_key not in cube_reb_cache:
|
|
1335
|
+
print(f"\nReading the time series as observed "
|
|
1336
|
+
f"(slit_width={slit_width}, expos={expos})...")
|
|
1337
|
+
# The exposures form a synthesis, one column each, which goes onto
|
|
1338
|
+
# the detector like a single snapshot.
|
|
1339
|
+
synthesis, raster_meta = raster.synthesis(raster_plan, slit_width, expos,
|
|
1340
|
+
repeat=raster_repeat)
|
|
1341
|
+
summed_input = synthesis.summed(reference_line)
|
|
1342
|
+
cube_sim = None
|
|
1343
|
+
if synthesis.evenly_spaced(reference_line):
|
|
1344
|
+
cube_sim = synthesis.summed_cube(reference_line, summed_input, raster_meta)
|
|
1345
|
+
raster_summed[cube_reb_key] = cube_sim
|
|
1346
|
+
raster_syntheses[cube_reb_key] = (synthesis, raster_meta, summed_input)
|
|
1347
|
+
print(f" {len(raster_meta['positions'])} exposures, {raster.strips_read} "
|
|
1348
|
+
f"strips read so far")
|
|
1349
|
+
|
|
1350
|
+
if cube_reb_key not in cube_reb_cache:
|
|
1351
|
+
print(
|
|
1352
|
+
f"\nRebinning atmosphere "
|
|
1353
|
+
f"(slit_width={slit_width}, "
|
|
1354
|
+
f"plate_scale={DET.plate_scale_angle}, "
|
|
1355
|
+
f"wvl_res={DET.wvl_res})..."
|
|
1356
|
+
)
|
|
1357
|
+
SIM_rebin = Simulation(
|
|
1358
|
+
expos=1.0 * u.s,
|
|
1359
|
+
n_iter=n_iter,
|
|
1360
|
+
slit_width=slit_width,
|
|
1361
|
+
ncpu=ncpu,
|
|
1362
|
+
instrument=instrument,
|
|
1363
|
+
psf=False,
|
|
1364
|
+
)
|
|
1365
|
+
if uniform_intensity_mode:
|
|
1366
|
+
cube_reb_cache[cube_reb_key] = create_uniform_intensity_cube(
|
|
1367
|
+
total_intensity=uniform_intensity,
|
|
1368
|
+
rest_wavelength=uniform_rest_wavelength,
|
|
1369
|
+
thermal_width=uniform_thermal_width,
|
|
1370
|
+
det=DET,
|
|
1371
|
+
sim=SIM_rebin,
|
|
1372
|
+
n_slit_pixels=offchip_bin_slit,
|
|
1373
|
+
tel=TEL,
|
|
1374
|
+
)
|
|
1375
|
+
else:
|
|
1376
|
+
# A slit wider than the one psf_params is for spreads each line
|
|
1377
|
+
# beyond the synthesis window's margin, so the window is padded
|
|
1378
|
+
# to hold it. The reference slit's window is unchanged.
|
|
1379
|
+
if synthesis is None:
|
|
1380
|
+
rebinned = rebin_atmosphere(cube_sim, DET, SIM_rebin)
|
|
1381
|
+
else:
|
|
1382
|
+
rebinned = rebin_spectra(
|
|
1383
|
+
synthesis, reference_line, DET, SIM_rebin, summed=summed_input,
|
|
1384
|
+
meta=raster_meta if synthesis_series_mode else file_meta)
|
|
1385
|
+
cube_reb_cache[cube_reb_key] = pad_spectral_axis(
|
|
1386
|
+
rebinned, spectral_psf_margin(TEL, DET, slit_width))
|
|
1387
|
+
if not offchip_checked:
|
|
1388
|
+
# The scene has the same length along the slit for every
|
|
1389
|
+
# combination, so each one's number of rows is known now,
|
|
1390
|
+
# before any has run.
|
|
1391
|
+
_check_offchip_bins(cube_reb_cache[cube_reb_key].meta["fov_along_slit"])
|
|
1392
|
+
offchip_checked = True
|
|
1393
|
+
|
|
1394
|
+
cube_reb = cube_reb_cache[cube_reb_key]
|
|
1395
|
+
|
|
1396
|
+
if rebin_cache_key not in rebin_cache:
|
|
1397
|
+
# Apply off-chip binning
|
|
1398
|
+
cube_reb_binned = rebin_slit_offchip(cube_reb, offchip_bin_slit)
|
|
1399
|
+
|
|
1400
|
+
# A weighted ground-truth fit uses the expected DN uncertainties,
|
|
1401
|
+
# which depend on the exposure and the instrument, so it is
|
|
1402
|
+
# fitted for each combination below.
|
|
1403
|
+
ground_truth = (None if weighted_truth
|
|
1404
|
+
else _fit_ground_truth(cube_reb_binned, offchip_bin_slit))
|
|
1405
|
+
rebin_cache[rebin_cache_key] = (cube_reb_binned, ground_truth)
|
|
1406
|
+
# Key by (slit_width_arcsec, offchip_bin_slit) so sweeps over
|
|
1407
|
+
# several binning factors at a fixed slit width keep all their
|
|
1408
|
+
# cubes; a single key would silently keep only the first. A time
|
|
1409
|
+
# series adds the exposure time, which changes the cube too.
|
|
1410
|
+
cube_key = ((sampling_key[0], expos.to_value(u.s), raster_repeat, offchip_bin_slit)
|
|
1411
|
+
if raster_mode else (sampling_key[0], offchip_bin_slit))
|
|
1412
|
+
cube_reb_dict.setdefault(cube_key, cube_reb_binned)
|
|
1413
|
+
if raster_mode:
|
|
1414
|
+
raster_cubes[cube_key] = raster_summed[cube_reb_key]
|
|
1415
|
+
|
|
1416
|
+
cube_reb_binned, ground_truth = rebin_cache[rebin_cache_key]
|
|
1417
|
+
|
|
1418
|
+
# The cube the Monte Carlo observes: the scene laid onto the pixels
|
|
1419
|
+
# through the telescope, with each photon counted at its own
|
|
1420
|
+
# wavelength. With the PSF on, the scene is also blurred by the PSF
|
|
1421
|
+
# before it is laid onto the pixels. This is exact, whereas blurring
|
|
1422
|
+
# the pixels afterwards moves a narrow line within its pixel. With
|
|
1423
|
+
# slit_mask on, the slit's ends and fiducials are cut out in the same
|
|
1424
|
+
# step. The ground truth is the scene without any of these. A uniform
|
|
1425
|
+
# intensity is made again through the spectral response, which
|
|
1426
|
+
# depends on the slit, the telescope and spectral_psf.
|
|
1427
|
+
cube_obs = cube_reb
|
|
1428
|
+
# The observed cube includes the telescope's throughput as well as its
|
|
1429
|
+
# PSF, so every telescope and filter setting is in the key, except the
|
|
1430
|
+
# one for visible light, which the Monte Carlo adds. With the pinholes
|
|
1431
|
+
# on, it also includes the light they let past the filter.
|
|
1432
|
+
filter_params = (_extract_config_params(filter_obj, "filter")
|
|
1433
|
+
if filter_obj is not None else {})
|
|
1434
|
+
filter_params.pop("filter.visible_decade_thickness", None)
|
|
1435
|
+
telescope_key = _params_to_key({**_extract_config_params(TEL, "telescope"),
|
|
1436
|
+
**filter_params})
|
|
1437
|
+
pinholes = _pinholes_on(SIM)
|
|
1438
|
+
if uniform_intensity_mode:
|
|
1439
|
+
if psf:
|
|
1440
|
+
psf_key = (*cube_reb_key, spectral_psf, telescope_key, pinholes)
|
|
1441
|
+
if psf_key not in observed_cache:
|
|
1442
|
+
observed_cache[psf_key] = create_uniform_intensity_cube(
|
|
1443
|
+
total_intensity=uniform_intensity,
|
|
1444
|
+
rest_wavelength=uniform_rest_wavelength,
|
|
1445
|
+
thermal_width=uniform_thermal_width,
|
|
1446
|
+
det=DET,
|
|
1447
|
+
sim=SIM,
|
|
1448
|
+
n_slit_pixels=offchip_bin_slit,
|
|
1449
|
+
tel=TEL,
|
|
1450
|
+
)
|
|
1451
|
+
cube_obs = observed_cache[psf_key]
|
|
1452
|
+
else:
|
|
1453
|
+
psf_key = (*cube_reb_key, psf, slit_mask, spectral_psf, psf_boundary, telescope_key,
|
|
1454
|
+
pinholes)
|
|
1455
|
+
if psf_key not in observed_cache:
|
|
1456
|
+
print("Laying the scene onto the detector through the telescope...")
|
|
1457
|
+
SIM_obs = Simulation(expos=1.0 * u.s, n_iter=n_iter, slit_width=slit_width,
|
|
1458
|
+
ncpu=ncpu, instrument=instrument, psf=psf,
|
|
1459
|
+
psf_boundary=psf_boundary, spectral_psf=spectral_psf,
|
|
1460
|
+
slit_mask=slit_mask,
|
|
1461
|
+
enable_pinholes=SIM.enable_pinholes,
|
|
1462
|
+
pinhole_sizes=list(SIM.pinhole_sizes),
|
|
1463
|
+
pinhole_positions=list(SIM.pinhole_positions),
|
|
1464
|
+
pinhole_positions_spectral=list(
|
|
1465
|
+
SIM.pinhole_positions_spectral))
|
|
1466
|
+
across = across_slit_fwhm(TEL) if psf else None
|
|
1467
|
+
extend = psf_boundary == "replicate"
|
|
1468
|
+
if atmosphere_series_mode:
|
|
1469
|
+
scene = raster_summed[cube_reb_key] if across is None else raster.summed_cube(
|
|
1470
|
+
raster_plan, slit_width, expos, reference_line, repeat=raster_repeat,
|
|
1471
|
+
across_slit=across, extend=extend)
|
|
1472
|
+
observed = rebin_atmosphere(scene, DET, SIM_obs, tel=TEL)
|
|
1473
|
+
elif synthesis_series_mode:
|
|
1474
|
+
series_synthesis, series_meta, series_summed = raster_syntheses[cube_reb_key]
|
|
1475
|
+
if across is not None:
|
|
1476
|
+
series_synthesis, series_meta = raster.synthesis(
|
|
1477
|
+
raster_plan, slit_width, expos, repeat=raster_repeat,
|
|
1478
|
+
across_slit=across, extend=extend)
|
|
1479
|
+
series_summed = None
|
|
1480
|
+
observed = rebin_spectra(series_synthesis, reference_line, DET, SIM_obs,
|
|
1481
|
+
summed=series_summed, meta=series_meta, tel=TEL)
|
|
1482
|
+
elif synthesis is None:
|
|
1483
|
+
observed = rebin_atmosphere(cube_sim, DET, SIM_obs, tel=TEL)
|
|
1484
|
+
else:
|
|
1485
|
+
observed = rebin_spectra(synthesis, reference_line, DET, SIM_obs,
|
|
1486
|
+
summed=summed_input, meta=file_meta, tel=TEL)
|
|
1487
|
+
observed_cache[psf_key] = observed
|
|
1488
|
+
cube_obs = observed_cache[psf_key]
|
|
1489
|
+
|
|
1490
|
+
# Progress output
|
|
1491
|
+
print(f"\n--- Combination {combination_idx}/{total_combinations} ---")
|
|
1492
|
+
for k, v in combo.items():
|
|
1493
|
+
print(f" {k}: {v}")
|
|
1494
|
+
if offchip_bin_slit > 1:
|
|
1495
|
+
print(f" offchip_bin_slit: {offchip_bin_slit}")
|
|
1496
|
+
if raster_mode and raster_plan.repeats > 1:
|
|
1497
|
+
print(f" raster.repeat: {raster_repeat}")
|
|
1498
|
+
if not combo and offchip_bin_slit == 1 and not (raster_mode and raster_plan.repeats > 1):
|
|
1499
|
+
print(" (single combination - all parameters fixed)")
|
|
1500
|
+
print(f" Calculated dark current: {DET.dark_current:.2e}")
|
|
1501
|
+
if instrument == "SWC":
|
|
1502
|
+
print(f" Microroughness sigma: {TEL.microroughness_sigma}")
|
|
1503
|
+
if enable_pinholes and pinhole_sizes:
|
|
1504
|
+
print(f" Pinhole sizes: {pinhole_sizes}")
|
|
1505
|
+
print(f" Pinhole positions: {pinhole_positions}")
|
|
1506
|
+
|
|
1507
|
+
if weighted_truth:
|
|
1508
|
+
# Weight the ground truth by the DN uncertainty expected with no
|
|
1509
|
+
# noise, so it is the same fit, of the same line, as for the
|
|
1510
|
+
# measured spectra.
|
|
1511
|
+
expected = expected_dn_uncertainty(cube_obs, expos, DET, TEL, SIM, offchip_bin_slit,
|
|
1512
|
+
uniform_mode=uniform_intensity_mode)
|
|
1513
|
+
if expected.shape != cube_reb_binned.data.shape:
|
|
1514
|
+
raise RuntimeError(
|
|
1515
|
+
f"The detector's pixels, shaped {expected.shape}, do not match the "
|
|
1516
|
+
f"spectra the ground truth is fitted to, shaped "
|
|
1517
|
+
f"{cube_reb_binned.data.shape}.")
|
|
1518
|
+
ground_truth = _fit_ground_truth(cube_reb_binned, offchip_bin_slit, expected)
|
|
1519
|
+
|
|
1520
|
+
# Run Monte Carlo
|
|
1521
|
+
first_dn_signal, dn_fit_stats, first_photon_signal, photon_fit_stats = monte_carlo(
|
|
1522
|
+
cube_obs, expos, DET, TEL, SIM,
|
|
1523
|
+
n_iter=SIM.n_iter,
|
|
1524
|
+
fit_config=fit_config,
|
|
1525
|
+
offchip_bin_slit=offchip_bin_slit,
|
|
1526
|
+
fit_signals=fit_signals,
|
|
1527
|
+
uniform_mode=uniform_intensity_mode,
|
|
1528
|
+
)
|
|
1529
|
+
|
|
1530
|
+
# Only store results on rank 0 (MPI-aware)
|
|
1531
|
+
if _mpi_rank == 0 and first_dn_signal is not None:
|
|
1532
|
+
# Build parameters dict from actual config objects so all fields
|
|
1533
|
+
# (including those using class defaults) are recorded.
|
|
1534
|
+
parameters = {}
|
|
1535
|
+
parameters.update(_extract_config_params(SIM, "simulation"))
|
|
1536
|
+
parameters.update(_extract_config_params(DET, "detector"))
|
|
1537
|
+
if instrument == "SWC":
|
|
1538
|
+
parameters.update(_extract_config_params(filter_obj, "filter"))
|
|
1539
|
+
parameters.update(_extract_config_params(TEL, "telescope"))
|
|
1540
|
+
# Add offchip_bin_slit to the parameters dict
|
|
1541
|
+
parameters["offchip_bin_slit"] = offchip_bin_slit
|
|
1542
|
+
if raster_mode:
|
|
1543
|
+
parameters["raster.repeat"] = raster_repeat
|
|
1544
|
+
|
|
1545
|
+
param_key = _params_to_key(parameters)
|
|
1546
|
+
|
|
1547
|
+
all_results[param_key] = {
|
|
1548
|
+
"parameters": parameters,
|
|
1549
|
+
"config_objects": {
|
|
1550
|
+
"detector": DET,
|
|
1551
|
+
"telescope": TEL,
|
|
1552
|
+
"simulation": SIM,
|
|
1553
|
+
},
|
|
1554
|
+
"first_dn_signal_data": first_dn_signal.data,
|
|
1555
|
+
"first_dn_signal_unit": first_dn_signal.unit,
|
|
1556
|
+
"first_photon_signal_data": first_photon_signal.data,
|
|
1557
|
+
"first_photon_signal_unit": first_photon_signal.unit,
|
|
1558
|
+
"first_signal_wcs": first_dn_signal.wcs,
|
|
1559
|
+
"dn_fit_stats": dn_fit_stats,
|
|
1560
|
+
"photon_fit_stats": photon_fit_stats,
|
|
1561
|
+
"ground_truth": ground_truth,
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
del first_dn_signal, first_photon_signal, dn_fit_stats, photon_fit_stats
|
|
1565
|
+
|
|
1566
|
+
# Package results (rank 0 only in MPI mode)
|
|
1567
|
+
if _mpi_rank == 0:
|
|
1568
|
+
results = {
|
|
1569
|
+
"all_combinations": all_results,
|
|
1570
|
+
# All sweep dimensions and their value lists
|
|
1571
|
+
"sweep_dimensions": sweep_dims,
|
|
1572
|
+
# All fixed (non-swept) parameter values
|
|
1573
|
+
"fixed_params": {
|
|
1574
|
+
**{f"simulation.{k}": v for k, v in sim_fixed.items()},
|
|
1575
|
+
**{f"detector.{k}": v for k, v in det_fixed.items()},
|
|
1576
|
+
**{f"telescope.{k}": v for k, v in tel_fixed.items()},
|
|
1577
|
+
**(
|
|
1578
|
+
{f"filter.{k}": v for k, v in fil_fixed.items()}
|
|
1579
|
+
if instrument == "SWC"
|
|
1580
|
+
else {}
|
|
1581
|
+
),
|
|
1582
|
+
"offchip_bin_slit": offchip_bin_slits[0] if len(offchip_bin_slits) == 1 else None,
|
|
1583
|
+
**({"raster.repeat": 0} if raster_mode and raster_plan.repeats == 1 else {}),
|
|
1584
|
+
},
|
|
1585
|
+
"fit_config": fit_config,
|
|
1586
|
+
"fit_signals": fit_signals,
|
|
1587
|
+
# Whether the fits were weighted: DN by each pixel's uncertainty,
|
|
1588
|
+
# photons by their Poisson likelihood. Results made before this was
|
|
1589
|
+
# recorded were unweighted.
|
|
1590
|
+
"fit_weighted": weighted_fits,
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
# Save
|
|
1594
|
+
git_commit_id = get_git_commit_id()
|
|
1595
|
+
software_version = _get_software_version()
|
|
1596
|
+
|
|
1597
|
+
output_file = Path(f"run/result/{Path(args.config).stem}.h5")
|
|
1598
|
+
|
|
1599
|
+
print(f"\nSaving results to {output_file}")
|
|
1600
|
+
save_data = {
|
|
1601
|
+
"results": results,
|
|
1602
|
+
"config": config,
|
|
1603
|
+
"instrument": instrument,
|
|
1604
|
+
"cube_sim": cube_sim,
|
|
1605
|
+
"cube_reb_dict": cube_reb_dict,
|
|
1606
|
+
"git_commit_id": git_commit_id,
|
|
1607
|
+
"software_version": software_version,
|
|
1608
|
+
}
|
|
1609
|
+
if raster_mode:
|
|
1610
|
+
# How the series was observed, and the cube each combination saw;
|
|
1611
|
+
# cube_sim is the last combination's.
|
|
1612
|
+
save_data["raster"] = {
|
|
1613
|
+
"plan": raster_plan,
|
|
1614
|
+
"settings": synthesis_settings,
|
|
1615
|
+
"series": [str(p) for p in series.paths],
|
|
1616
|
+
"times": series.times,
|
|
1617
|
+
"hdf5_dbase_root": raster.goft_dbase_root if atmosphere_series_mode else None,
|
|
1618
|
+
"cubes": raster_cubes,
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
output_file = save_results(output_file, save_data)
|
|
1622
|
+
|
|
1623
|
+
print(f"Saved results to {output_file} ({os.path.getsize(output_file) / 1e6:.1f} MB)")
|
|
1624
|
+
# Move aside a results pickle of the same name left by an older
|
|
1625
|
+
# version, so it is not read in place of these results.
|
|
1626
|
+
stale = output_file.with_suffix(".pkl")
|
|
1627
|
+
if stale.is_file():
|
|
1628
|
+
# Do not overwrite a file moved aside before, which may be the
|
|
1629
|
+
# only copy of even older results.
|
|
1630
|
+
aside = stale.with_name(stale.name + ".old")
|
|
1631
|
+
number = 0
|
|
1632
|
+
while aside.exists():
|
|
1633
|
+
number += 1
|
|
1634
|
+
aside = stale.with_name(f"{stale.name}.old.{number}")
|
|
1635
|
+
try:
|
|
1636
|
+
os.replace(stale, aside)
|
|
1637
|
+
print(f"Moved {stale}, from an older version, to {aside.name}")
|
|
1638
|
+
except OSError as error:
|
|
1639
|
+
print(f"Could not move {stale}, from an older version, aside ({error}); "
|
|
1640
|
+
f"the results are {output_file}, not it")
|
|
1641
|
+
print(f"Software version: {software_version} | Git commit: {git_commit_id}")
|
|
1642
|
+
print(f"Instrument response simulation complete! Total combinations: {total_combinations}")
|
|
1643
|
+
|
|
1644
|
+
|
|
1645
|
+
if __name__ == "__main__":
|
|
1646
|
+
main()
|