eclipse 0.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eclipse/__init__.py +95 -0
- eclipse/analysis.py +853 -0
- eclipse/atmosphere.py +859 -0
- eclipse/band.py +477 -0
- eclipse/cli.py +138 -0
- eclipse/config.py +1710 -0
- eclipse/continuum.py +413 -0
- eclipse/data/eis_response/EIS_EffArea_A.004 +61 -0
- eclipse/data/eis_response/EIS_EffArea_B.004 +60 -0
- eclipse/data/eis_response/EIS_EffArea_B.005 +64 -0
- eclipse/data/eis_response/README_dz2025 +50 -0
- eclipse/data/eis_response/fit_eis_ea_2023-05-04_smooth.sav +0 -0
- eclipse/data/eis_response/source.txt +33 -0
- eclipse/data/science_cases.yaml +365 -0
- eclipse/data/throughput/grating_reflection_efficiency.dat +25 -0
- eclipse/data/throughput/primary_mirror_coating_reflectance.dat +25 -0
- eclipse/data/throughput/source.txt +6 -0
- eclipse/data/throughput/throughput_aluminium_1000_angstrom.dat +503 -0
- eclipse/data/throughput/throughput_aluminium_oxide_1000_angstrom.dat +503 -0
- eclipse/data/throughput/throughput_carbon_1000_angstrom.dat +503 -0
- eclipse/data_processing.py +904 -0
- eclipse/deprecated.py +161 -0
- eclipse/eis_calibration.py +573 -0
- eclipse/extern/__init__.py +0 -0
- eclipse/extern/mpfit.py +2388 -0
- eclipse/fitting.py +1734 -0
- eclipse/frame.py +594 -0
- eclipse/full_frame.py +944 -0
- eclipse/main.py +1646 -0
- eclipse/monte_carlo.py +660 -0
- eclipse/pinhole_diffraction.py +409 -0
- eclipse/psf.py +6 -0
- eclipse/radiometric.py +1328 -0
- eclipse/raster.py +869 -0
- eclipse/readout.py +1002 -0
- eclipse/resolution_budget.py +108 -0
- eclipse/results_file.py +1226 -0
- eclipse/sampling.py +351 -0
- eclipse/science_cases.py +364 -0
- eclipse/synthesis.py +2492 -0
- eclipse/synthesis_cli.py +12 -0
- eclipse/synthesis_file.py +882 -0
- eclipse/utils.py +1145 -0
- eclipse-0.13.0.dist-info/METADATA +70 -0
- eclipse-0.13.0.dist-info/RECORD +50 -0
- eclipse-0.13.0.dist-info/WHEEL +5 -0
- eclipse-0.13.0.dist-info/entry_points.txt +7 -0
- eclipse-0.13.0.dist-info/licenses/LICENSE +3 -0
- eclipse-0.13.0.dist-info/top_level.txt +2 -0
- euvst_response/__init__.py +129 -0
eclipse/results_file.py
ADDED
|
@@ -0,0 +1,1226 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Results file: the instrument simulation's results as HDF5.
|
|
3
|
+
|
|
4
|
+
`eclipse --config run.yaml` writes `run/result/run.h5`, which
|
|
5
|
+
:func:`eclipse.load_instrument_response_results` reads. Reading a
|
|
6
|
+
file runs no code from it, and any language can read it.
|
|
7
|
+
|
|
8
|
+
File layout, version 1
|
|
9
|
+
----------------------
|
|
10
|
+
Root attributes `format` (`"eclipse-results"`) and `version` (`1`). The
|
|
11
|
+
results are a tree. An array with dimensions is a dataset, with a `unit`
|
|
12
|
+
attribute if it is a quantity; a cube, or a mapping, list or tuple holding
|
|
13
|
+
such arrays, is a group; anything else is a JSON attribute of its group.
|
|
14
|
+
|
|
15
|
+
A mapping keyed by names lists them in order, as JSON, in `eclipse_order`.
|
|
16
|
+
Other groups have an `eclipse_type` (plain text, as are `eclipse_length`,
|
|
17
|
+
`target` and `unit`):
|
|
18
|
+
|
|
19
|
+
- `ndcube`: an NDCube's `data` with its `unit`, `wcs` and `meta`.
|
|
20
|
+
- `map`: a mapping whose keys cannot name members, such as results keyed
|
|
21
|
+
by parameter tuples; the keys as JSON in `eclipse_keys`, the values as
|
|
22
|
+
members `0`, `1`, ...
|
|
23
|
+
- `list`, `tuple`: the length in `eclipse_length`, the items as members.
|
|
24
|
+
- `ref`: a value found in more than one place, written once and elsewhere
|
|
25
|
+
as a group whose `target` is the path it was written at.
|
|
26
|
+
|
|
27
|
+
Values JSON cannot hold are objects with an `__eclipse__` entry naming the
|
|
28
|
+
kind: `float` (not finite), `numpy`, `quantity`, `array`, `unit`, `tuple`,
|
|
29
|
+
`map`, `date`, `datetime`, `set`, `frozenset`, `bytes` (base 64), `path`,
|
|
30
|
+
`resource` (a path inside the installed package), `numpy_type`, `wcs` (a
|
|
31
|
+
FITS header plus the unrounded `cunit`, `crpix`, `crval`, and `cdelt` and
|
|
32
|
+
`pc` or `crota`, or `cd`) and `dataclass` (a configuration object's
|
|
33
|
+
`class`, `fields` and `derived` values). Only the classes in
|
|
34
|
+
:func:`_dataclass_registry` are rebuilt.
|
|
35
|
+
|
|
36
|
+
Results pickles from older versions still load, with a warning, and
|
|
37
|
+
:func:`convert_results_pickle` converts one.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import base64
|
|
43
|
+
import dataclasses
|
|
44
|
+
import datetime
|
|
45
|
+
import functools
|
|
46
|
+
import json
|
|
47
|
+
import math
|
|
48
|
+
import os
|
|
49
|
+
import re
|
|
50
|
+
import sys
|
|
51
|
+
import warnings
|
|
52
|
+
from collections.abc import Mapping
|
|
53
|
+
from importlib.resources import files
|
|
54
|
+
from pathlib import Path, PurePosixPath
|
|
55
|
+
|
|
56
|
+
import astropy.units as u
|
|
57
|
+
import h5py
|
|
58
|
+
import numpy as np
|
|
59
|
+
from astropy.io import fits
|
|
60
|
+
from astropy.utils.masked import Masked
|
|
61
|
+
from astropy.wcs import WCS
|
|
62
|
+
from ndcube import NDCube
|
|
63
|
+
|
|
64
|
+
from .atmosphere import _check_format, _is_pickle, _new_partial
|
|
65
|
+
|
|
66
|
+
__all__ = ["FORMAT_NAME", "FORMAT_VERSION", "save_results", "load_results",
|
|
67
|
+
"convert_results_pickle", "is_results_file"]
|
|
68
|
+
|
|
69
|
+
FORMAT_NAME = "eclipse-results"
|
|
70
|
+
FORMAT_VERSION = 1
|
|
71
|
+
|
|
72
|
+
# Key that tags a JSON value this module must rebuild.
|
|
73
|
+
TAG = "__eclipse__"
|
|
74
|
+
# Group attributes that describe the group rather than hold one of its values.
|
|
75
|
+
TYPE, ORDER, KEYS, LENGTH = "eclipse_type", "eclipse_order", "eclipse_keys", "eclipse_length"
|
|
76
|
+
_RESERVED = {TYPE, ORDER, KEYS, LENGTH, "format", "version"}
|
|
77
|
+
|
|
78
|
+
# Suffixes older versions and scripts gave pickles, as accepted by the synthesis.
|
|
79
|
+
_PICKLE_SUFFIXES = (".pkl", ".pickle", ".dill")
|
|
80
|
+
|
|
81
|
+
# Arrays with at least this many elements are compressed.
|
|
82
|
+
_COMPRESS_FROM = 1024
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@functools.lru_cache(maxsize=None)
|
|
86
|
+
def _dataclass_registry() -> dict:
|
|
87
|
+
"""Classes that may be rebuilt from a file, keyed by name.
|
|
88
|
+
|
|
89
|
+
Imported when first needed, since the time series settings import the
|
|
90
|
+
synthesis, which reading results does not otherwise need.
|
|
91
|
+
"""
|
|
92
|
+
from .config import (AluminiumFilter, Detector_EIS, Detector_SWC,
|
|
93
|
+
Simulation, Telescope_EIS, Telescope_EUVST)
|
|
94
|
+
from .fitting import FitComponent, FitConfig
|
|
95
|
+
from .raster import RasterPlan, SynthesisSettings
|
|
96
|
+
|
|
97
|
+
return {cls.__name__: cls for cls in (
|
|
98
|
+
AluminiumFilter, Detector_EIS, Detector_SWC, Simulation,
|
|
99
|
+
Telescope_EIS, Telescope_EUVST, FitComponent, FitConfig,
|
|
100
|
+
RasterPlan, SynthesisSettings,
|
|
101
|
+
)}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _package_root() -> Path:
|
|
105
|
+
"""Directory of this ECLIPSE installation's package data."""
|
|
106
|
+
return Path(os.fspath(files("eclipse")))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _moved_package_file(path):
|
|
110
|
+
"""
|
|
111
|
+
This installation's copy of a packaged table named under the package's former name.
|
|
112
|
+
|
|
113
|
+
Older versions were installed as `euvst_response`, so their results can
|
|
114
|
+
name the packaged tables inside it, a folder an upgrade removes. None if
|
|
115
|
+
`path` exists, is not such a table, or the table is not in this installation.
|
|
116
|
+
A path that cannot be reached counts as missing.
|
|
117
|
+
"""
|
|
118
|
+
path = Path(os.fspath(path))
|
|
119
|
+
parts = path.parts
|
|
120
|
+
for i in range(len(parts) - 2, -1, -1):
|
|
121
|
+
if parts[i] == "euvst_response" and parts[i + 1] == "data":
|
|
122
|
+
relative = _package_parts(Path(*parts[i + 1:]).as_posix())
|
|
123
|
+
if relative is None or os.path.exists(path):
|
|
124
|
+
return None
|
|
125
|
+
moved = _package_root().joinpath(*relative)
|
|
126
|
+
return moved if moved.is_file() else None
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ----------------------------------------------------------------------
|
|
131
|
+
# WCS
|
|
132
|
+
# ----------------------------------------------------------------------
|
|
133
|
+
def _wcs_to_tree(wcs: WCS) -> dict:
|
|
134
|
+
"""
|
|
135
|
+
A WCS as a FITS header, plus the numbers of its axes unchanged.
|
|
136
|
+
|
|
137
|
+
A header gives the axes in SI units and to 14 digits, so a wavelength
|
|
138
|
+
axis in cm would come back in m, and a step of 0.2 arcsec as
|
|
139
|
+
0.2000000000000016. The coordinates agree to that precision, but not
|
|
140
|
+
the numbers a caller reads from `wcs.wcs`. So the units, reference
|
|
141
|
+
pixels and values, and the steps and rotation or CD matrix are kept
|
|
142
|
+
beside the header as given, and restored on reading.
|
|
143
|
+
"""
|
|
144
|
+
# Read these before to_header(), which normalises a WCS in place; it is
|
|
145
|
+
# called on a copy so that saving a cube leaves the caller's WCS alone.
|
|
146
|
+
tree = {"cunit": [str(c) for c in wcs.wcs.cunit], "crpix": wcs.wcs.crpix.tolist(),
|
|
147
|
+
"crval": wcs.wcs.crval.tolist()}
|
|
148
|
+
# The rotation in the form the WCS uses, checked in wcslib's order:
|
|
149
|
+
# a PC matrix (the default), then a CD matrix, then CROTA.
|
|
150
|
+
lin = wcs.wcs
|
|
151
|
+
if lin.has_pc() or not (lin.has_cd() or lin.has_crota()):
|
|
152
|
+
tree.update(cdelt=lin.cdelt.tolist(),
|
|
153
|
+
pc=lin.pc.tolist() if lin.has_pc() else np.eye(lin.naxis).tolist())
|
|
154
|
+
elif lin.has_cd():
|
|
155
|
+
tree["cd"] = lin.cd.tolist()
|
|
156
|
+
else:
|
|
157
|
+
tree.update(cdelt=lin.cdelt.tolist(), crota=lin.crota.tolist())
|
|
158
|
+
tree["header"] = wcs.deepcopy().to_header().tostring(sep="\n")
|
|
159
|
+
if wcs.has_distortion:
|
|
160
|
+
_warn("A results file does not hold a WCS's distortion, so this WCS's is left out.")
|
|
161
|
+
return tree
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# Header cards a WCS is rebuilt from: its axes, and any frame, time and
|
|
165
|
+
# observer. The number of axes (wcslib allocates its square) and the steps
|
|
166
|
+
# and rotation or CD matrix come from the values kept beside the header,
|
|
167
|
+
# since a header gives a CD matrix as a PC matrix and steps, which would
|
|
168
|
+
# take precedence. Other cards are left out, such as a distortion's, which
|
|
169
|
+
# astropy tabulates at whatever size the header asks.
|
|
170
|
+
_WCS_CARDS = re.compile(
|
|
171
|
+
r"WCSNAME|(CTYPE|CUNIT|CRPIX|CRVAL|CNAME|CRDER|CSYER|CZPHS|CPERI)\d{1,2}"
|
|
172
|
+
r"|(PV|PS)\d{1,2}_\d{1,2}|LONPOLE|LATPOLE|RADESYS|EQUINOX|RESTFRQ|RESTWAV|VELREF"
|
|
173
|
+
r"|SPECSYS|SSYSOBS|SSYSSRC|VELOSYS|ZSOURCE|VELANGL|TIMESYS|TREFPOS|TREFDIR|PLEPHEM"
|
|
174
|
+
r"|TIMEUNIT|TIMEDEL|TIMEPIXR|TIMEOFFS|TIMSYER|TIMRDER|TSTART|TSTOP|TELAPSE|XPOSURE"
|
|
175
|
+
r"|JEPOCH|BEPOCH|(DATE|MJD)-(OBS|BEG|AVG|END)|DATEREF|MJDREF[IF]?|OBSGEO-[XYZBLH]"
|
|
176
|
+
r"|OBSORBIT|DSUN_OBS|HGLN_OBS|HGLT_OBS|CRLN_OBS|CRLT_OBS|RSUN_REF|[ABC]_RADIUS"
|
|
177
|
+
r"|BLON_OBS|BLAT_OBS|BDIS_OBS")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _wcs_from_tree(tree: dict) -> WCS:
|
|
181
|
+
"""Rebuild a WCS written by :func:`_wcs_to_tree`."""
|
|
182
|
+
header = fits.Header.fromstring(tree["header"], sep="\n")
|
|
183
|
+
wcs = WCS(fits.Header([card for card in header.cards if _WCS_CARDS.fullmatch(card.keyword)]))
|
|
184
|
+
if wcs.naxis != len(tree["cunit"]):
|
|
185
|
+
raise ValueError("The results file holds a WCS whose header does not have its axes.")
|
|
186
|
+
for axis, unit in enumerate(tree["cunit"]):
|
|
187
|
+
wcs.wcs.cunit[axis] = unit
|
|
188
|
+
wcs.wcs.crpix, wcs.wcs.crval = tree["crpix"], tree["crval"]
|
|
189
|
+
if "cd" in tree:
|
|
190
|
+
wcs.wcs.cd = tree["cd"]
|
|
191
|
+
elif "crota" in tree:
|
|
192
|
+
wcs.wcs.cdelt, wcs.wcs.crota = tree["cdelt"], tree["crota"]
|
|
193
|
+
else:
|
|
194
|
+
wcs.wcs.cdelt, wcs.wcs.pc = tree["cdelt"], tree["pc"]
|
|
195
|
+
return wcs
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
# ----------------------------------------------------------------------
|
|
199
|
+
# Values as JSON
|
|
200
|
+
# ----------------------------------------------------------------------
|
|
201
|
+
def _is_array(value) -> bool:
|
|
202
|
+
"""Whether `value` is a dataset: an array of numbers or booleans with dimensions.
|
|
203
|
+
|
|
204
|
+
A masked array is not, as its mask would be lost; the JSON writer refuses it.
|
|
205
|
+
"""
|
|
206
|
+
return (isinstance(value, np.ndarray) and value.ndim > 0 and value.dtype.kind in "biufc"
|
|
207
|
+
and not isinstance(value, (np.ma.MaskedArray, Masked)))
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _holds_array(value) -> bool:
|
|
211
|
+
if _is_array(value) or isinstance(value, NDCube):
|
|
212
|
+
return True
|
|
213
|
+
if isinstance(value, dict):
|
|
214
|
+
return any(_holds_array(v) for v in value.values())
|
|
215
|
+
if isinstance(value, (list, tuple)):
|
|
216
|
+
return any(_holds_array(v) for v in value)
|
|
217
|
+
return False
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _warn(message: str) -> None:
|
|
221
|
+
"""Warn with `message`, attributed to the first caller outside this module."""
|
|
222
|
+
frame, level = sys._getframe(1), 2
|
|
223
|
+
while frame is not None and frame.f_globals.get("__name__") == __name__:
|
|
224
|
+
frame, level = frame.f_back, level + 1
|
|
225
|
+
warnings.warn(message, UserWarning, stacklevel=level)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _unit_text(unit) -> str:
|
|
229
|
+
"""Text that reads back as `unit`; TypeError if there is none."""
|
|
230
|
+
if not isinstance(unit, u.UnitBase):
|
|
231
|
+
raise TypeError(f"The unit {unit} cannot be written to a results file, as it is not a "
|
|
232
|
+
f"plain unit.")
|
|
233
|
+
return _checked_unit_text(unit)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@functools.lru_cache(maxsize=None)
|
|
237
|
+
def _checked_unit_text(unit: u.UnitBase) -> str:
|
|
238
|
+
# Write the whole scale, since astropy's text gives it to six digits.
|
|
239
|
+
text = unit.to_string()
|
|
240
|
+
if unit.scale != 1:
|
|
241
|
+
text = (f"{float(unit.scale)!r} "
|
|
242
|
+
f"{u.CompositeUnit(1, unit.bases, unit.powers).to_string()}").strip()
|
|
243
|
+
try:
|
|
244
|
+
same = u.Unit(text) == unit
|
|
245
|
+
except ValueError:
|
|
246
|
+
same = False
|
|
247
|
+
# A unit defined only in a script, for example, is unknown to a reader.
|
|
248
|
+
if not same:
|
|
249
|
+
raise TypeError(f"The unit {unit} cannot be written to a results file, as it would not "
|
|
250
|
+
f"read back as itself.")
|
|
251
|
+
return text
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _package_parts(text: str):
|
|
255
|
+
"""Parts of `text`, a path relative to the package, or None if it could lead outside it."""
|
|
256
|
+
parts = PurePosixPath(text).parts
|
|
257
|
+
if parts and all(part != ".." and re.fullmatch(r"[\w.-]+", part) for part in parts):
|
|
258
|
+
return parts
|
|
259
|
+
return None
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _jsonable(value):
|
|
263
|
+
"""`value` in a form json can write, with kinds JSON lacks tagged."""
|
|
264
|
+
if value is None or isinstance(value, (bool, int, str)):
|
|
265
|
+
return value
|
|
266
|
+
if isinstance(value, np.generic):
|
|
267
|
+
item = value.item()
|
|
268
|
+
# A longdouble, for example, has no Python equivalent, and a NumPy
|
|
269
|
+
# date or time has none that keeps its units.
|
|
270
|
+
if isinstance(item, np.generic) or value.dtype.kind not in "biufS":
|
|
271
|
+
raise TypeError(f"Values of type {type(value).__name__} cannot be written to a "
|
|
272
|
+
f"results file.")
|
|
273
|
+
if value.dtype.kind == "S":
|
|
274
|
+
return _jsonable(item)
|
|
275
|
+
# Keep the type, as for a key of a cube's sampling or a setting given
|
|
276
|
+
# in single precision.
|
|
277
|
+
return {TAG: "numpy", "dtype": value.dtype.str, "value": _jsonable(item)}
|
|
278
|
+
if isinstance(value, float):
|
|
279
|
+
return value if math.isfinite(value) else {TAG: "float", "value": repr(value)}
|
|
280
|
+
if isinstance(value, (np.ma.MaskedArray, Masked)):
|
|
281
|
+
raise TypeError("A masked array cannot be written to a results file: its mask would be "
|
|
282
|
+
"lost.")
|
|
283
|
+
if isinstance(value, u.Quantity):
|
|
284
|
+
return {TAG: "quantity", "value": _jsonable(value.value), "unit": _unit_text(value.unit)}
|
|
285
|
+
if isinstance(value, np.ndarray):
|
|
286
|
+
if value.dtype.kind not in "biufU":
|
|
287
|
+
raise TypeError(f"An array of {value.dtype} cannot be written to a results file.")
|
|
288
|
+
return {TAG: "array", "dtype": value.dtype.str, "shape": list(value.shape),
|
|
289
|
+
"value": _jsonable(value.tolist())}
|
|
290
|
+
if isinstance(value, u.UnitBase):
|
|
291
|
+
return {TAG: "unit", "value": _unit_text(value)}
|
|
292
|
+
if isinstance(value, WCS):
|
|
293
|
+
return {TAG: "wcs", **_wcs_to_tree(value)}
|
|
294
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
295
|
+
registry = _dataclass_registry()
|
|
296
|
+
if registry.get(type(value).__name__) is not type(value):
|
|
297
|
+
raise TypeError(f"A {type(value).__name__} cannot be written to a results file: "
|
|
298
|
+
f"only {', '.join(sorted(registry))} are rebuilt when it is read.")
|
|
299
|
+
# Also keep the fields the object derives, such as a detector's dark
|
|
300
|
+
# current from its temperature, with the values the run used.
|
|
301
|
+
derived = {field.name: getattr(value, field.name) for field in dataclasses.fields(value)
|
|
302
|
+
if not field.init and hasattr(value, field.name)}
|
|
303
|
+
return {TAG: "dataclass", "class": type(value).__name__,
|
|
304
|
+
"fields": {name: _jsonable(field) for name, field in _init_fields(value).items()},
|
|
305
|
+
"derived": {name: _jsonable(field) for name, field in derived.items()}}
|
|
306
|
+
if isinstance(value, type) and issubclass(value, np.generic):
|
|
307
|
+
try:
|
|
308
|
+
name = _plain_dtype(value).name
|
|
309
|
+
except (TypeError, ValueError):
|
|
310
|
+
raise TypeError(f"The type {value.__name__} cannot be written to a results "
|
|
311
|
+
f"file.") from None
|
|
312
|
+
return {TAG: "numpy_type", "value": name}
|
|
313
|
+
# YAML reads !!set and !!binary as these, so a configuration can hold them.
|
|
314
|
+
if isinstance(value, (set, frozenset)):
|
|
315
|
+
items = [_jsonable(v) for v in value]
|
|
316
|
+
try:
|
|
317
|
+
items = sorted(items)
|
|
318
|
+
except TypeError:
|
|
319
|
+
pass
|
|
320
|
+
return {TAG: "frozenset" if isinstance(value, frozenset) else "set", "items": items}
|
|
321
|
+
if isinstance(value, bytes):
|
|
322
|
+
return {TAG: "bytes", "value": base64.b64encode(value).decode("ascii")}
|
|
323
|
+
if isinstance(value, datetime.datetime):
|
|
324
|
+
return {TAG: "datetime", "value": value.isoformat()}
|
|
325
|
+
if isinstance(value, datetime.date):
|
|
326
|
+
return {TAG: "date", "value": value.isoformat()}
|
|
327
|
+
# Paths, and anything else with a place on the filesystem, such as the
|
|
328
|
+
# throughput tables found through importlib.resources, which need not be
|
|
329
|
+
# a pathlib.Path. A path inside the package is written relative to it, so
|
|
330
|
+
# the reader finds its own copy; any other is written as it is.
|
|
331
|
+
if isinstance(value, Path) or hasattr(value, "__fspath__"):
|
|
332
|
+
try:
|
|
333
|
+
relative = Path(os.fspath(value)).relative_to(_package_root()).as_posix()
|
|
334
|
+
except ValueError:
|
|
335
|
+
relative = None
|
|
336
|
+
if relative is not None and _package_parts(relative):
|
|
337
|
+
return {TAG: "resource", "value": relative}
|
|
338
|
+
return {TAG: "path", "value": os.fspath(value)}
|
|
339
|
+
if isinstance(value, tuple):
|
|
340
|
+
return {TAG: "tuple", "items": [_jsonable(v) for v in value]}
|
|
341
|
+
if isinstance(value, list):
|
|
342
|
+
return [_jsonable(v) for v in value]
|
|
343
|
+
if isinstance(value, dict):
|
|
344
|
+
if all(isinstance(k, str) for k in value) and TAG not in value:
|
|
345
|
+
return {k: _jsonable(v) for k, v in value.items()}
|
|
346
|
+
return {TAG: "map", "items": [[_jsonable(k), _jsonable(v)] for k, v in value.items()]}
|
|
347
|
+
raise TypeError(f"Values of type {type(value).__name__} cannot be written to a results file.")
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
# The value each setting had in runs made before it existed. An object or
|
|
351
|
+
# file from then that lacks one gets this value, with a warning only where it
|
|
352
|
+
# differs from today's default. Fits were unweighted before
|
|
353
|
+
# FitConfig.weighted, and None is the detector material's value.
|
|
354
|
+
_MATERIAL = {"fano_factor": None, "pair_energy_300k": None, "pair_energy_slope": None}
|
|
355
|
+
_EARLIER_VALUES = {
|
|
356
|
+
"FitConfig": {"weighted": False},
|
|
357
|
+
"Telescope_EUVST": {"resolution_budget": "none"},
|
|
358
|
+
"Detector_SWC": {**_MATERIAL,
|
|
359
|
+
"dispersion": (198.885944, 1.2516577, -1.361114e-4),
|
|
360
|
+
"wavelength_offset": 0.0 * u.Angstrom,
|
|
361
|
+
"lit_band": (170.0 * u.Angstrom, 212.3 * u.Angstrom),
|
|
362
|
+
"slit_image_tilt": (0.0, 0.0)},
|
|
363
|
+
"Detector_EIS": dict(_MATERIAL),
|
|
364
|
+
"AluminiumFilter": {"visible_decade_thickness": 170.0 * u.angstrom},
|
|
365
|
+
"Simulation": {"pinhole_visible_wavelength": 600.0 * u.nm, "readout_dark_current": False,
|
|
366
|
+
"focal_plane_dispersion": False},
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _is_default(cls, name: str, value) -> bool:
|
|
371
|
+
"""Whether `value` is the default of `cls`'s setting `name`."""
|
|
372
|
+
field = next(field for field in dataclasses.fields(cls) if field.name == name)
|
|
373
|
+
if field.default is not dataclasses.MISSING:
|
|
374
|
+
default = field.default
|
|
375
|
+
elif field.default_factory is not dataclasses.MISSING:
|
|
376
|
+
default = field.default_factory()
|
|
377
|
+
else:
|
|
378
|
+
return False
|
|
379
|
+
if value is None or default is None:
|
|
380
|
+
return value is None and default is None
|
|
381
|
+
if isinstance(value, str) or isinstance(default, str):
|
|
382
|
+
return value == default
|
|
383
|
+
try:
|
|
384
|
+
value, default = u.Quantity(value), u.Quantity(default)
|
|
385
|
+
except (TypeError, ValueError):
|
|
386
|
+
return False
|
|
387
|
+
return value.shape == default.shape and bool(np.all(value == default))
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _init_fields(value) -> dict:
|
|
391
|
+
"""Arguments a configuration object was made with, as far as it has them.
|
|
392
|
+
|
|
393
|
+
Fields derived in __post_init__, such as a detector's dark current from
|
|
394
|
+
its temperature, are written separately as `derived` and restored with
|
|
395
|
+
the run's values. An object from an older version's pickle may lack a
|
|
396
|
+
field added since. It then gets today's default, or, for fields in
|
|
397
|
+
`_EARLIER_VALUES` such as FitConfig's `weighted`, the value runs had
|
|
398
|
+
before the field existed. An object from another version may have a
|
|
399
|
+
setting this version lacks. It is written too, so that reading the file
|
|
400
|
+
reports that it is left out.
|
|
401
|
+
"""
|
|
402
|
+
fields, missing, earlier = {}, [], []
|
|
403
|
+
# Use what the instance itself holds, so a missing field is not silently
|
|
404
|
+
# taken from the class default.
|
|
405
|
+
own = getattr(value, "__dict__", None)
|
|
406
|
+
before = _EARLIER_VALUES.get(type(value).__name__, {})
|
|
407
|
+
for field in dataclasses.fields(value):
|
|
408
|
+
if not field.init:
|
|
409
|
+
continue
|
|
410
|
+
if own is None:
|
|
411
|
+
fields[field.name] = getattr(value, field.name)
|
|
412
|
+
elif field.name in own:
|
|
413
|
+
fields[field.name] = own[field.name]
|
|
414
|
+
elif field.name in before:
|
|
415
|
+
fields[field.name] = before[field.name]
|
|
416
|
+
earlier.append(field.name)
|
|
417
|
+
else:
|
|
418
|
+
missing.append(field.name)
|
|
419
|
+
for name in earlier:
|
|
420
|
+
if _is_default(type(value), name, before[name]):
|
|
421
|
+
continue
|
|
422
|
+
_warn(f"This {type(value).__name__} has no {name}, which ECLIPSE added after it "
|
|
423
|
+
f"was made; it gets {before[name]!r}, as runs then had.")
|
|
424
|
+
if own is not None:
|
|
425
|
+
names = {field.name for field in dataclasses.fields(value)}
|
|
426
|
+
fields.update({key: item for key, item in own.items() if key not in names
|
|
427
|
+
and not key.startswith("_") and not hasattr(type(value), key)})
|
|
428
|
+
if missing:
|
|
429
|
+
_warn(f"This {type(value).__name__} has no {', '.join(missing)}, which ECLIPSE added "
|
|
430
|
+
f"after it was made; it gets today's default.")
|
|
431
|
+
return fields
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _unjson(value, reading: _Reading):
|
|
435
|
+
"""Rebuild what :func:`_jsonable` wrote; `reading` is the state of the file read."""
|
|
436
|
+
if isinstance(value, list):
|
|
437
|
+
return [_unjson(v, reading) for v in value]
|
|
438
|
+
if not isinstance(value, dict):
|
|
439
|
+
return value
|
|
440
|
+
tag = value.get(TAG)
|
|
441
|
+
if tag is None:
|
|
442
|
+
return {k: _unjson(v, reading) for k, v in value.items()}
|
|
443
|
+
if tag == "quantity":
|
|
444
|
+
return u.Quantity(_unjson(value["value"], reading), u.Unit(value["unit"]), dtype=None)
|
|
445
|
+
if tag == "numpy":
|
|
446
|
+
return _plain_dtype(value["dtype"]).type(_unjson(value["value"], reading))
|
|
447
|
+
if tag == "float":
|
|
448
|
+
return float(value["value"])
|
|
449
|
+
if tag == "array":
|
|
450
|
+
dtype = _plain_dtype(value["dtype"])
|
|
451
|
+
items = np.array(_unjson(value["value"], reading),
|
|
452
|
+
dtype=object if dtype.kind == "U" else dtype)
|
|
453
|
+
# Text items must fit the width the file declares.
|
|
454
|
+
if dtype.kind == "U" and not all(isinstance(item, str) and len(item) <= dtype.itemsize // 4
|
|
455
|
+
for item in items.flat):
|
|
456
|
+
raise ValueError("A text array in the results file holds more than text of its width.")
|
|
457
|
+
# Use the shape as written, which an empty array's items do not give.
|
|
458
|
+
shape = value["shape"]
|
|
459
|
+
if (not all(isinstance(size, int) and size >= 0 for size in shape)
|
|
460
|
+
or math.prod(shape) != items.size):
|
|
461
|
+
raise ValueError("An array in the results file does not have the shape it gives.")
|
|
462
|
+
return items.astype(dtype, copy=False).reshape(shape)
|
|
463
|
+
if tag == "unit":
|
|
464
|
+
return u.Unit(value["value"])
|
|
465
|
+
if tag == "wcs":
|
|
466
|
+
return _wcs_from_tree(value)
|
|
467
|
+
if tag == "dataclass":
|
|
468
|
+
registry = _dataclass_registry()
|
|
469
|
+
name = value["class"]
|
|
470
|
+
if name not in registry:
|
|
471
|
+
raise ValueError(f"The results file names a class ECLIPSE will not construct: "
|
|
472
|
+
f"{name!r}. Only {', '.join(sorted(registry))} are rebuilt, so "
|
|
473
|
+
f"that reading a file cannot construct anything the file chooses.")
|
|
474
|
+
return _rebuild(registry[name],
|
|
475
|
+
{k: _unjson(v, reading) for k, v in value["fields"].items()},
|
|
476
|
+
{k: _unjson(v, reading) for k, v in value["derived"].items()},
|
|
477
|
+
reading)
|
|
478
|
+
if tag == "numpy_type":
|
|
479
|
+
return _plain_dtype(value["value"]).type
|
|
480
|
+
if tag in ("set", "frozenset"):
|
|
481
|
+
return (set if tag == "set" else frozenset)(_unjson(v, reading) for v in value["items"])
|
|
482
|
+
if tag == "bytes":
|
|
483
|
+
return base64.b64decode(value["value"])
|
|
484
|
+
if tag == "datetime":
|
|
485
|
+
return datetime.datetime.fromisoformat(value["value"])
|
|
486
|
+
if tag == "date":
|
|
487
|
+
return datetime.date.fromisoformat(value["value"])
|
|
488
|
+
if tag == "path":
|
|
489
|
+
return Path(value["value"])
|
|
490
|
+
if tag == "resource":
|
|
491
|
+
# Check the path as written, before touching the filesystem, so a
|
|
492
|
+
# file cannot make the reader look outside the package at all.
|
|
493
|
+
parts = _package_parts(value["value"])
|
|
494
|
+
if parts is None:
|
|
495
|
+
raise ValueError(f"The results file names a package file, {value['value']!r}, "
|
|
496
|
+
f"outside the package.")
|
|
497
|
+
return _package_root().joinpath(*parts)
|
|
498
|
+
if tag == "tuple":
|
|
499
|
+
return tuple(_unjson(v, reading) for v in value["items"])
|
|
500
|
+
if tag == "map":
|
|
501
|
+
return {_unjson(k, reading): _unjson(v, reading) for k, v in value["items"]}
|
|
502
|
+
raise ValueError(f"The results file holds a value tagged {tag!r}, which this ECLIPSE "
|
|
503
|
+
f"does not know.")
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _plain_dtype(name) -> np.dtype:
|
|
507
|
+
"""Dtype `name`, which must be plain numbers, booleans or text, as ECLIPSE writes.
|
|
508
|
+
|
|
509
|
+
A dtype can also give each element its own shape, which would let a few
|
|
510
|
+
bytes of a crafted file fill any amount of memory.
|
|
511
|
+
"""
|
|
512
|
+
dtype = np.dtype(name)
|
|
513
|
+
if dtype.kind not in "biufcU" or dtype.subdtype is not None or dtype.names is not None:
|
|
514
|
+
raise ValueError(f"The results file holds values of type {name!r}, which a results "
|
|
515
|
+
f"file does not.")
|
|
516
|
+
return dtype
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _rebuild(cls, stored: dict, derived: dict, reading: _Reading):
|
|
520
|
+
"""
|
|
521
|
+
Configuration object rebuilt from what a file stored.
|
|
522
|
+
|
|
523
|
+
The object is constructed again, so it is checked and computes its
|
|
524
|
+
derived fields, which are then set to the values the run had. If this
|
|
525
|
+
version would not make the object, because a setting has gone or a
|
|
526
|
+
check has been added since, the file still reads: the object is rebuilt
|
|
527
|
+
as stored, unchecked, with a warning given once `reading` is done.
|
|
528
|
+
"""
|
|
529
|
+
known = {field.name for field in dataclasses.fields(cls) if field.init}
|
|
530
|
+
# A setting stored under an old name is read under its new one.
|
|
531
|
+
stored = {getattr(cls, "_OLD_NAMES", {}).get(key, key): item for key, item in stored.items()}
|
|
532
|
+
# A setting in _EARLIER_VALUES that the file lacks gets the value runs
|
|
533
|
+
# had before it existed.
|
|
534
|
+
before = {name: item for name, item in _EARLIER_VALUES.get(cls.__name__, {}).items()
|
|
535
|
+
if name not in stored}
|
|
536
|
+
for name, item in before.items():
|
|
537
|
+
if _is_default(cls, name, item):
|
|
538
|
+
continue
|
|
539
|
+
reading.notes[f"The {cls.__name__} in the results file has no {name}, which ECLIPSE "
|
|
540
|
+
f"added after it was made; it gets {item!r}, as runs then had."] = None
|
|
541
|
+
stored = {**stored, **before}
|
|
542
|
+
# A setting added since the file was made gets today's default, with a
|
|
543
|
+
# warning, as the run did not have it. One with no default makes the
|
|
544
|
+
# construction below fail, and is reported there.
|
|
545
|
+
missing = sorted(field.name for field in dataclasses.fields(cls)
|
|
546
|
+
if field.init and field.name not in stored
|
|
547
|
+
and (field.default is not dataclasses.MISSING
|
|
548
|
+
or field.default_factory is not dataclasses.MISSING))
|
|
549
|
+
if missing:
|
|
550
|
+
reading.notes[f"The {cls.__name__} in the results file has no {', '.join(missing)}, "
|
|
551
|
+
f"which ECLIPSE added after it was made; "
|
|
552
|
+
f"{'it gets' if len(missing) == 1 else 'they get'} today's "
|
|
553
|
+
f"default{'' if len(missing) == 1 else 's'}."] = None
|
|
554
|
+
gone = sorted(set(stored) - known)
|
|
555
|
+
if gone:
|
|
556
|
+
reading.notes[f"The {cls.__name__} in the results file has {', '.join(gone)}, which "
|
|
557
|
+
f"this version of ECLIPSE does not have, so "
|
|
558
|
+
f"{'it is' if len(gone) == 1 else 'they are'} left out."] = None
|
|
559
|
+
arguments = {key: item for key, item in stored.items() if key in known}
|
|
560
|
+
try:
|
|
561
|
+
obj = cls(**arguments)
|
|
562
|
+
except (TypeError, ValueError, AttributeError) as error:
|
|
563
|
+
reading.notes[f"The {cls.__name__} in the results file is not one this version of "
|
|
564
|
+
f"ECLIPSE would make ({error}), so it is rebuilt as it was stored, "
|
|
565
|
+
f"unchecked."] = None
|
|
566
|
+
obj = cls.__new__(cls)
|
|
567
|
+
for field in dataclasses.fields(cls):
|
|
568
|
+
if field.name in arguments:
|
|
569
|
+
object.__setattr__(obj, field.name, arguments[field.name])
|
|
570
|
+
elif field.default is not dataclasses.MISSING:
|
|
571
|
+
object.__setattr__(obj, field.name, field.default)
|
|
572
|
+
elif field.default_factory is not dataclasses.MISSING:
|
|
573
|
+
object.__setattr__(obj, field.name, field.default_factory())
|
|
574
|
+
# Set only the derived fields, so a file cannot set anything else past
|
|
575
|
+
# the checks.
|
|
576
|
+
worked_out = {field.name for field in dataclasses.fields(cls) if not field.init}
|
|
577
|
+
for key, item in derived.items():
|
|
578
|
+
if key in worked_out:
|
|
579
|
+
object.__setattr__(obj, key, item)
|
|
580
|
+
return obj
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _to_json(value) -> str:
|
|
584
|
+
# Strict JSON, which any language reads; non-finite numbers are tagged.
|
|
585
|
+
return json.dumps(_jsonable(value), allow_nan=False)
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
# ----------------------------------------------------------------------
|
|
589
|
+
# The tree as groups
|
|
590
|
+
# ----------------------------------------------------------------------
|
|
591
|
+
def _is_name(key) -> bool:
|
|
592
|
+
"""Whether `key` can name a group member, as a dataset or an attribute."""
|
|
593
|
+
if (not isinstance(key, str) or key in ("", ".") or key in _RESERVED
|
|
594
|
+
or "/" in key or "\0" in key):
|
|
595
|
+
return False
|
|
596
|
+
# HDF5 names are UTF-8, which cannot encode a lone surrogate, for example.
|
|
597
|
+
try:
|
|
598
|
+
key.encode("utf-8")
|
|
599
|
+
except UnicodeEncodeError:
|
|
600
|
+
return False
|
|
601
|
+
return True
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
class _Unwritable(TypeError):
|
|
605
|
+
"""A value a results file cannot hold, and its path in the results."""
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _put(group: h5py.Group, name: str, value, compression, written: dict) -> None:
|
|
609
|
+
"""
|
|
610
|
+
Write `value` as member `name` of `group`.
|
|
611
|
+
|
|
612
|
+
A value holding arrays becomes a dataset or group, and anything else a
|
|
613
|
+
JSON attribute. `written` maps the id of each object written so far as a
|
|
614
|
+
dataset or group to the object and its path, so an object that appears
|
|
615
|
+
again is written as a reference rather than written again.
|
|
616
|
+
"""
|
|
617
|
+
try:
|
|
618
|
+
_put_value(group, name, value, compression, written)
|
|
619
|
+
except _Unwritable:
|
|
620
|
+
raise
|
|
621
|
+
except (TypeError, RecursionError) as error:
|
|
622
|
+
# Give the value's path, which a save failing at the end of a run must report.
|
|
623
|
+
reason = ("A value that holds itself cannot be written to a results file."
|
|
624
|
+
if isinstance(error, RecursionError) else error)
|
|
625
|
+
raise _Unwritable(f"{group.name.rstrip('/')}/{name}: {reason}") from error
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _put_value(group: h5py.Group, name: str, value, compression, written: dict) -> None:
|
|
629
|
+
if not _holds_array(value):
|
|
630
|
+
group.attrs[name] = _to_json(value)
|
|
631
|
+
return
|
|
632
|
+
if id(value) in written:
|
|
633
|
+
reference = group.create_group(name)
|
|
634
|
+
reference.attrs[TYPE] = "ref"
|
|
635
|
+
reference.attrs["target"] = written[id(value)][1]
|
|
636
|
+
return
|
|
637
|
+
if _is_array(value):
|
|
638
|
+
data = value.value if isinstance(value, u.Quantity) else value
|
|
639
|
+
unit = _unit_text(value.unit) if isinstance(value, u.Quantity) else None
|
|
640
|
+
options = {}
|
|
641
|
+
if compression and data.size >= _COMPRESS_FROM:
|
|
642
|
+
options = {"compression": compression, "shuffle": True,
|
|
643
|
+
**({"compression_opts": 1} if compression == "gzip" else {})}
|
|
644
|
+
node = group.create_dataset(name, data=data, **options)
|
|
645
|
+
if unit is not None:
|
|
646
|
+
node.attrs["unit"] = unit
|
|
647
|
+
elif isinstance(value, NDCube):
|
|
648
|
+
left_out = [part for part in ("mask", "uncertainty", "psf")
|
|
649
|
+
if getattr(value, part, None) is not None]
|
|
650
|
+
extra = getattr(value, "extra_coords", None)
|
|
651
|
+
if extra is not None and not getattr(extra, "is_empty", True):
|
|
652
|
+
left_out.append("extra coordinates")
|
|
653
|
+
if len(getattr(value, "global_coords", None) or {}):
|
|
654
|
+
left_out.append("global coordinates")
|
|
655
|
+
if left_out:
|
|
656
|
+
_warn(f"{group.name.rstrip('/')}/{name}: a results file does not hold a cube's "
|
|
657
|
+
f"{' or '.join(left_out)}, so this cube's are left out.")
|
|
658
|
+
if value.unit is not None:
|
|
659
|
+
_unit_text(value.unit)
|
|
660
|
+
node = group.create_group(name, track_order=True)
|
|
661
|
+
node.attrs[TYPE] = "ndcube"
|
|
662
|
+
# Write the data with the cube's unit, as for any other quantity.
|
|
663
|
+
data = np.asarray(value.data)
|
|
664
|
+
if value.unit is not None:
|
|
665
|
+
data = u.Quantity(data, value.unit, copy=False, dtype=None)
|
|
666
|
+
_put(node, "data", data, compression, written)
|
|
667
|
+
_put(node, "wcs", value.wcs, compression, written)
|
|
668
|
+
# The cube's own meta; one holding arrays that another cube shares is
|
|
669
|
+
# written once.
|
|
670
|
+
meta = value.meta if isinstance(value.meta, dict) else dict(value.meta or {})
|
|
671
|
+
_put(node, "meta", meta, compression, written)
|
|
672
|
+
elif isinstance(value, dict):
|
|
673
|
+
node = group.create_group(name, track_order=True)
|
|
674
|
+
_put_dict(node, value, compression, written)
|
|
675
|
+
else:
|
|
676
|
+
node = group.create_group(name, track_order=True)
|
|
677
|
+
node.attrs[TYPE] = "tuple" if isinstance(value, tuple) else "list"
|
|
678
|
+
node.attrs[LENGTH] = len(value)
|
|
679
|
+
for index, item in enumerate(value):
|
|
680
|
+
_put(node, str(index), item, compression, written)
|
|
681
|
+
# Keep the object itself with its path, so no object written meanwhile
|
|
682
|
+
# can reuse its id.
|
|
683
|
+
written[id(value)] = (value, node.name)
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _put_dict(group: h5py.Group, value: dict, compression, written: dict) -> None:
|
|
687
|
+
if all(_is_name(key) for key in value):
|
|
688
|
+
group.attrs[ORDER] = json.dumps(list(value))
|
|
689
|
+
for key, item in value.items():
|
|
690
|
+
_put(group, key, item, compression, written)
|
|
691
|
+
else:
|
|
692
|
+
group.attrs[TYPE] = "map"
|
|
693
|
+
group.attrs[KEYS] = _to_json(list(value))
|
|
694
|
+
for index, item in enumerate(value.values()):
|
|
695
|
+
_put(group, str(index), item, compression, written)
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
@dataclasses.dataclass
|
|
699
|
+
class _Reading:
|
|
700
|
+
"""State of the reading of one file."""
|
|
701
|
+
|
|
702
|
+
size: int # file size in bytes
|
|
703
|
+
stored: int = 0 # bytes stored by the datasets read so far
|
|
704
|
+
seen: set = dataclasses.field(default_factory=set) # objects reached
|
|
705
|
+
inside: set = dataclasses.field(default_factory=set) # groups being read
|
|
706
|
+
done: dict = dataclasses.field(default_factory=dict) # value read at each path
|
|
707
|
+
notes: dict = dataclasses.field(default_factory=dict) # warnings to give once read
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _attribute(node, name: str, path: Path):
|
|
711
|
+
"""Attribute `name` of `node`, or None if absent; one string or integer, as ECLIPSE writes."""
|
|
712
|
+
if name not in node.attrs:
|
|
713
|
+
return None
|
|
714
|
+
# Check before reading: each element of an array attribute, or each field
|
|
715
|
+
# of a compound one, can refer to the same text elsewhere in the file,
|
|
716
|
+
# which reading would copy for each.
|
|
717
|
+
attribute = node.attrs.get_id(name)
|
|
718
|
+
dtype = attribute.dtype
|
|
719
|
+
if (attribute.shape is None or math.prod(attribute.shape) > 1
|
|
720
|
+
or not (h5py.check_string_dtype(dtype) or dtype.kind in "iu")):
|
|
721
|
+
raise ValueError(f"{path}: attribute {name!r} of {node.name} is not one string or "
|
|
722
|
+
f"integer, as a results file writes them.")
|
|
723
|
+
return node.attrs[name]
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def _get(group: h5py.Group, name: str, path: Path, reading: _Reading):
|
|
727
|
+
"""Read member `name` of `group`: a dataset, a group or a JSON attribute."""
|
|
728
|
+
# A name with a slash in it would be looked up through other groups.
|
|
729
|
+
if not _is_name(name):
|
|
730
|
+
raise ValueError(f"{path}: {group.name} has a member named {name!r}, which a results "
|
|
731
|
+
f"file does not.")
|
|
732
|
+
if name in group.attrs:
|
|
733
|
+
return _unjson(json.loads(_attribute(group, name, path)), reading)
|
|
734
|
+
link = group.get(name, getlink=True)
|
|
735
|
+
if link is None:
|
|
736
|
+
raise ValueError(f"{path}: {group.name} has no member {name!r}.")
|
|
737
|
+
# A link, or a dataset whose data lives elsewhere, would make the reader
|
|
738
|
+
# open another file, which a results file never needs.
|
|
739
|
+
if not isinstance(link, h5py.HardLink):
|
|
740
|
+
raise ValueError(f"{path}: {group.name}/{name} is a link, which a results file "
|
|
741
|
+
f"does not hold.")
|
|
742
|
+
return _read(group[name], path, reading)
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def _read(node, path: Path, reading: _Reading):
|
|
746
|
+
"""Read `node`, only once however often the results refer to it."""
|
|
747
|
+
if node.name in reading.done:
|
|
748
|
+
return reading.done[node.name]
|
|
749
|
+
# ECLIPSE writes each object once, so one reached twice other than by a
|
|
750
|
+
# reference is not ECLIPSE's: it would be read twice, or without end if
|
|
751
|
+
# reached from inside itself. Objects are tracked by their number in the
|
|
752
|
+
# file rather than kept open, since an open dataset keeps what it read.
|
|
753
|
+
number = h5py.h5g.get_objinfo(node.id).objno
|
|
754
|
+
if number in reading.seen:
|
|
755
|
+
raise ValueError(f"{path}: {node.name} is reached from more than one place, which a "
|
|
756
|
+
f"results file does not do.")
|
|
757
|
+
reading.seen.add(number)
|
|
758
|
+
if isinstance(node, h5py.Dataset):
|
|
759
|
+
_check_dataset(node, path, reading)
|
|
760
|
+
data = node[()]
|
|
761
|
+
unit = _attribute(node, "unit", path)
|
|
762
|
+
value = data if unit is None else u.Quantity(data, u.Unit(unit), copy=False, dtype=None)
|
|
763
|
+
else:
|
|
764
|
+
reading.inside.add(node.name)
|
|
765
|
+
if _attribute(node, TYPE, path) == "ref":
|
|
766
|
+
value = _follow(node, path, reading)
|
|
767
|
+
else:
|
|
768
|
+
value = _get_group(node, path, reading)
|
|
769
|
+
reading.inside.discard(node.name)
|
|
770
|
+
reading.done[node.name] = value
|
|
771
|
+
return value
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def _follow(reference: h5py.Group, path: Path, reading: _Reading):
|
|
775
|
+
"""Object `reference` refers to, read through hard links within the file."""
|
|
776
|
+
target = _attribute(reference, "target", path)
|
|
777
|
+
if not isinstance(target, str) or not target.startswith("/"):
|
|
778
|
+
raise ValueError(f"{path}: {reference.name} refers to nothing a results file holds.")
|
|
779
|
+
if target in reading.inside:
|
|
780
|
+
raise ValueError(f"{path}: {reference.name} refers to {target}, which it is inside.")
|
|
781
|
+
if target in reading.done:
|
|
782
|
+
return reading.done[target]
|
|
783
|
+
node = reference.file
|
|
784
|
+
for part in target[1:].split("/"):
|
|
785
|
+
if (not isinstance(node, h5py.Group) or not _is_name(part)
|
|
786
|
+
or not isinstance(node.get(part, getlink=True), h5py.HardLink)):
|
|
787
|
+
raise ValueError(f"{path}: {reference.name} refers to {target}, which the file "
|
|
788
|
+
f"does not hold.")
|
|
789
|
+
node = node[part]
|
|
790
|
+
return _read(node, path, reading)
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
# Filters ECLIPSE compresses arrays with; any other would make HDF5 look for
|
|
794
|
+
# a plugin to read them.
|
|
795
|
+
_FILTERS = {h5py.h5z.FILTER_DEFLATE, h5py.h5z.FILTER_SHUFFLE}
|
|
796
|
+
# Most bytes deflate can give back per byte stored: 258 per two-bit code.
|
|
797
|
+
_DEFLATE_MOST = 1032
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _check_dataset(node: h5py.Dataset, path: Path, reading: _Reading) -> None:
|
|
801
|
+
"""
|
|
802
|
+
Refuse, before reading, a dataset that ECLIPSE would not have written.
|
|
803
|
+
|
|
804
|
+
What the file declares of its data is checked against what it stores,
|
|
805
|
+
since a crafted file could otherwise declare more than it holds, which
|
|
806
|
+
reading would make up, or have one stored piece read as many.
|
|
807
|
+
"""
|
|
808
|
+
if node.is_virtual or node.external:
|
|
809
|
+
raise ValueError(f"{path}: {node.name} keeps its data in another file, which a "
|
|
810
|
+
f"results file does not do.")
|
|
811
|
+
dtype = node.dtype
|
|
812
|
+
if dtype.kind not in "biufc" or dtype.subdtype is not None or dtype.names is not None:
|
|
813
|
+
raise ValueError(f"{path}: {node.name} holds values of type {dtype}, which a results "
|
|
814
|
+
f"file does not.")
|
|
815
|
+
plist = node.id.get_create_plist()
|
|
816
|
+
filters = {plist.get_filter(i)[0] for i in range(plist.get_nfilters())}
|
|
817
|
+
if not filters <= _FILTERS:
|
|
818
|
+
raise ValueError(f"{path}: {node.name} is compressed in a way a results file is not.")
|
|
819
|
+
stored, whole = 0, True
|
|
820
|
+
if node.chunks is None:
|
|
821
|
+
stored = node.id.get_storage_size()
|
|
822
|
+
else:
|
|
823
|
+
# Every chunk of the grid must be stored, as an unwritten one would
|
|
824
|
+
# read as made-up values. Count them first, so a grid declared far
|
|
825
|
+
# larger than the file holds is not looped over.
|
|
826
|
+
steps = node.chunks
|
|
827
|
+
grid = [-(-size // step) for size, step in zip(node.shape, steps)]
|
|
828
|
+
whole = node.id.get_num_chunks() == math.prod(grid)
|
|
829
|
+
for index in np.ndindex(*grid) if whole else ():
|
|
830
|
+
chunk = node.id.get_chunk_info_by_coord(tuple(i * step for i, step in zip(index, steps)))
|
|
831
|
+
if chunk.byte_offset is None:
|
|
832
|
+
whole = False
|
|
833
|
+
break
|
|
834
|
+
stored += chunk.size
|
|
835
|
+
most = stored * (_DEFLATE_MOST if h5py.h5z.FILTER_DEFLATE in filters else 1)
|
|
836
|
+
if not whole or node.nbytes > most:
|
|
837
|
+
raise ValueError(f"{path}: {node.name} does not hold all of its data.")
|
|
838
|
+
reading.stored += stored
|
|
839
|
+
if reading.stored > reading.size:
|
|
840
|
+
raise ValueError(f"{path}: {node.name} shares what it stores with another dataset, "
|
|
841
|
+
f"which a results file does not do.")
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _describing(group: h5py.Group, name: str, path: Path):
|
|
845
|
+
"""Attribute `name` describing `group`, which a results file always writes."""
|
|
846
|
+
value = _attribute(group, name, path)
|
|
847
|
+
if value is None:
|
|
848
|
+
raise ValueError(f"{path}: {group.name} has no {name!r} attribute, which a results "
|
|
849
|
+
f"file gives it.")
|
|
850
|
+
return value
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def _get_group(group: h5py.Group, path: Path, reading: _Reading):
|
|
854
|
+
kind = _attribute(group, TYPE, path)
|
|
855
|
+
if kind == "ndcube":
|
|
856
|
+
data = _get(group, "data", path, reading)
|
|
857
|
+
unit = None
|
|
858
|
+
if isinstance(data, u.Quantity):
|
|
859
|
+
data, unit = data.value, data.unit
|
|
860
|
+
return NDCube(data, wcs=_get(group, "wcs", path, reading), unit=unit,
|
|
861
|
+
meta=_get(group, "meta", path, reading))
|
|
862
|
+
if kind == "map":
|
|
863
|
+
keys = _unjson(json.loads(_describing(group, KEYS, path)), reading)
|
|
864
|
+
return {key: _get(group, str(index), path, reading) for index, key in enumerate(keys)}
|
|
865
|
+
if kind in ("list", "tuple"):
|
|
866
|
+
length = int(_describing(group, LENGTH, path))
|
|
867
|
+
items = [_get(group, str(index), path, reading) for index in range(length)]
|
|
868
|
+
return items if kind == "list" else tuple(items)
|
|
869
|
+
if kind is not None:
|
|
870
|
+
raise ValueError(f"{path}: {group.name} is of a kind, {kind!r}, this ECLIPSE does "
|
|
871
|
+
f"not know.")
|
|
872
|
+
order = json.loads(_describing(group, ORDER, path))
|
|
873
|
+
members = (set(group) | set(group.attrs)) - _RESERVED
|
|
874
|
+
if (not isinstance(order, list) or len(set(order)) != len(order)
|
|
875
|
+
or set(order) != members):
|
|
876
|
+
raise ValueError(f"{path}: the members of {group.name} are not those it lists.")
|
|
877
|
+
return {key: _get(group, key, path, reading) for key in order}
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
# ----------------------------------------------------------------------
|
|
881
|
+
# Files
|
|
882
|
+
# ----------------------------------------------------------------------
|
|
883
|
+
def is_results_file(path: str | Path) -> bool:
|
|
884
|
+
"""
|
|
885
|
+
Whether `path` is HDF5, as a results file is, rather than an older pickle.
|
|
886
|
+
|
|
887
|
+
:func:`load_results` checks which kind of HDF5 file it is.
|
|
888
|
+
"""
|
|
889
|
+
return h5py.is_hdf5(os.fspath(Path(path).expanduser()))
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def save_results(path: str | Path, payload: dict, *, compression: str | None = "gzip") -> Path:
|
|
893
|
+
"""
|
|
894
|
+
Write `payload` as a results file.
|
|
895
|
+
|
|
896
|
+
Parameters
|
|
897
|
+
----------
|
|
898
|
+
path : str or Path
|
|
899
|
+
Where to write. An existing file is replaced only once the new one
|
|
900
|
+
is complete, so a failed write leaves it unchanged. A pickle suffix
|
|
901
|
+
such as `.pkl` is replaced with `.h5`, since the file is not a
|
|
902
|
+
pickle.
|
|
903
|
+
payload : dict
|
|
904
|
+
Tree to write, keyed by strings that can name an HDF5 member.
|
|
905
|
+
compression : {"gzip", None}, optional
|
|
906
|
+
Compress the larger arrays with gzip, which any HDF5 reader can
|
|
907
|
+
undo; None writes them uncompressed, which is faster for a large run.
|
|
908
|
+
|
|
909
|
+
Returns
|
|
910
|
+
-------
|
|
911
|
+
Path
|
|
912
|
+
File written.
|
|
913
|
+
"""
|
|
914
|
+
path = Path(path).expanduser()
|
|
915
|
+
if path.suffix.lower() in _PICKLE_SUFFIXES:
|
|
916
|
+
path = path.with_suffix(".h5")
|
|
917
|
+
warnings.warn(f"The results were written to {path.name}, an HDF5 file, rather than "
|
|
918
|
+
f"a pickle's name that would misdescribe it.", UserWarning, stacklevel=2)
|
|
919
|
+
if path.is_dir():
|
|
920
|
+
raise IsADirectoryError(f"{path} is a directory; name the results file to write.")
|
|
921
|
+
if compression not in ("gzip", None):
|
|
922
|
+
raise ValueError(f"compression must be 'gzip' or None, got {compression!r}.")
|
|
923
|
+
if not isinstance(payload, Mapping):
|
|
924
|
+
raise TypeError(f"The results must be a mapping, got {type(payload).__name__}.")
|
|
925
|
+
bad = [key for key in payload if not _is_name(key)]
|
|
926
|
+
if bad:
|
|
927
|
+
raise ValueError(f"The results cannot be keyed by {bad}: the keys have to name "
|
|
928
|
+
f"HDF5 members, and {sorted(_RESERVED)} are taken.")
|
|
929
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
930
|
+
# Write through a symlink to its target, as writing to the link would.
|
|
931
|
+
destination = Path(os.path.realpath(path)) if path.is_symlink() else path
|
|
932
|
+
# Do not replace a read-only file, as writing into it would fail.
|
|
933
|
+
if destination.exists() and not os.access(destination, os.W_OK):
|
|
934
|
+
raise PermissionError(f"{destination} is read-only, so the results are not written "
|
|
935
|
+
f"over it.")
|
|
936
|
+
partial = _new_partial(destination)
|
|
937
|
+
try:
|
|
938
|
+
with h5py.File(partial, "w", track_order=True) as f:
|
|
939
|
+
f.attrs["format"] = FORMAT_NAME
|
|
940
|
+
f.attrs["version"] = FORMAT_VERSION
|
|
941
|
+
_put_dict(f, dict(payload), compression, {})
|
|
942
|
+
os.replace(partial, destination)
|
|
943
|
+
except BaseException:
|
|
944
|
+
partial.unlink(missing_ok=True)
|
|
945
|
+
raise
|
|
946
|
+
return path
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
# HDF5 file suffixes. A pickle with one is not read, since a reader would
|
|
950
|
+
# take such a file to be safe to open.
|
|
951
|
+
_HDF5_SUFFIXES = (".h5", ".hdf5", ".hdf", ".he5")
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def load_results(path: str | Path, *, _stacklevel: int = 2) -> dict:
|
|
955
|
+
"""
|
|
956
|
+
Read a results file, or a results pickle written by an older version.
|
|
957
|
+
|
|
958
|
+
A pickle still loads, with a warning, which also mentions a later run's
|
|
959
|
+
`.h5` beside it. A pickle named as an HDF5 file is refused, since a
|
|
960
|
+
reader would take it to be safe to open, and unpickling runs whatever
|
|
961
|
+
code the file holds. A `.pkl` path that does not exist, as a script
|
|
962
|
+
written for an older version may ask for, reads the `.h5` the
|
|
963
|
+
simulation writes in its place, also with a warning.
|
|
964
|
+
|
|
965
|
+
Parameters
|
|
966
|
+
----------
|
|
967
|
+
path : str or Path
|
|
968
|
+
File to read.
|
|
969
|
+
|
|
970
|
+
Returns
|
|
971
|
+
-------
|
|
972
|
+
dict
|
|
973
|
+
Stored payload.
|
|
974
|
+
"""
|
|
975
|
+
path = Path(path).expanduser()
|
|
976
|
+
# The .h5 the simulation writes in place of a pickle of this name.
|
|
977
|
+
written = path.with_suffix(".h5") if path.suffix.lower() in _PICKLE_SUFFIXES else None
|
|
978
|
+
if written is not None and not path.exists() and written.is_file():
|
|
979
|
+
warnings.warn(f"{path} does not exist, so {written}, which the instrument simulation "
|
|
980
|
+
f"now writes in its place, is read instead. Name the .h5 file to read it "
|
|
981
|
+
f"without this warning.", FutureWarning, stacklevel=_stacklevel)
|
|
982
|
+
path, written = written, None
|
|
983
|
+
|
|
984
|
+
if path.is_dir():
|
|
985
|
+
raise IsADirectoryError(f"{path} is a directory; name the results file to read.")
|
|
986
|
+
if not path.is_file():
|
|
987
|
+
if path.suffix.lower() in _HDF5_SUFFIXES:
|
|
988
|
+
for older in (path.with_suffix(suffix) for suffix in _PICKLE_SUFFIXES):
|
|
989
|
+
if older.is_file():
|
|
990
|
+
raise FileNotFoundError(
|
|
991
|
+
f"No results file {path}. {older}, beside it, is a results pickle from "
|
|
992
|
+
f"an older version of ECLIPSE: load_instrument_response_results reads "
|
|
993
|
+
f"it, and eclipse.convert_results_pickle rewrites it as a "
|
|
994
|
+
f"results file.")
|
|
995
|
+
raise FileNotFoundError(f"No results file {path}.")
|
|
996
|
+
if not is_results_file(path):
|
|
997
|
+
if not _is_pickle(path):
|
|
998
|
+
raise ValueError(f"{path} is neither a results file nor a results pickle.")
|
|
999
|
+
if path.suffix.lower() in _HDF5_SUFFIXES:
|
|
1000
|
+
raise ValueError(f"{path} is named as an HDF5 file but is a pickle, which is not "
|
|
1001
|
+
f"read under such a name, since unpickling runs whatever code the "
|
|
1002
|
+
f"file holds: rename it to .pkl if it is a results pickle you "
|
|
1003
|
+
f"trust.")
|
|
1004
|
+
later = ""
|
|
1005
|
+
if (written is not None and written.is_file()
|
|
1006
|
+
and written.stat().st_mtime > path.stat().st_mtime):
|
|
1007
|
+
later = (f" {written}, beside it, is newer: the results of a later run, as the "
|
|
1008
|
+
f"simulation now writes them.")
|
|
1009
|
+
warnings.warn(
|
|
1010
|
+
f"{path} is a results pickle, as older versions of ECLIPSE wrote them. Pickles "
|
|
1011
|
+
f"are deprecated and will not be read in a future release: convert it with "
|
|
1012
|
+
f"eclipse.convert_results_pickle, or re-run the simulation. Reading a "
|
|
1013
|
+
f"pickle runs whatever code it holds, so only read files you trust.{later}",
|
|
1014
|
+
FutureWarning, stacklevel=_stacklevel)
|
|
1015
|
+
payload = _load_pickle(path)
|
|
1016
|
+
_give_earlier_values(payload)
|
|
1017
|
+
_move_old_tables(payload)
|
|
1018
|
+
if _from_before_weighting(payload):
|
|
1019
|
+
warnings.warn(_BEFORE_WEIGHTING.format(path=path), UserWarning,
|
|
1020
|
+
stacklevel=_stacklevel)
|
|
1021
|
+
return payload
|
|
1022
|
+
|
|
1023
|
+
# A damaged or hand-made file can fail anywhere in the reading. Errors are
|
|
1024
|
+
# raised again naming the file, as an OSError if the filesystem fails,
|
|
1025
|
+
# except a MemoryError.
|
|
1026
|
+
reading = _Reading(size=path.stat().st_size)
|
|
1027
|
+
try:
|
|
1028
|
+
with h5py.File(path, "r") as f:
|
|
1029
|
+
# Check these like any other attribute before the format check reads them.
|
|
1030
|
+
for name in ("format", "version"):
|
|
1031
|
+
_attribute(f, name, path)
|
|
1032
|
+
_check_format(f, path, kind="results", format_name=FORMAT_NAME,
|
|
1033
|
+
format_version=FORMAT_VERSION, documented=False)
|
|
1034
|
+
results = _get_group(f, path, reading)
|
|
1035
|
+
except MemoryError:
|
|
1036
|
+
raise
|
|
1037
|
+
except OSError as error:
|
|
1038
|
+
if str(path) in str(error):
|
|
1039
|
+
raise
|
|
1040
|
+
raise OSError(f"{path} could not be read: {error}") from error
|
|
1041
|
+
except ValueError as error:
|
|
1042
|
+
if str(error).startswith(str(path)):
|
|
1043
|
+
raise
|
|
1044
|
+
raise ValueError(f"{path} is not a results file this ECLIPSE can read: "
|
|
1045
|
+
f"{error}") from error
|
|
1046
|
+
except Exception as error:
|
|
1047
|
+
raise ValueError(f"{path} is not a results file this ECLIPSE can read: "
|
|
1048
|
+
f"{type(error).__name__}: {error}") from error
|
|
1049
|
+
# Give each warning from the reading once, as the caller's, such as for a
|
|
1050
|
+
# configuration object this version would not make.
|
|
1051
|
+
for message in reading.notes:
|
|
1052
|
+
warnings.warn(message, UserWarning, stacklevel=_stacklevel)
|
|
1053
|
+
if not isinstance(results, dict):
|
|
1054
|
+
raise ValueError(f"{path} is not a results file this ECLIPSE can read: it holds a "
|
|
1055
|
+
f"{type(results).__name__}, not a mapping.")
|
|
1056
|
+
if _from_before_weighting(results):
|
|
1057
|
+
warnings.warn(_BEFORE_WEIGHTING.format(path=path), UserWarning, stacklevel=_stacklevel)
|
|
1058
|
+
return results
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
_BEFORE_WEIGHTING = ("The results in {path} were made before ECLIPSE weighted its fits by "
|
|
1062
|
+
"each pixel's uncertainty, so their fits are unweighted; fit_weighted is "
|
|
1063
|
+
"set to False.")
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
def _from_before_weighting(payload) -> bool:
|
|
1067
|
+
"""
|
|
1068
|
+
Whether the results were made before fits could be weighted.
|
|
1069
|
+
|
|
1070
|
+
If so, their fits were unweighted, so `fit_weighted` is set to False. So
|
|
1071
|
+
is `weighted` of a pickled FitConfig, which lacks it and would otherwise
|
|
1072
|
+
read the class default.
|
|
1073
|
+
"""
|
|
1074
|
+
results = payload.get("results") if isinstance(payload, dict) else None
|
|
1075
|
+
if not isinstance(results, dict) or "all_combinations" not in results:
|
|
1076
|
+
return False
|
|
1077
|
+
if "fit_weighted" in results:
|
|
1078
|
+
return False
|
|
1079
|
+
results["fit_weighted"] = False
|
|
1080
|
+
own = getattr(results.get("fit_config"), "__dict__", None)
|
|
1081
|
+
if own is not None and "weighted" not in own:
|
|
1082
|
+
object.__setattr__(results["fit_config"], "weighted", False)
|
|
1083
|
+
return True
|
|
1084
|
+
|
|
1085
|
+
|
|
1086
|
+
def _configuration_objects(value, seen=None):
|
|
1087
|
+
"""Each configuration object in an unpickled payload, however deeply it is held."""
|
|
1088
|
+
seen = set() if seen is None else seen
|
|
1089
|
+
if id(value) in seen:
|
|
1090
|
+
return
|
|
1091
|
+
seen.add(id(value))
|
|
1092
|
+
if isinstance(value, dict):
|
|
1093
|
+
items = list(value.values())
|
|
1094
|
+
elif isinstance(value, (list, tuple)):
|
|
1095
|
+
items = list(value)
|
|
1096
|
+
elif dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
1097
|
+
own = getattr(value, "__dict__", None)
|
|
1098
|
+
if own is None:
|
|
1099
|
+
return
|
|
1100
|
+
yield value
|
|
1101
|
+
items = list(own.values())
|
|
1102
|
+
else:
|
|
1103
|
+
return
|
|
1104
|
+
for item in items:
|
|
1105
|
+
yield from _configuration_objects(item, seen)
|
|
1106
|
+
|
|
1107
|
+
|
|
1108
|
+
def _give_earlier_values(payload) -> None:
|
|
1109
|
+
"""
|
|
1110
|
+
Give each configuration object in an unpickled payload the `_EARLIER_VALUES` it lacks.
|
|
1111
|
+
|
|
1112
|
+
An object pickled before a setting existed would otherwise read the
|
|
1113
|
+
class default, such as a telescope's `resolution_budget`, which runs
|
|
1114
|
+
then did not have.
|
|
1115
|
+
"""
|
|
1116
|
+
for value in _configuration_objects(payload):
|
|
1117
|
+
own = value.__dict__
|
|
1118
|
+
for name, item in _EARLIER_VALUES.get(type(value).__name__, {}).items():
|
|
1119
|
+
if name not in own:
|
|
1120
|
+
object.__setattr__(value, name, item)
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def _move_old_tables(payload) -> None:
|
|
1124
|
+
"""
|
|
1125
|
+
Point missing packaged tables named under the former name at this installation's copies.
|
|
1126
|
+
|
|
1127
|
+
Changes the configuration objects of an unpickled payload in place; see
|
|
1128
|
+
:func:`_moved_package_file`.
|
|
1129
|
+
"""
|
|
1130
|
+
for value in _configuration_objects(payload):
|
|
1131
|
+
for name, item in list(value.__dict__.items()):
|
|
1132
|
+
if isinstance(item, Path) or hasattr(item, "__fspath__"):
|
|
1133
|
+
moved = _moved_package_file(item)
|
|
1134
|
+
if moved is not None:
|
|
1135
|
+
object.__setattr__(value, name, moved)
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
def _load_pickle(path: Path) -> dict:
|
|
1139
|
+
from .utils import _unpickle
|
|
1140
|
+
|
|
1141
|
+
with open(path, "rb") as handle:
|
|
1142
|
+
try:
|
|
1143
|
+
return _unpickle(handle)
|
|
1144
|
+
# Such as a pickle of a class only another version or branch of ECLIPSE had.
|
|
1145
|
+
except (AttributeError, ImportError) as error:
|
|
1146
|
+
raise ValueError(f"{path} cannot be unpickled by this version of ECLIPSE: "
|
|
1147
|
+
f"{error}") from error
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def convert_results_pickle(pickle_path: str | Path, path: str | Path | None = None,
|
|
1151
|
+
overwrite: bool = False) -> Path:
|
|
1152
|
+
"""
|
|
1153
|
+
Rewrite a results pickle from ECLIPSE 0.11.0 and earlier as a results file.
|
|
1154
|
+
|
|
1155
|
+
Everything in the pickle is kept, and the new file is read back as a
|
|
1156
|
+
check. A setting that did not exist when the pickle was made gets
|
|
1157
|
+
today's default, with a warning, unless runs then had another value:
|
|
1158
|
+
their fits were unweighted, so the file records `fit_weighted` False,
|
|
1159
|
+
and `weighted` False for a FitConfig in it. Missing packaged tables
|
|
1160
|
+
named in the former folder `euvst_response/data` are recorded as this
|
|
1161
|
+
installation's copies. A pickle holding no results, such as a synthesis
|
|
1162
|
+
pickle, is refused. Reading a pickle runs
|
|
1163
|
+
whatever code it holds, so only convert files you trust.
|
|
1164
|
+
|
|
1165
|
+
Parameters
|
|
1166
|
+
----------
|
|
1167
|
+
pickle_path : str or Path
|
|
1168
|
+
Pickle to convert.
|
|
1169
|
+
path : str or Path, optional
|
|
1170
|
+
Results file to write. Default None, which writes it beside the
|
|
1171
|
+
pickle, with the same name ending in `.h5`.
|
|
1172
|
+
overwrite : bool, optional
|
|
1173
|
+
Replace a file already at `path`, which is otherwise refused.
|
|
1174
|
+
Default False.
|
|
1175
|
+
|
|
1176
|
+
Returns
|
|
1177
|
+
-------
|
|
1178
|
+
Path
|
|
1179
|
+
File written.
|
|
1180
|
+
"""
|
|
1181
|
+
pickle_path = Path(pickle_path).expanduser()
|
|
1182
|
+
if is_results_file(pickle_path):
|
|
1183
|
+
raise ValueError(f"{pickle_path} is already an HDF5 file.")
|
|
1184
|
+
if not _is_pickle(pickle_path):
|
|
1185
|
+
raise ValueError(f"{pickle_path} is not a pickle.")
|
|
1186
|
+
target = pickle_path.with_suffix(".h5") if path is None else Path(path).expanduser()
|
|
1187
|
+
if target.suffix.lower() in _PICKLE_SUFFIXES:
|
|
1188
|
+
target = target.with_suffix(".h5")
|
|
1189
|
+
if target.is_dir():
|
|
1190
|
+
raise IsADirectoryError(f"{target} is a directory; name the results file to write.")
|
|
1191
|
+
if target.resolve() == pickle_path.resolve():
|
|
1192
|
+
raise ValueError(f"{target} is the pickle itself; name another file to write.")
|
|
1193
|
+
if target.exists() and not overwrite:
|
|
1194
|
+
raise FileExistsError(f"{target} exists; pass overwrite=True to replace it.")
|
|
1195
|
+
# Write through a symlink to its target, as writing to the link would.
|
|
1196
|
+
requested = target
|
|
1197
|
+
if target.is_symlink():
|
|
1198
|
+
target = Path(os.path.realpath(target))
|
|
1199
|
+
payload = _load_pickle(pickle_path)
|
|
1200
|
+
results = payload.get("results") if isinstance(payload, dict) else None
|
|
1201
|
+
if not isinstance(results, dict) or "all_combinations" not in results:
|
|
1202
|
+
raise ValueError(f"{pickle_path} holds no results. A synthesis pickle converts with "
|
|
1203
|
+
f"eclipse.convert_synthesis_pickle.")
|
|
1204
|
+
_move_old_tables(payload)
|
|
1205
|
+
# Recorded in the new file, so reading it does not warn again.
|
|
1206
|
+
if _from_before_weighting(payload):
|
|
1207
|
+
warnings.warn(_BEFORE_WEIGHTING.format(path=pickle_path), UserWarning, stacklevel=2)
|
|
1208
|
+
# Write under a temporary name and read it back before replacing the
|
|
1209
|
+
# target, so the result is never an unreadable file or a lost one.
|
|
1210
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1211
|
+
staging = _new_partial(target)
|
|
1212
|
+
try:
|
|
1213
|
+
save_results(staging, payload)
|
|
1214
|
+
del payload, results
|
|
1215
|
+
try:
|
|
1216
|
+
load_results(staging, _stacklevel=3)
|
|
1217
|
+
except (MemoryError, OSError):
|
|
1218
|
+
raise
|
|
1219
|
+
except Exception as error:
|
|
1220
|
+
raise ValueError(f"{pickle_path} converts to a file that cannot be read back, "
|
|
1221
|
+
f"so {target} is not written: {error}") from error
|
|
1222
|
+
os.replace(staging, target)
|
|
1223
|
+
except BaseException:
|
|
1224
|
+
staging.unlink(missing_ok=True)
|
|
1225
|
+
raise
|
|
1226
|
+
return requested
|