eclipse 0.13.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. eclipse/__init__.py +95 -0
  2. eclipse/analysis.py +853 -0
  3. eclipse/atmosphere.py +859 -0
  4. eclipse/band.py +477 -0
  5. eclipse/cli.py +138 -0
  6. eclipse/config.py +1710 -0
  7. eclipse/continuum.py +413 -0
  8. eclipse/data/eis_response/EIS_EffArea_A.004 +61 -0
  9. eclipse/data/eis_response/EIS_EffArea_B.004 +60 -0
  10. eclipse/data/eis_response/EIS_EffArea_B.005 +64 -0
  11. eclipse/data/eis_response/README_dz2025 +50 -0
  12. eclipse/data/eis_response/fit_eis_ea_2023-05-04_smooth.sav +0 -0
  13. eclipse/data/eis_response/source.txt +33 -0
  14. eclipse/data/science_cases.yaml +365 -0
  15. eclipse/data/throughput/grating_reflection_efficiency.dat +25 -0
  16. eclipse/data/throughput/primary_mirror_coating_reflectance.dat +25 -0
  17. eclipse/data/throughput/source.txt +6 -0
  18. eclipse/data/throughput/throughput_aluminium_1000_angstrom.dat +503 -0
  19. eclipse/data/throughput/throughput_aluminium_oxide_1000_angstrom.dat +503 -0
  20. eclipse/data/throughput/throughput_carbon_1000_angstrom.dat +503 -0
  21. eclipse/data_processing.py +904 -0
  22. eclipse/deprecated.py +161 -0
  23. eclipse/eis_calibration.py +573 -0
  24. eclipse/extern/__init__.py +0 -0
  25. eclipse/extern/mpfit.py +2388 -0
  26. eclipse/fitting.py +1734 -0
  27. eclipse/frame.py +594 -0
  28. eclipse/full_frame.py +944 -0
  29. eclipse/main.py +1646 -0
  30. eclipse/monte_carlo.py +660 -0
  31. eclipse/pinhole_diffraction.py +409 -0
  32. eclipse/psf.py +6 -0
  33. eclipse/radiometric.py +1328 -0
  34. eclipse/raster.py +869 -0
  35. eclipse/readout.py +1002 -0
  36. eclipse/resolution_budget.py +108 -0
  37. eclipse/results_file.py +1226 -0
  38. eclipse/sampling.py +351 -0
  39. eclipse/science_cases.py +364 -0
  40. eclipse/synthesis.py +2492 -0
  41. eclipse/synthesis_cli.py +12 -0
  42. eclipse/synthesis_file.py +882 -0
  43. eclipse/utils.py +1145 -0
  44. eclipse-0.13.0.dist-info/METADATA +70 -0
  45. eclipse-0.13.0.dist-info/RECORD +50 -0
  46. eclipse-0.13.0.dist-info/WHEEL +5 -0
  47. eclipse-0.13.0.dist-info/entry_points.txt +7 -0
  48. eclipse-0.13.0.dist-info/licenses/LICENSE +3 -0
  49. eclipse-0.13.0.dist-info/top_level.txt +2 -0
  50. euvst_response/__init__.py +129 -0
@@ -0,0 +1,1226 @@
1
+ """
2
+ Results file: the instrument simulation's results as HDF5.
3
+
4
+ `eclipse --config run.yaml` writes `run/result/run.h5`, which
5
+ :func:`eclipse.load_instrument_response_results` reads. Reading a
6
+ file runs no code from it, and any language can read it.
7
+
8
+ File layout, version 1
9
+ ----------------------
10
+ Root attributes `format` (`"eclipse-results"`) and `version` (`1`). The
11
+ results are a tree. An array with dimensions is a dataset, with a `unit`
12
+ attribute if it is a quantity; a cube, or a mapping, list or tuple holding
13
+ such arrays, is a group; anything else is a JSON attribute of its group.
14
+
15
+ A mapping keyed by names lists them in order, as JSON, in `eclipse_order`.
16
+ Other groups have an `eclipse_type` (plain text, as are `eclipse_length`,
17
+ `target` and `unit`):
18
+
19
+ - `ndcube`: an NDCube's `data` with its `unit`, `wcs` and `meta`.
20
+ - `map`: a mapping whose keys cannot name members, such as results keyed
21
+ by parameter tuples; the keys as JSON in `eclipse_keys`, the values as
22
+ members `0`, `1`, ...
23
+ - `list`, `tuple`: the length in `eclipse_length`, the items as members.
24
+ - `ref`: a value found in more than one place, written once and elsewhere
25
+ as a group whose `target` is the path it was written at.
26
+
27
+ Values JSON cannot hold are objects with an `__eclipse__` entry naming the
28
+ kind: `float` (not finite), `numpy`, `quantity`, `array`, `unit`, `tuple`,
29
+ `map`, `date`, `datetime`, `set`, `frozenset`, `bytes` (base 64), `path`,
30
+ `resource` (a path inside the installed package), `numpy_type`, `wcs` (a
31
+ FITS header plus the unrounded `cunit`, `crpix`, `crval`, and `cdelt` and
32
+ `pc` or `crota`, or `cd`) and `dataclass` (a configuration object's
33
+ `class`, `fields` and `derived` values). Only the classes in
34
+ :func:`_dataclass_registry` are rebuilt.
35
+
36
+ Results pickles from older versions still load, with a warning, and
37
+ :func:`convert_results_pickle` converts one.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import base64
43
+ import dataclasses
44
+ import datetime
45
+ import functools
46
+ import json
47
+ import math
48
+ import os
49
+ import re
50
+ import sys
51
+ import warnings
52
+ from collections.abc import Mapping
53
+ from importlib.resources import files
54
+ from pathlib import Path, PurePosixPath
55
+
56
+ import astropy.units as u
57
+ import h5py
58
+ import numpy as np
59
+ from astropy.io import fits
60
+ from astropy.utils.masked import Masked
61
+ from astropy.wcs import WCS
62
+ from ndcube import NDCube
63
+
64
+ from .atmosphere import _check_format, _is_pickle, _new_partial
65
+
66
+ __all__ = ["FORMAT_NAME", "FORMAT_VERSION", "save_results", "load_results",
67
+ "convert_results_pickle", "is_results_file"]
68
+
69
+ FORMAT_NAME = "eclipse-results"
70
+ FORMAT_VERSION = 1
71
+
72
+ # Key that tags a JSON value this module must rebuild.
73
+ TAG = "__eclipse__"
74
+ # Group attributes that describe the group rather than hold one of its values.
75
+ TYPE, ORDER, KEYS, LENGTH = "eclipse_type", "eclipse_order", "eclipse_keys", "eclipse_length"
76
+ _RESERVED = {TYPE, ORDER, KEYS, LENGTH, "format", "version"}
77
+
78
+ # Suffixes older versions and scripts gave pickles, as accepted by the synthesis.
79
+ _PICKLE_SUFFIXES = (".pkl", ".pickle", ".dill")
80
+
81
+ # Arrays with at least this many elements are compressed.
82
+ _COMPRESS_FROM = 1024
83
+
84
+
85
+ @functools.lru_cache(maxsize=None)
86
+ def _dataclass_registry() -> dict:
87
+ """Classes that may be rebuilt from a file, keyed by name.
88
+
89
+ Imported when first needed, since the time series settings import the
90
+ synthesis, which reading results does not otherwise need.
91
+ """
92
+ from .config import (AluminiumFilter, Detector_EIS, Detector_SWC,
93
+ Simulation, Telescope_EIS, Telescope_EUVST)
94
+ from .fitting import FitComponent, FitConfig
95
+ from .raster import RasterPlan, SynthesisSettings
96
+
97
+ return {cls.__name__: cls for cls in (
98
+ AluminiumFilter, Detector_EIS, Detector_SWC, Simulation,
99
+ Telescope_EIS, Telescope_EUVST, FitComponent, FitConfig,
100
+ RasterPlan, SynthesisSettings,
101
+ )}
102
+
103
+
104
+ def _package_root() -> Path:
105
+ """Directory of this ECLIPSE installation's package data."""
106
+ return Path(os.fspath(files("eclipse")))
107
+
108
+
109
+ def _moved_package_file(path):
110
+ """
111
+ This installation's copy of a packaged table named under the package's former name.
112
+
113
+ Older versions were installed as `euvst_response`, so their results can
114
+ name the packaged tables inside it, a folder an upgrade removes. None if
115
+ `path` exists, is not such a table, or the table is not in this installation.
116
+ A path that cannot be reached counts as missing.
117
+ """
118
+ path = Path(os.fspath(path))
119
+ parts = path.parts
120
+ for i in range(len(parts) - 2, -1, -1):
121
+ if parts[i] == "euvst_response" and parts[i + 1] == "data":
122
+ relative = _package_parts(Path(*parts[i + 1:]).as_posix())
123
+ if relative is None or os.path.exists(path):
124
+ return None
125
+ moved = _package_root().joinpath(*relative)
126
+ return moved if moved.is_file() else None
127
+ return None
128
+
129
+
130
+ # ----------------------------------------------------------------------
131
+ # WCS
132
+ # ----------------------------------------------------------------------
133
+ def _wcs_to_tree(wcs: WCS) -> dict:
134
+ """
135
+ A WCS as a FITS header, plus the numbers of its axes unchanged.
136
+
137
+ A header gives the axes in SI units and to 14 digits, so a wavelength
138
+ axis in cm would come back in m, and a step of 0.2 arcsec as
139
+ 0.2000000000000016. The coordinates agree to that precision, but not
140
+ the numbers a caller reads from `wcs.wcs`. So the units, reference
141
+ pixels and values, and the steps and rotation or CD matrix are kept
142
+ beside the header as given, and restored on reading.
143
+ """
144
+ # Read these before to_header(), which normalises a WCS in place; it is
145
+ # called on a copy so that saving a cube leaves the caller's WCS alone.
146
+ tree = {"cunit": [str(c) for c in wcs.wcs.cunit], "crpix": wcs.wcs.crpix.tolist(),
147
+ "crval": wcs.wcs.crval.tolist()}
148
+ # The rotation in the form the WCS uses, checked in wcslib's order:
149
+ # a PC matrix (the default), then a CD matrix, then CROTA.
150
+ lin = wcs.wcs
151
+ if lin.has_pc() or not (lin.has_cd() or lin.has_crota()):
152
+ tree.update(cdelt=lin.cdelt.tolist(),
153
+ pc=lin.pc.tolist() if lin.has_pc() else np.eye(lin.naxis).tolist())
154
+ elif lin.has_cd():
155
+ tree["cd"] = lin.cd.tolist()
156
+ else:
157
+ tree.update(cdelt=lin.cdelt.tolist(), crota=lin.crota.tolist())
158
+ tree["header"] = wcs.deepcopy().to_header().tostring(sep="\n")
159
+ if wcs.has_distortion:
160
+ _warn("A results file does not hold a WCS's distortion, so this WCS's is left out.")
161
+ return tree
162
+
163
+
164
+ # Header cards a WCS is rebuilt from: its axes, and any frame, time and
165
+ # observer. The number of axes (wcslib allocates its square) and the steps
166
+ # and rotation or CD matrix come from the values kept beside the header,
167
+ # since a header gives a CD matrix as a PC matrix and steps, which would
168
+ # take precedence. Other cards are left out, such as a distortion's, which
169
+ # astropy tabulates at whatever size the header asks.
170
+ _WCS_CARDS = re.compile(
171
+ r"WCSNAME|(CTYPE|CUNIT|CRPIX|CRVAL|CNAME|CRDER|CSYER|CZPHS|CPERI)\d{1,2}"
172
+ r"|(PV|PS)\d{1,2}_\d{1,2}|LONPOLE|LATPOLE|RADESYS|EQUINOX|RESTFRQ|RESTWAV|VELREF"
173
+ r"|SPECSYS|SSYSOBS|SSYSSRC|VELOSYS|ZSOURCE|VELANGL|TIMESYS|TREFPOS|TREFDIR|PLEPHEM"
174
+ r"|TIMEUNIT|TIMEDEL|TIMEPIXR|TIMEOFFS|TIMSYER|TIMRDER|TSTART|TSTOP|TELAPSE|XPOSURE"
175
+ r"|JEPOCH|BEPOCH|(DATE|MJD)-(OBS|BEG|AVG|END)|DATEREF|MJDREF[IF]?|OBSGEO-[XYZBLH]"
176
+ r"|OBSORBIT|DSUN_OBS|HGLN_OBS|HGLT_OBS|CRLN_OBS|CRLT_OBS|RSUN_REF|[ABC]_RADIUS"
177
+ r"|BLON_OBS|BLAT_OBS|BDIS_OBS")
178
+
179
+
180
+ def _wcs_from_tree(tree: dict) -> WCS:
181
+ """Rebuild a WCS written by :func:`_wcs_to_tree`."""
182
+ header = fits.Header.fromstring(tree["header"], sep="\n")
183
+ wcs = WCS(fits.Header([card for card in header.cards if _WCS_CARDS.fullmatch(card.keyword)]))
184
+ if wcs.naxis != len(tree["cunit"]):
185
+ raise ValueError("The results file holds a WCS whose header does not have its axes.")
186
+ for axis, unit in enumerate(tree["cunit"]):
187
+ wcs.wcs.cunit[axis] = unit
188
+ wcs.wcs.crpix, wcs.wcs.crval = tree["crpix"], tree["crval"]
189
+ if "cd" in tree:
190
+ wcs.wcs.cd = tree["cd"]
191
+ elif "crota" in tree:
192
+ wcs.wcs.cdelt, wcs.wcs.crota = tree["cdelt"], tree["crota"]
193
+ else:
194
+ wcs.wcs.cdelt, wcs.wcs.pc = tree["cdelt"], tree["pc"]
195
+ return wcs
196
+
197
+
198
+ # ----------------------------------------------------------------------
199
+ # Values as JSON
200
+ # ----------------------------------------------------------------------
201
+ def _is_array(value) -> bool:
202
+ """Whether `value` is a dataset: an array of numbers or booleans with dimensions.
203
+
204
+ A masked array is not, as its mask would be lost; the JSON writer refuses it.
205
+ """
206
+ return (isinstance(value, np.ndarray) and value.ndim > 0 and value.dtype.kind in "biufc"
207
+ and not isinstance(value, (np.ma.MaskedArray, Masked)))
208
+
209
+
210
+ def _holds_array(value) -> bool:
211
+ if _is_array(value) or isinstance(value, NDCube):
212
+ return True
213
+ if isinstance(value, dict):
214
+ return any(_holds_array(v) for v in value.values())
215
+ if isinstance(value, (list, tuple)):
216
+ return any(_holds_array(v) for v in value)
217
+ return False
218
+
219
+
220
+ def _warn(message: str) -> None:
221
+ """Warn with `message`, attributed to the first caller outside this module."""
222
+ frame, level = sys._getframe(1), 2
223
+ while frame is not None and frame.f_globals.get("__name__") == __name__:
224
+ frame, level = frame.f_back, level + 1
225
+ warnings.warn(message, UserWarning, stacklevel=level)
226
+
227
+
228
+ def _unit_text(unit) -> str:
229
+ """Text that reads back as `unit`; TypeError if there is none."""
230
+ if not isinstance(unit, u.UnitBase):
231
+ raise TypeError(f"The unit {unit} cannot be written to a results file, as it is not a "
232
+ f"plain unit.")
233
+ return _checked_unit_text(unit)
234
+
235
+
236
+ @functools.lru_cache(maxsize=None)
237
+ def _checked_unit_text(unit: u.UnitBase) -> str:
238
+ # Write the whole scale, since astropy's text gives it to six digits.
239
+ text = unit.to_string()
240
+ if unit.scale != 1:
241
+ text = (f"{float(unit.scale)!r} "
242
+ f"{u.CompositeUnit(1, unit.bases, unit.powers).to_string()}").strip()
243
+ try:
244
+ same = u.Unit(text) == unit
245
+ except ValueError:
246
+ same = False
247
+ # A unit defined only in a script, for example, is unknown to a reader.
248
+ if not same:
249
+ raise TypeError(f"The unit {unit} cannot be written to a results file, as it would not "
250
+ f"read back as itself.")
251
+ return text
252
+
253
+
254
+ def _package_parts(text: str):
255
+ """Parts of `text`, a path relative to the package, or None if it could lead outside it."""
256
+ parts = PurePosixPath(text).parts
257
+ if parts and all(part != ".." and re.fullmatch(r"[\w.-]+", part) for part in parts):
258
+ return parts
259
+ return None
260
+
261
+
262
+ def _jsonable(value):
263
+ """`value` in a form json can write, with kinds JSON lacks tagged."""
264
+ if value is None or isinstance(value, (bool, int, str)):
265
+ return value
266
+ if isinstance(value, np.generic):
267
+ item = value.item()
268
+ # A longdouble, for example, has no Python equivalent, and a NumPy
269
+ # date or time has none that keeps its units.
270
+ if isinstance(item, np.generic) or value.dtype.kind not in "biufS":
271
+ raise TypeError(f"Values of type {type(value).__name__} cannot be written to a "
272
+ f"results file.")
273
+ if value.dtype.kind == "S":
274
+ return _jsonable(item)
275
+ # Keep the type, as for a key of a cube's sampling or a setting given
276
+ # in single precision.
277
+ return {TAG: "numpy", "dtype": value.dtype.str, "value": _jsonable(item)}
278
+ if isinstance(value, float):
279
+ return value if math.isfinite(value) else {TAG: "float", "value": repr(value)}
280
+ if isinstance(value, (np.ma.MaskedArray, Masked)):
281
+ raise TypeError("A masked array cannot be written to a results file: its mask would be "
282
+ "lost.")
283
+ if isinstance(value, u.Quantity):
284
+ return {TAG: "quantity", "value": _jsonable(value.value), "unit": _unit_text(value.unit)}
285
+ if isinstance(value, np.ndarray):
286
+ if value.dtype.kind not in "biufU":
287
+ raise TypeError(f"An array of {value.dtype} cannot be written to a results file.")
288
+ return {TAG: "array", "dtype": value.dtype.str, "shape": list(value.shape),
289
+ "value": _jsonable(value.tolist())}
290
+ if isinstance(value, u.UnitBase):
291
+ return {TAG: "unit", "value": _unit_text(value)}
292
+ if isinstance(value, WCS):
293
+ return {TAG: "wcs", **_wcs_to_tree(value)}
294
+ if dataclasses.is_dataclass(value) and not isinstance(value, type):
295
+ registry = _dataclass_registry()
296
+ if registry.get(type(value).__name__) is not type(value):
297
+ raise TypeError(f"A {type(value).__name__} cannot be written to a results file: "
298
+ f"only {', '.join(sorted(registry))} are rebuilt when it is read.")
299
+ # Also keep the fields the object derives, such as a detector's dark
300
+ # current from its temperature, with the values the run used.
301
+ derived = {field.name: getattr(value, field.name) for field in dataclasses.fields(value)
302
+ if not field.init and hasattr(value, field.name)}
303
+ return {TAG: "dataclass", "class": type(value).__name__,
304
+ "fields": {name: _jsonable(field) for name, field in _init_fields(value).items()},
305
+ "derived": {name: _jsonable(field) for name, field in derived.items()}}
306
+ if isinstance(value, type) and issubclass(value, np.generic):
307
+ try:
308
+ name = _plain_dtype(value).name
309
+ except (TypeError, ValueError):
310
+ raise TypeError(f"The type {value.__name__} cannot be written to a results "
311
+ f"file.") from None
312
+ return {TAG: "numpy_type", "value": name}
313
+ # YAML reads !!set and !!binary as these, so a configuration can hold them.
314
+ if isinstance(value, (set, frozenset)):
315
+ items = [_jsonable(v) for v in value]
316
+ try:
317
+ items = sorted(items)
318
+ except TypeError:
319
+ pass
320
+ return {TAG: "frozenset" if isinstance(value, frozenset) else "set", "items": items}
321
+ if isinstance(value, bytes):
322
+ return {TAG: "bytes", "value": base64.b64encode(value).decode("ascii")}
323
+ if isinstance(value, datetime.datetime):
324
+ return {TAG: "datetime", "value": value.isoformat()}
325
+ if isinstance(value, datetime.date):
326
+ return {TAG: "date", "value": value.isoformat()}
327
+ # Paths, and anything else with a place on the filesystem, such as the
328
+ # throughput tables found through importlib.resources, which need not be
329
+ # a pathlib.Path. A path inside the package is written relative to it, so
330
+ # the reader finds its own copy; any other is written as it is.
331
+ if isinstance(value, Path) or hasattr(value, "__fspath__"):
332
+ try:
333
+ relative = Path(os.fspath(value)).relative_to(_package_root()).as_posix()
334
+ except ValueError:
335
+ relative = None
336
+ if relative is not None and _package_parts(relative):
337
+ return {TAG: "resource", "value": relative}
338
+ return {TAG: "path", "value": os.fspath(value)}
339
+ if isinstance(value, tuple):
340
+ return {TAG: "tuple", "items": [_jsonable(v) for v in value]}
341
+ if isinstance(value, list):
342
+ return [_jsonable(v) for v in value]
343
+ if isinstance(value, dict):
344
+ if all(isinstance(k, str) for k in value) and TAG not in value:
345
+ return {k: _jsonable(v) for k, v in value.items()}
346
+ return {TAG: "map", "items": [[_jsonable(k), _jsonable(v)] for k, v in value.items()]}
347
+ raise TypeError(f"Values of type {type(value).__name__} cannot be written to a results file.")
348
+
349
+
350
+ # The value each setting had in runs made before it existed. An object or
351
+ # file from then that lacks one gets this value, with a warning only where it
352
+ # differs from today's default. Fits were unweighted before
353
+ # FitConfig.weighted, and None is the detector material's value.
354
+ _MATERIAL = {"fano_factor": None, "pair_energy_300k": None, "pair_energy_slope": None}
355
+ _EARLIER_VALUES = {
356
+ "FitConfig": {"weighted": False},
357
+ "Telescope_EUVST": {"resolution_budget": "none"},
358
+ "Detector_SWC": {**_MATERIAL,
359
+ "dispersion": (198.885944, 1.2516577, -1.361114e-4),
360
+ "wavelength_offset": 0.0 * u.Angstrom,
361
+ "lit_band": (170.0 * u.Angstrom, 212.3 * u.Angstrom),
362
+ "slit_image_tilt": (0.0, 0.0)},
363
+ "Detector_EIS": dict(_MATERIAL),
364
+ "AluminiumFilter": {"visible_decade_thickness": 170.0 * u.angstrom},
365
+ "Simulation": {"pinhole_visible_wavelength": 600.0 * u.nm, "readout_dark_current": False,
366
+ "focal_plane_dispersion": False},
367
+ }
368
+
369
+
370
+ def _is_default(cls, name: str, value) -> bool:
371
+ """Whether `value` is the default of `cls`'s setting `name`."""
372
+ field = next(field for field in dataclasses.fields(cls) if field.name == name)
373
+ if field.default is not dataclasses.MISSING:
374
+ default = field.default
375
+ elif field.default_factory is not dataclasses.MISSING:
376
+ default = field.default_factory()
377
+ else:
378
+ return False
379
+ if value is None or default is None:
380
+ return value is None and default is None
381
+ if isinstance(value, str) or isinstance(default, str):
382
+ return value == default
383
+ try:
384
+ value, default = u.Quantity(value), u.Quantity(default)
385
+ except (TypeError, ValueError):
386
+ return False
387
+ return value.shape == default.shape and bool(np.all(value == default))
388
+
389
+
390
+ def _init_fields(value) -> dict:
391
+ """Arguments a configuration object was made with, as far as it has them.
392
+
393
+ Fields derived in __post_init__, such as a detector's dark current from
394
+ its temperature, are written separately as `derived` and restored with
395
+ the run's values. An object from an older version's pickle may lack a
396
+ field added since. It then gets today's default, or, for fields in
397
+ `_EARLIER_VALUES` such as FitConfig's `weighted`, the value runs had
398
+ before the field existed. An object from another version may have a
399
+ setting this version lacks. It is written too, so that reading the file
400
+ reports that it is left out.
401
+ """
402
+ fields, missing, earlier = {}, [], []
403
+ # Use what the instance itself holds, so a missing field is not silently
404
+ # taken from the class default.
405
+ own = getattr(value, "__dict__", None)
406
+ before = _EARLIER_VALUES.get(type(value).__name__, {})
407
+ for field in dataclasses.fields(value):
408
+ if not field.init:
409
+ continue
410
+ if own is None:
411
+ fields[field.name] = getattr(value, field.name)
412
+ elif field.name in own:
413
+ fields[field.name] = own[field.name]
414
+ elif field.name in before:
415
+ fields[field.name] = before[field.name]
416
+ earlier.append(field.name)
417
+ else:
418
+ missing.append(field.name)
419
+ for name in earlier:
420
+ if _is_default(type(value), name, before[name]):
421
+ continue
422
+ _warn(f"This {type(value).__name__} has no {name}, which ECLIPSE added after it "
423
+ f"was made; it gets {before[name]!r}, as runs then had.")
424
+ if own is not None:
425
+ names = {field.name for field in dataclasses.fields(value)}
426
+ fields.update({key: item for key, item in own.items() if key not in names
427
+ and not key.startswith("_") and not hasattr(type(value), key)})
428
+ if missing:
429
+ _warn(f"This {type(value).__name__} has no {', '.join(missing)}, which ECLIPSE added "
430
+ f"after it was made; it gets today's default.")
431
+ return fields
432
+
433
+
434
+ def _unjson(value, reading: _Reading):
435
+ """Rebuild what :func:`_jsonable` wrote; `reading` is the state of the file read."""
436
+ if isinstance(value, list):
437
+ return [_unjson(v, reading) for v in value]
438
+ if not isinstance(value, dict):
439
+ return value
440
+ tag = value.get(TAG)
441
+ if tag is None:
442
+ return {k: _unjson(v, reading) for k, v in value.items()}
443
+ if tag == "quantity":
444
+ return u.Quantity(_unjson(value["value"], reading), u.Unit(value["unit"]), dtype=None)
445
+ if tag == "numpy":
446
+ return _plain_dtype(value["dtype"]).type(_unjson(value["value"], reading))
447
+ if tag == "float":
448
+ return float(value["value"])
449
+ if tag == "array":
450
+ dtype = _plain_dtype(value["dtype"])
451
+ items = np.array(_unjson(value["value"], reading),
452
+ dtype=object if dtype.kind == "U" else dtype)
453
+ # Text items must fit the width the file declares.
454
+ if dtype.kind == "U" and not all(isinstance(item, str) and len(item) <= dtype.itemsize // 4
455
+ for item in items.flat):
456
+ raise ValueError("A text array in the results file holds more than text of its width.")
457
+ # Use the shape as written, which an empty array's items do not give.
458
+ shape = value["shape"]
459
+ if (not all(isinstance(size, int) and size >= 0 for size in shape)
460
+ or math.prod(shape) != items.size):
461
+ raise ValueError("An array in the results file does not have the shape it gives.")
462
+ return items.astype(dtype, copy=False).reshape(shape)
463
+ if tag == "unit":
464
+ return u.Unit(value["value"])
465
+ if tag == "wcs":
466
+ return _wcs_from_tree(value)
467
+ if tag == "dataclass":
468
+ registry = _dataclass_registry()
469
+ name = value["class"]
470
+ if name not in registry:
471
+ raise ValueError(f"The results file names a class ECLIPSE will not construct: "
472
+ f"{name!r}. Only {', '.join(sorted(registry))} are rebuilt, so "
473
+ f"that reading a file cannot construct anything the file chooses.")
474
+ return _rebuild(registry[name],
475
+ {k: _unjson(v, reading) for k, v in value["fields"].items()},
476
+ {k: _unjson(v, reading) for k, v in value["derived"].items()},
477
+ reading)
478
+ if tag == "numpy_type":
479
+ return _plain_dtype(value["value"]).type
480
+ if tag in ("set", "frozenset"):
481
+ return (set if tag == "set" else frozenset)(_unjson(v, reading) for v in value["items"])
482
+ if tag == "bytes":
483
+ return base64.b64decode(value["value"])
484
+ if tag == "datetime":
485
+ return datetime.datetime.fromisoformat(value["value"])
486
+ if tag == "date":
487
+ return datetime.date.fromisoformat(value["value"])
488
+ if tag == "path":
489
+ return Path(value["value"])
490
+ if tag == "resource":
491
+ # Check the path as written, before touching the filesystem, so a
492
+ # file cannot make the reader look outside the package at all.
493
+ parts = _package_parts(value["value"])
494
+ if parts is None:
495
+ raise ValueError(f"The results file names a package file, {value['value']!r}, "
496
+ f"outside the package.")
497
+ return _package_root().joinpath(*parts)
498
+ if tag == "tuple":
499
+ return tuple(_unjson(v, reading) for v in value["items"])
500
+ if tag == "map":
501
+ return {_unjson(k, reading): _unjson(v, reading) for k, v in value["items"]}
502
+ raise ValueError(f"The results file holds a value tagged {tag!r}, which this ECLIPSE "
503
+ f"does not know.")
504
+
505
+
506
+ def _plain_dtype(name) -> np.dtype:
507
+ """Dtype `name`, which must be plain numbers, booleans or text, as ECLIPSE writes.
508
+
509
+ A dtype can also give each element its own shape, which would let a few
510
+ bytes of a crafted file fill any amount of memory.
511
+ """
512
+ dtype = np.dtype(name)
513
+ if dtype.kind not in "biufcU" or dtype.subdtype is not None or dtype.names is not None:
514
+ raise ValueError(f"The results file holds values of type {name!r}, which a results "
515
+ f"file does not.")
516
+ return dtype
517
+
518
+
519
+ def _rebuild(cls, stored: dict, derived: dict, reading: _Reading):
520
+ """
521
+ Configuration object rebuilt from what a file stored.
522
+
523
+ The object is constructed again, so it is checked and computes its
524
+ derived fields, which are then set to the values the run had. If this
525
+ version would not make the object, because a setting has gone or a
526
+ check has been added since, the file still reads: the object is rebuilt
527
+ as stored, unchecked, with a warning given once `reading` is done.
528
+ """
529
+ known = {field.name for field in dataclasses.fields(cls) if field.init}
530
+ # A setting stored under an old name is read under its new one.
531
+ stored = {getattr(cls, "_OLD_NAMES", {}).get(key, key): item for key, item in stored.items()}
532
+ # A setting in _EARLIER_VALUES that the file lacks gets the value runs
533
+ # had before it existed.
534
+ before = {name: item for name, item in _EARLIER_VALUES.get(cls.__name__, {}).items()
535
+ if name not in stored}
536
+ for name, item in before.items():
537
+ if _is_default(cls, name, item):
538
+ continue
539
+ reading.notes[f"The {cls.__name__} in the results file has no {name}, which ECLIPSE "
540
+ f"added after it was made; it gets {item!r}, as runs then had."] = None
541
+ stored = {**stored, **before}
542
+ # A setting added since the file was made gets today's default, with a
543
+ # warning, as the run did not have it. One with no default makes the
544
+ # construction below fail, and is reported there.
545
+ missing = sorted(field.name for field in dataclasses.fields(cls)
546
+ if field.init and field.name not in stored
547
+ and (field.default is not dataclasses.MISSING
548
+ or field.default_factory is not dataclasses.MISSING))
549
+ if missing:
550
+ reading.notes[f"The {cls.__name__} in the results file has no {', '.join(missing)}, "
551
+ f"which ECLIPSE added after it was made; "
552
+ f"{'it gets' if len(missing) == 1 else 'they get'} today's "
553
+ f"default{'' if len(missing) == 1 else 's'}."] = None
554
+ gone = sorted(set(stored) - known)
555
+ if gone:
556
+ reading.notes[f"The {cls.__name__} in the results file has {', '.join(gone)}, which "
557
+ f"this version of ECLIPSE does not have, so "
558
+ f"{'it is' if len(gone) == 1 else 'they are'} left out."] = None
559
+ arguments = {key: item for key, item in stored.items() if key in known}
560
+ try:
561
+ obj = cls(**arguments)
562
+ except (TypeError, ValueError, AttributeError) as error:
563
+ reading.notes[f"The {cls.__name__} in the results file is not one this version of "
564
+ f"ECLIPSE would make ({error}), so it is rebuilt as it was stored, "
565
+ f"unchecked."] = None
566
+ obj = cls.__new__(cls)
567
+ for field in dataclasses.fields(cls):
568
+ if field.name in arguments:
569
+ object.__setattr__(obj, field.name, arguments[field.name])
570
+ elif field.default is not dataclasses.MISSING:
571
+ object.__setattr__(obj, field.name, field.default)
572
+ elif field.default_factory is not dataclasses.MISSING:
573
+ object.__setattr__(obj, field.name, field.default_factory())
574
+ # Set only the derived fields, so a file cannot set anything else past
575
+ # the checks.
576
+ worked_out = {field.name for field in dataclasses.fields(cls) if not field.init}
577
+ for key, item in derived.items():
578
+ if key in worked_out:
579
+ object.__setattr__(obj, key, item)
580
+ return obj
581
+
582
+
583
+ def _to_json(value) -> str:
584
+ # Strict JSON, which any language reads; non-finite numbers are tagged.
585
+ return json.dumps(_jsonable(value), allow_nan=False)
586
+
587
+
588
+ # ----------------------------------------------------------------------
589
+ # The tree as groups
590
+ # ----------------------------------------------------------------------
591
+ def _is_name(key) -> bool:
592
+ """Whether `key` can name a group member, as a dataset or an attribute."""
593
+ if (not isinstance(key, str) or key in ("", ".") or key in _RESERVED
594
+ or "/" in key or "\0" in key):
595
+ return False
596
+ # HDF5 names are UTF-8, which cannot encode a lone surrogate, for example.
597
+ try:
598
+ key.encode("utf-8")
599
+ except UnicodeEncodeError:
600
+ return False
601
+ return True
602
+
603
+
604
+ class _Unwritable(TypeError):
605
+ """A value a results file cannot hold, and its path in the results."""
606
+
607
+
608
+ def _put(group: h5py.Group, name: str, value, compression, written: dict) -> None:
609
+ """
610
+ Write `value` as member `name` of `group`.
611
+
612
+ A value holding arrays becomes a dataset or group, and anything else a
613
+ JSON attribute. `written` maps the id of each object written so far as a
614
+ dataset or group to the object and its path, so an object that appears
615
+ again is written as a reference rather than written again.
616
+ """
617
+ try:
618
+ _put_value(group, name, value, compression, written)
619
+ except _Unwritable:
620
+ raise
621
+ except (TypeError, RecursionError) as error:
622
+ # Give the value's path, which a save failing at the end of a run must report.
623
+ reason = ("A value that holds itself cannot be written to a results file."
624
+ if isinstance(error, RecursionError) else error)
625
+ raise _Unwritable(f"{group.name.rstrip('/')}/{name}: {reason}") from error
626
+
627
+
628
+ def _put_value(group: h5py.Group, name: str, value, compression, written: dict) -> None:
629
+ if not _holds_array(value):
630
+ group.attrs[name] = _to_json(value)
631
+ return
632
+ if id(value) in written:
633
+ reference = group.create_group(name)
634
+ reference.attrs[TYPE] = "ref"
635
+ reference.attrs["target"] = written[id(value)][1]
636
+ return
637
+ if _is_array(value):
638
+ data = value.value if isinstance(value, u.Quantity) else value
639
+ unit = _unit_text(value.unit) if isinstance(value, u.Quantity) else None
640
+ options = {}
641
+ if compression and data.size >= _COMPRESS_FROM:
642
+ options = {"compression": compression, "shuffle": True,
643
+ **({"compression_opts": 1} if compression == "gzip" else {})}
644
+ node = group.create_dataset(name, data=data, **options)
645
+ if unit is not None:
646
+ node.attrs["unit"] = unit
647
+ elif isinstance(value, NDCube):
648
+ left_out = [part for part in ("mask", "uncertainty", "psf")
649
+ if getattr(value, part, None) is not None]
650
+ extra = getattr(value, "extra_coords", None)
651
+ if extra is not None and not getattr(extra, "is_empty", True):
652
+ left_out.append("extra coordinates")
653
+ if len(getattr(value, "global_coords", None) or {}):
654
+ left_out.append("global coordinates")
655
+ if left_out:
656
+ _warn(f"{group.name.rstrip('/')}/{name}: a results file does not hold a cube's "
657
+ f"{' or '.join(left_out)}, so this cube's are left out.")
658
+ if value.unit is not None:
659
+ _unit_text(value.unit)
660
+ node = group.create_group(name, track_order=True)
661
+ node.attrs[TYPE] = "ndcube"
662
+ # Write the data with the cube's unit, as for any other quantity.
663
+ data = np.asarray(value.data)
664
+ if value.unit is not None:
665
+ data = u.Quantity(data, value.unit, copy=False, dtype=None)
666
+ _put(node, "data", data, compression, written)
667
+ _put(node, "wcs", value.wcs, compression, written)
668
+ # The cube's own meta; one holding arrays that another cube shares is
669
+ # written once.
670
+ meta = value.meta if isinstance(value.meta, dict) else dict(value.meta or {})
671
+ _put(node, "meta", meta, compression, written)
672
+ elif isinstance(value, dict):
673
+ node = group.create_group(name, track_order=True)
674
+ _put_dict(node, value, compression, written)
675
+ else:
676
+ node = group.create_group(name, track_order=True)
677
+ node.attrs[TYPE] = "tuple" if isinstance(value, tuple) else "list"
678
+ node.attrs[LENGTH] = len(value)
679
+ for index, item in enumerate(value):
680
+ _put(node, str(index), item, compression, written)
681
+ # Keep the object itself with its path, so no object written meanwhile
682
+ # can reuse its id.
683
+ written[id(value)] = (value, node.name)
684
+
685
+
686
+ def _put_dict(group: h5py.Group, value: dict, compression, written: dict) -> None:
687
+ if all(_is_name(key) for key in value):
688
+ group.attrs[ORDER] = json.dumps(list(value))
689
+ for key, item in value.items():
690
+ _put(group, key, item, compression, written)
691
+ else:
692
+ group.attrs[TYPE] = "map"
693
+ group.attrs[KEYS] = _to_json(list(value))
694
+ for index, item in enumerate(value.values()):
695
+ _put(group, str(index), item, compression, written)
696
+
697
+
698
+ @dataclasses.dataclass
699
+ class _Reading:
700
+ """State of the reading of one file."""
701
+
702
+ size: int # file size in bytes
703
+ stored: int = 0 # bytes stored by the datasets read so far
704
+ seen: set = dataclasses.field(default_factory=set) # objects reached
705
+ inside: set = dataclasses.field(default_factory=set) # groups being read
706
+ done: dict = dataclasses.field(default_factory=dict) # value read at each path
707
+ notes: dict = dataclasses.field(default_factory=dict) # warnings to give once read
708
+
709
+
710
+ def _attribute(node, name: str, path: Path):
711
+ """Attribute `name` of `node`, or None if absent; one string or integer, as ECLIPSE writes."""
712
+ if name not in node.attrs:
713
+ return None
714
+ # Check before reading: each element of an array attribute, or each field
715
+ # of a compound one, can refer to the same text elsewhere in the file,
716
+ # which reading would copy for each.
717
+ attribute = node.attrs.get_id(name)
718
+ dtype = attribute.dtype
719
+ if (attribute.shape is None or math.prod(attribute.shape) > 1
720
+ or not (h5py.check_string_dtype(dtype) or dtype.kind in "iu")):
721
+ raise ValueError(f"{path}: attribute {name!r} of {node.name} is not one string or "
722
+ f"integer, as a results file writes them.")
723
+ return node.attrs[name]
724
+
725
+
726
+ def _get(group: h5py.Group, name: str, path: Path, reading: _Reading):
727
+ """Read member `name` of `group`: a dataset, a group or a JSON attribute."""
728
+ # A name with a slash in it would be looked up through other groups.
729
+ if not _is_name(name):
730
+ raise ValueError(f"{path}: {group.name} has a member named {name!r}, which a results "
731
+ f"file does not.")
732
+ if name in group.attrs:
733
+ return _unjson(json.loads(_attribute(group, name, path)), reading)
734
+ link = group.get(name, getlink=True)
735
+ if link is None:
736
+ raise ValueError(f"{path}: {group.name} has no member {name!r}.")
737
+ # A link, or a dataset whose data lives elsewhere, would make the reader
738
+ # open another file, which a results file never needs.
739
+ if not isinstance(link, h5py.HardLink):
740
+ raise ValueError(f"{path}: {group.name}/{name} is a link, which a results file "
741
+ f"does not hold.")
742
+ return _read(group[name], path, reading)
743
+
744
+
745
+ def _read(node, path: Path, reading: _Reading):
746
+ """Read `node`, only once however often the results refer to it."""
747
+ if node.name in reading.done:
748
+ return reading.done[node.name]
749
+ # ECLIPSE writes each object once, so one reached twice other than by a
750
+ # reference is not ECLIPSE's: it would be read twice, or without end if
751
+ # reached from inside itself. Objects are tracked by their number in the
752
+ # file rather than kept open, since an open dataset keeps what it read.
753
+ number = h5py.h5g.get_objinfo(node.id).objno
754
+ if number in reading.seen:
755
+ raise ValueError(f"{path}: {node.name} is reached from more than one place, which a "
756
+ f"results file does not do.")
757
+ reading.seen.add(number)
758
+ if isinstance(node, h5py.Dataset):
759
+ _check_dataset(node, path, reading)
760
+ data = node[()]
761
+ unit = _attribute(node, "unit", path)
762
+ value = data if unit is None else u.Quantity(data, u.Unit(unit), copy=False, dtype=None)
763
+ else:
764
+ reading.inside.add(node.name)
765
+ if _attribute(node, TYPE, path) == "ref":
766
+ value = _follow(node, path, reading)
767
+ else:
768
+ value = _get_group(node, path, reading)
769
+ reading.inside.discard(node.name)
770
+ reading.done[node.name] = value
771
+ return value
772
+
773
+
774
+ def _follow(reference: h5py.Group, path: Path, reading: _Reading):
775
+ """Object `reference` refers to, read through hard links within the file."""
776
+ target = _attribute(reference, "target", path)
777
+ if not isinstance(target, str) or not target.startswith("/"):
778
+ raise ValueError(f"{path}: {reference.name} refers to nothing a results file holds.")
779
+ if target in reading.inside:
780
+ raise ValueError(f"{path}: {reference.name} refers to {target}, which it is inside.")
781
+ if target in reading.done:
782
+ return reading.done[target]
783
+ node = reference.file
784
+ for part in target[1:].split("/"):
785
+ if (not isinstance(node, h5py.Group) or not _is_name(part)
786
+ or not isinstance(node.get(part, getlink=True), h5py.HardLink)):
787
+ raise ValueError(f"{path}: {reference.name} refers to {target}, which the file "
788
+ f"does not hold.")
789
+ node = node[part]
790
+ return _read(node, path, reading)
791
+
792
+
793
+ # Filters ECLIPSE compresses arrays with; any other would make HDF5 look for
794
+ # a plugin to read them.
795
+ _FILTERS = {h5py.h5z.FILTER_DEFLATE, h5py.h5z.FILTER_SHUFFLE}
796
+ # Most bytes deflate can give back per byte stored: 258 per two-bit code.
797
+ _DEFLATE_MOST = 1032
798
+
799
+
800
+ def _check_dataset(node: h5py.Dataset, path: Path, reading: _Reading) -> None:
801
+ """
802
+ Refuse, before reading, a dataset that ECLIPSE would not have written.
803
+
804
+ What the file declares of its data is checked against what it stores,
805
+ since a crafted file could otherwise declare more than it holds, which
806
+ reading would make up, or have one stored piece read as many.
807
+ """
808
+ if node.is_virtual or node.external:
809
+ raise ValueError(f"{path}: {node.name} keeps its data in another file, which a "
810
+ f"results file does not do.")
811
+ dtype = node.dtype
812
+ if dtype.kind not in "biufc" or dtype.subdtype is not None or dtype.names is not None:
813
+ raise ValueError(f"{path}: {node.name} holds values of type {dtype}, which a results "
814
+ f"file does not.")
815
+ plist = node.id.get_create_plist()
816
+ filters = {plist.get_filter(i)[0] for i in range(plist.get_nfilters())}
817
+ if not filters <= _FILTERS:
818
+ raise ValueError(f"{path}: {node.name} is compressed in a way a results file is not.")
819
+ stored, whole = 0, True
820
+ if node.chunks is None:
821
+ stored = node.id.get_storage_size()
822
+ else:
823
+ # Every chunk of the grid must be stored, as an unwritten one would
824
+ # read as made-up values. Count them first, so a grid declared far
825
+ # larger than the file holds is not looped over.
826
+ steps = node.chunks
827
+ grid = [-(-size // step) for size, step in zip(node.shape, steps)]
828
+ whole = node.id.get_num_chunks() == math.prod(grid)
829
+ for index in np.ndindex(*grid) if whole else ():
830
+ chunk = node.id.get_chunk_info_by_coord(tuple(i * step for i, step in zip(index, steps)))
831
+ if chunk.byte_offset is None:
832
+ whole = False
833
+ break
834
+ stored += chunk.size
835
+ most = stored * (_DEFLATE_MOST if h5py.h5z.FILTER_DEFLATE in filters else 1)
836
+ if not whole or node.nbytes > most:
837
+ raise ValueError(f"{path}: {node.name} does not hold all of its data.")
838
+ reading.stored += stored
839
+ if reading.stored > reading.size:
840
+ raise ValueError(f"{path}: {node.name} shares what it stores with another dataset, "
841
+ f"which a results file does not do.")
842
+
843
+
844
+ def _describing(group: h5py.Group, name: str, path: Path):
845
+ """Attribute `name` describing `group`, which a results file always writes."""
846
+ value = _attribute(group, name, path)
847
+ if value is None:
848
+ raise ValueError(f"{path}: {group.name} has no {name!r} attribute, which a results "
849
+ f"file gives it.")
850
+ return value
851
+
852
+
853
+ def _get_group(group: h5py.Group, path: Path, reading: _Reading):
854
+ kind = _attribute(group, TYPE, path)
855
+ if kind == "ndcube":
856
+ data = _get(group, "data", path, reading)
857
+ unit = None
858
+ if isinstance(data, u.Quantity):
859
+ data, unit = data.value, data.unit
860
+ return NDCube(data, wcs=_get(group, "wcs", path, reading), unit=unit,
861
+ meta=_get(group, "meta", path, reading))
862
+ if kind == "map":
863
+ keys = _unjson(json.loads(_describing(group, KEYS, path)), reading)
864
+ return {key: _get(group, str(index), path, reading) for index, key in enumerate(keys)}
865
+ if kind in ("list", "tuple"):
866
+ length = int(_describing(group, LENGTH, path))
867
+ items = [_get(group, str(index), path, reading) for index in range(length)]
868
+ return items if kind == "list" else tuple(items)
869
+ if kind is not None:
870
+ raise ValueError(f"{path}: {group.name} is of a kind, {kind!r}, this ECLIPSE does "
871
+ f"not know.")
872
+ order = json.loads(_describing(group, ORDER, path))
873
+ members = (set(group) | set(group.attrs)) - _RESERVED
874
+ if (not isinstance(order, list) or len(set(order)) != len(order)
875
+ or set(order) != members):
876
+ raise ValueError(f"{path}: the members of {group.name} are not those it lists.")
877
+ return {key: _get(group, key, path, reading) for key in order}
878
+
879
+
880
+ # ----------------------------------------------------------------------
881
+ # Files
882
+ # ----------------------------------------------------------------------
883
+ def is_results_file(path: str | Path) -> bool:
884
+ """
885
+ Whether `path` is HDF5, as a results file is, rather than an older pickle.
886
+
887
+ :func:`load_results` checks which kind of HDF5 file it is.
888
+ """
889
+ return h5py.is_hdf5(os.fspath(Path(path).expanduser()))
890
+
891
+
892
+ def save_results(path: str | Path, payload: dict, *, compression: str | None = "gzip") -> Path:
893
+ """
894
+ Write `payload` as a results file.
895
+
896
+ Parameters
897
+ ----------
898
+ path : str or Path
899
+ Where to write. An existing file is replaced only once the new one
900
+ is complete, so a failed write leaves it unchanged. A pickle suffix
901
+ such as `.pkl` is replaced with `.h5`, since the file is not a
902
+ pickle.
903
+ payload : dict
904
+ Tree to write, keyed by strings that can name an HDF5 member.
905
+ compression : {"gzip", None}, optional
906
+ Compress the larger arrays with gzip, which any HDF5 reader can
907
+ undo; None writes them uncompressed, which is faster for a large run.
908
+
909
+ Returns
910
+ -------
911
+ Path
912
+ File written.
913
+ """
914
+ path = Path(path).expanduser()
915
+ if path.suffix.lower() in _PICKLE_SUFFIXES:
916
+ path = path.with_suffix(".h5")
917
+ warnings.warn(f"The results were written to {path.name}, an HDF5 file, rather than "
918
+ f"a pickle's name that would misdescribe it.", UserWarning, stacklevel=2)
919
+ if path.is_dir():
920
+ raise IsADirectoryError(f"{path} is a directory; name the results file to write.")
921
+ if compression not in ("gzip", None):
922
+ raise ValueError(f"compression must be 'gzip' or None, got {compression!r}.")
923
+ if not isinstance(payload, Mapping):
924
+ raise TypeError(f"The results must be a mapping, got {type(payload).__name__}.")
925
+ bad = [key for key in payload if not _is_name(key)]
926
+ if bad:
927
+ raise ValueError(f"The results cannot be keyed by {bad}: the keys have to name "
928
+ f"HDF5 members, and {sorted(_RESERVED)} are taken.")
929
+ path.parent.mkdir(parents=True, exist_ok=True)
930
+ # Write through a symlink to its target, as writing to the link would.
931
+ destination = Path(os.path.realpath(path)) if path.is_symlink() else path
932
+ # Do not replace a read-only file, as writing into it would fail.
933
+ if destination.exists() and not os.access(destination, os.W_OK):
934
+ raise PermissionError(f"{destination} is read-only, so the results are not written "
935
+ f"over it.")
936
+ partial = _new_partial(destination)
937
+ try:
938
+ with h5py.File(partial, "w", track_order=True) as f:
939
+ f.attrs["format"] = FORMAT_NAME
940
+ f.attrs["version"] = FORMAT_VERSION
941
+ _put_dict(f, dict(payload), compression, {})
942
+ os.replace(partial, destination)
943
+ except BaseException:
944
+ partial.unlink(missing_ok=True)
945
+ raise
946
+ return path
947
+
948
+
949
+ # HDF5 file suffixes. A pickle with one is not read, since a reader would
950
+ # take such a file to be safe to open.
951
+ _HDF5_SUFFIXES = (".h5", ".hdf5", ".hdf", ".he5")
952
+
953
+
954
+ def load_results(path: str | Path, *, _stacklevel: int = 2) -> dict:
955
+ """
956
+ Read a results file, or a results pickle written by an older version.
957
+
958
+ A pickle still loads, with a warning, which also mentions a later run's
959
+ `.h5` beside it. A pickle named as an HDF5 file is refused, since a
960
+ reader would take it to be safe to open, and unpickling runs whatever
961
+ code the file holds. A `.pkl` path that does not exist, as a script
962
+ written for an older version may ask for, reads the `.h5` the
963
+ simulation writes in its place, also with a warning.
964
+
965
+ Parameters
966
+ ----------
967
+ path : str or Path
968
+ File to read.
969
+
970
+ Returns
971
+ -------
972
+ dict
973
+ Stored payload.
974
+ """
975
+ path = Path(path).expanduser()
976
+ # The .h5 the simulation writes in place of a pickle of this name.
977
+ written = path.with_suffix(".h5") if path.suffix.lower() in _PICKLE_SUFFIXES else None
978
+ if written is not None and not path.exists() and written.is_file():
979
+ warnings.warn(f"{path} does not exist, so {written}, which the instrument simulation "
980
+ f"now writes in its place, is read instead. Name the .h5 file to read it "
981
+ f"without this warning.", FutureWarning, stacklevel=_stacklevel)
982
+ path, written = written, None
983
+
984
+ if path.is_dir():
985
+ raise IsADirectoryError(f"{path} is a directory; name the results file to read.")
986
+ if not path.is_file():
987
+ if path.suffix.lower() in _HDF5_SUFFIXES:
988
+ for older in (path.with_suffix(suffix) for suffix in _PICKLE_SUFFIXES):
989
+ if older.is_file():
990
+ raise FileNotFoundError(
991
+ f"No results file {path}. {older}, beside it, is a results pickle from "
992
+ f"an older version of ECLIPSE: load_instrument_response_results reads "
993
+ f"it, and eclipse.convert_results_pickle rewrites it as a "
994
+ f"results file.")
995
+ raise FileNotFoundError(f"No results file {path}.")
996
+ if not is_results_file(path):
997
+ if not _is_pickle(path):
998
+ raise ValueError(f"{path} is neither a results file nor a results pickle.")
999
+ if path.suffix.lower() in _HDF5_SUFFIXES:
1000
+ raise ValueError(f"{path} is named as an HDF5 file but is a pickle, which is not "
1001
+ f"read under such a name, since unpickling runs whatever code the "
1002
+ f"file holds: rename it to .pkl if it is a results pickle you "
1003
+ f"trust.")
1004
+ later = ""
1005
+ if (written is not None and written.is_file()
1006
+ and written.stat().st_mtime > path.stat().st_mtime):
1007
+ later = (f" {written}, beside it, is newer: the results of a later run, as the "
1008
+ f"simulation now writes them.")
1009
+ warnings.warn(
1010
+ f"{path} is a results pickle, as older versions of ECLIPSE wrote them. Pickles "
1011
+ f"are deprecated and will not be read in a future release: convert it with "
1012
+ f"eclipse.convert_results_pickle, or re-run the simulation. Reading a "
1013
+ f"pickle runs whatever code it holds, so only read files you trust.{later}",
1014
+ FutureWarning, stacklevel=_stacklevel)
1015
+ payload = _load_pickle(path)
1016
+ _give_earlier_values(payload)
1017
+ _move_old_tables(payload)
1018
+ if _from_before_weighting(payload):
1019
+ warnings.warn(_BEFORE_WEIGHTING.format(path=path), UserWarning,
1020
+ stacklevel=_stacklevel)
1021
+ return payload
1022
+
1023
+ # A damaged or hand-made file can fail anywhere in the reading. Errors are
1024
+ # raised again naming the file, as an OSError if the filesystem fails,
1025
+ # except a MemoryError.
1026
+ reading = _Reading(size=path.stat().st_size)
1027
+ try:
1028
+ with h5py.File(path, "r") as f:
1029
+ # Check these like any other attribute before the format check reads them.
1030
+ for name in ("format", "version"):
1031
+ _attribute(f, name, path)
1032
+ _check_format(f, path, kind="results", format_name=FORMAT_NAME,
1033
+ format_version=FORMAT_VERSION, documented=False)
1034
+ results = _get_group(f, path, reading)
1035
+ except MemoryError:
1036
+ raise
1037
+ except OSError as error:
1038
+ if str(path) in str(error):
1039
+ raise
1040
+ raise OSError(f"{path} could not be read: {error}") from error
1041
+ except ValueError as error:
1042
+ if str(error).startswith(str(path)):
1043
+ raise
1044
+ raise ValueError(f"{path} is not a results file this ECLIPSE can read: "
1045
+ f"{error}") from error
1046
+ except Exception as error:
1047
+ raise ValueError(f"{path} is not a results file this ECLIPSE can read: "
1048
+ f"{type(error).__name__}: {error}") from error
1049
+ # Give each warning from the reading once, as the caller's, such as for a
1050
+ # configuration object this version would not make.
1051
+ for message in reading.notes:
1052
+ warnings.warn(message, UserWarning, stacklevel=_stacklevel)
1053
+ if not isinstance(results, dict):
1054
+ raise ValueError(f"{path} is not a results file this ECLIPSE can read: it holds a "
1055
+ f"{type(results).__name__}, not a mapping.")
1056
+ if _from_before_weighting(results):
1057
+ warnings.warn(_BEFORE_WEIGHTING.format(path=path), UserWarning, stacklevel=_stacklevel)
1058
+ return results
1059
+
1060
+
1061
+ _BEFORE_WEIGHTING = ("The results in {path} were made before ECLIPSE weighted its fits by "
1062
+ "each pixel's uncertainty, so their fits are unweighted; fit_weighted is "
1063
+ "set to False.")
1064
+
1065
+
1066
+ def _from_before_weighting(payload) -> bool:
1067
+ """
1068
+ Whether the results were made before fits could be weighted.
1069
+
1070
+ If so, their fits were unweighted, so `fit_weighted` is set to False. So
1071
+ is `weighted` of a pickled FitConfig, which lacks it and would otherwise
1072
+ read the class default.
1073
+ """
1074
+ results = payload.get("results") if isinstance(payload, dict) else None
1075
+ if not isinstance(results, dict) or "all_combinations" not in results:
1076
+ return False
1077
+ if "fit_weighted" in results:
1078
+ return False
1079
+ results["fit_weighted"] = False
1080
+ own = getattr(results.get("fit_config"), "__dict__", None)
1081
+ if own is not None and "weighted" not in own:
1082
+ object.__setattr__(results["fit_config"], "weighted", False)
1083
+ return True
1084
+
1085
+
1086
+ def _configuration_objects(value, seen=None):
1087
+ """Each configuration object in an unpickled payload, however deeply it is held."""
1088
+ seen = set() if seen is None else seen
1089
+ if id(value) in seen:
1090
+ return
1091
+ seen.add(id(value))
1092
+ if isinstance(value, dict):
1093
+ items = list(value.values())
1094
+ elif isinstance(value, (list, tuple)):
1095
+ items = list(value)
1096
+ elif dataclasses.is_dataclass(value) and not isinstance(value, type):
1097
+ own = getattr(value, "__dict__", None)
1098
+ if own is None:
1099
+ return
1100
+ yield value
1101
+ items = list(own.values())
1102
+ else:
1103
+ return
1104
+ for item in items:
1105
+ yield from _configuration_objects(item, seen)
1106
+
1107
+
1108
+ def _give_earlier_values(payload) -> None:
1109
+ """
1110
+ Give each configuration object in an unpickled payload the `_EARLIER_VALUES` it lacks.
1111
+
1112
+ An object pickled before a setting existed would otherwise read the
1113
+ class default, such as a telescope's `resolution_budget`, which runs
1114
+ then did not have.
1115
+ """
1116
+ for value in _configuration_objects(payload):
1117
+ own = value.__dict__
1118
+ for name, item in _EARLIER_VALUES.get(type(value).__name__, {}).items():
1119
+ if name not in own:
1120
+ object.__setattr__(value, name, item)
1121
+
1122
+
1123
+ def _move_old_tables(payload) -> None:
1124
+ """
1125
+ Point missing packaged tables named under the former name at this installation's copies.
1126
+
1127
+ Changes the configuration objects of an unpickled payload in place; see
1128
+ :func:`_moved_package_file`.
1129
+ """
1130
+ for value in _configuration_objects(payload):
1131
+ for name, item in list(value.__dict__.items()):
1132
+ if isinstance(item, Path) or hasattr(item, "__fspath__"):
1133
+ moved = _moved_package_file(item)
1134
+ if moved is not None:
1135
+ object.__setattr__(value, name, moved)
1136
+
1137
+
1138
+ def _load_pickle(path: Path) -> dict:
1139
+ from .utils import _unpickle
1140
+
1141
+ with open(path, "rb") as handle:
1142
+ try:
1143
+ return _unpickle(handle)
1144
+ # Such as a pickle of a class only another version or branch of ECLIPSE had.
1145
+ except (AttributeError, ImportError) as error:
1146
+ raise ValueError(f"{path} cannot be unpickled by this version of ECLIPSE: "
1147
+ f"{error}") from error
1148
+
1149
+
1150
+ def convert_results_pickle(pickle_path: str | Path, path: str | Path | None = None,
1151
+ overwrite: bool = False) -> Path:
1152
+ """
1153
+ Rewrite a results pickle from ECLIPSE 0.11.0 and earlier as a results file.
1154
+
1155
+ Everything in the pickle is kept, and the new file is read back as a
1156
+ check. A setting that did not exist when the pickle was made gets
1157
+ today's default, with a warning, unless runs then had another value:
1158
+ their fits were unweighted, so the file records `fit_weighted` False,
1159
+ and `weighted` False for a FitConfig in it. Missing packaged tables
1160
+ named in the former folder `euvst_response/data` are recorded as this
1161
+ installation's copies. A pickle holding no results, such as a synthesis
1162
+ pickle, is refused. Reading a pickle runs
1163
+ whatever code it holds, so only convert files you trust.
1164
+
1165
+ Parameters
1166
+ ----------
1167
+ pickle_path : str or Path
1168
+ Pickle to convert.
1169
+ path : str or Path, optional
1170
+ Results file to write. Default None, which writes it beside the
1171
+ pickle, with the same name ending in `.h5`.
1172
+ overwrite : bool, optional
1173
+ Replace a file already at `path`, which is otherwise refused.
1174
+ Default False.
1175
+
1176
+ Returns
1177
+ -------
1178
+ Path
1179
+ File written.
1180
+ """
1181
+ pickle_path = Path(pickle_path).expanduser()
1182
+ if is_results_file(pickle_path):
1183
+ raise ValueError(f"{pickle_path} is already an HDF5 file.")
1184
+ if not _is_pickle(pickle_path):
1185
+ raise ValueError(f"{pickle_path} is not a pickle.")
1186
+ target = pickle_path.with_suffix(".h5") if path is None else Path(path).expanduser()
1187
+ if target.suffix.lower() in _PICKLE_SUFFIXES:
1188
+ target = target.with_suffix(".h5")
1189
+ if target.is_dir():
1190
+ raise IsADirectoryError(f"{target} is a directory; name the results file to write.")
1191
+ if target.resolve() == pickle_path.resolve():
1192
+ raise ValueError(f"{target} is the pickle itself; name another file to write.")
1193
+ if target.exists() and not overwrite:
1194
+ raise FileExistsError(f"{target} exists; pass overwrite=True to replace it.")
1195
+ # Write through a symlink to its target, as writing to the link would.
1196
+ requested = target
1197
+ if target.is_symlink():
1198
+ target = Path(os.path.realpath(target))
1199
+ payload = _load_pickle(pickle_path)
1200
+ results = payload.get("results") if isinstance(payload, dict) else None
1201
+ if not isinstance(results, dict) or "all_combinations" not in results:
1202
+ raise ValueError(f"{pickle_path} holds no results. A synthesis pickle converts with "
1203
+ f"eclipse.convert_synthesis_pickle.")
1204
+ _move_old_tables(payload)
1205
+ # Recorded in the new file, so reading it does not warn again.
1206
+ if _from_before_weighting(payload):
1207
+ warnings.warn(_BEFORE_WEIGHTING.format(path=pickle_path), UserWarning, stacklevel=2)
1208
+ # Write under a temporary name and read it back before replacing the
1209
+ # target, so the result is never an unreadable file or a lost one.
1210
+ target.parent.mkdir(parents=True, exist_ok=True)
1211
+ staging = _new_partial(target)
1212
+ try:
1213
+ save_results(staging, payload)
1214
+ del payload, results
1215
+ try:
1216
+ load_results(staging, _stacklevel=3)
1217
+ except (MemoryError, OSError):
1218
+ raise
1219
+ except Exception as error:
1220
+ raise ValueError(f"{pickle_path} converts to a file that cannot be read back, "
1221
+ f"so {target} is not written: {error}") from error
1222
+ os.replace(staging, target)
1223
+ except BaseException:
1224
+ staging.unlink(missing_ok=True)
1225
+ raise
1226
+ return requested