ctao-calibpipe 0.1.0rc8__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ctao-calibpipe might be problematic. Click here for more details.
- calibpipe/_version.py +2 -2
- calibpipe/core/common_metadata_containers.py +3 -0
- calibpipe/database/adapter/adapter.py +1 -1
- calibpipe/database/adapter/database_containers/__init__.py +2 -0
- calibpipe/database/adapter/database_containers/common_metadata.py +2 -0
- calibpipe/database/adapter/database_containers/throughput.py +30 -0
- calibpipe/database/interfaces/table_handler.py +79 -97
- calibpipe/telescope/throughput/containers.py +59 -0
- calibpipe/tests/unittests/array/test_cross_calibration.py +417 -0
- calibpipe/tests/unittests/database/test_table_handler.py +95 -0
- calibpipe/tests/unittests/telescope/camera/test_calculate_camcalib_coefficients.py +347 -0
- calibpipe/tests/unittests/telescope/camera/test_produce_camcalib_test_data.py +42 -0
- calibpipe/tests/unittests/telescope/throughput/test_muon_throughput_calibrator.py +189 -0
- calibpipe/tools/camcalib_test_data.py +361 -0
- calibpipe/tools/camera_calibrator.py +558 -0
- calibpipe/tools/muon_throughput_calculator.py +239 -0
- calibpipe/tools/telescope_cross_calibration_calculator.py +721 -0
- {ctao_calibpipe-0.1.0rc8.dist-info → ctao_calibpipe-0.2.0.dist-info}/METADATA +3 -2
- {ctao_calibpipe-0.1.0rc8.dist-info → ctao_calibpipe-0.2.0.dist-info}/RECORD +24 -14
- {ctao_calibpipe-0.1.0rc8.dist-info → ctao_calibpipe-0.2.0.dist-info}/WHEEL +1 -1
- {ctao_calibpipe-0.1.0rc8.dist-info → ctao_calibpipe-0.2.0.dist-info}/entry_points.txt +4 -0
- {ctao_calibpipe-0.1.0rc8.dist-info → ctao_calibpipe-0.2.0.dist-info}/licenses/AUTHORS.md +0 -0
- {ctao_calibpipe-0.1.0rc8.dist-info → ctao_calibpipe-0.2.0.dist-info}/licenses/LICENSE +0 -0
- {ctao_calibpipe-0.1.0rc8.dist-info → ctao_calibpipe-0.2.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,721 @@
|
|
|
1
|
+
"""Calculate the relative throughput of the telescopes."""
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from itertools import combinations # noqa: D100
|
|
5
|
+
|
|
6
|
+
import astropy.units as u
|
|
7
|
+
import numpy as np
|
|
8
|
+
from astropy.table import Table, join
|
|
9
|
+
from ctapipe.core import (
|
|
10
|
+
TelescopeComponent,
|
|
11
|
+
Tool,
|
|
12
|
+
)
|
|
13
|
+
from ctapipe.core.traits import (
|
|
14
|
+
AstroQuantity,
|
|
15
|
+
Bool,
|
|
16
|
+
Dict,
|
|
17
|
+
Float,
|
|
18
|
+
FloatTelescopeParameter,
|
|
19
|
+
IntTelescopeParameter,
|
|
20
|
+
Path,
|
|
21
|
+
Unicode,
|
|
22
|
+
)
|
|
23
|
+
from ctapipe.instrument import SubarrayDescription
|
|
24
|
+
from ctapipe.io import read_table, write_table
|
|
25
|
+
from iminuit import Minuit
|
|
26
|
+
from scipy.stats import norm
|
|
27
|
+
from tables.exceptions import NoSuchNodeError
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class RelativeThroughputFitter(TelescopeComponent):
|
|
31
|
+
"""Perform relative throughput fitting for telescopes.
|
|
32
|
+
|
|
33
|
+
This component is used to fit the relative throughput of telescopes based on the
|
|
34
|
+
energy asymmetry of telescope pairs.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
reference_telescopes = IntTelescopeParameter(
|
|
38
|
+
default_value=None,
|
|
39
|
+
help="ID of the telescopes whose throughput kept fixed during the intercalibration minimization",
|
|
40
|
+
allow_none=True,
|
|
41
|
+
).tag(config=True)
|
|
42
|
+
throughput_normalization = FloatTelescopeParameter(
|
|
43
|
+
default_value=1.0,
|
|
44
|
+
help="Setting the overall telescope throughput normalization. "
|
|
45
|
+
"Depending the use case, it could reflect the "
|
|
46
|
+
"absolute optical throughput measured by the muon rings / illuminator, "
|
|
47
|
+
"if we want to compare or complement these methods. "
|
|
48
|
+
"Alternatively it could get an arbitrary number, e.g. one that sets the average throughput to 1, "
|
|
49
|
+
"if the user wants to 'flat-field' the array. "
|
|
50
|
+
"Finally it could be set to 1, if we want to identify outlier telescopes or study the aging.",
|
|
51
|
+
allow_none=False,
|
|
52
|
+
).tag(config=True)
|
|
53
|
+
|
|
54
|
+
def fit(self, subarray_name, subarray_data):
|
|
55
|
+
"""
|
|
56
|
+
Perform minimization for each telescope subsystem to compute optical throughput factors.
|
|
57
|
+
|
|
58
|
+
Parameters
|
|
59
|
+
----------
|
|
60
|
+
subarray_data : dict
|
|
61
|
+
A dictionary where keys are telescope pairs and values are dictionaries containing
|
|
62
|
+
"mean_asymmetry": the average energy asymmetry and "mean_uncertainty": the average
|
|
63
|
+
uncertainty of the asymmetry for each telescope pair.
|
|
64
|
+
subarray_name : str
|
|
65
|
+
The name of the intercalibration subarray, used to identify the telescopes.
|
|
66
|
+
Usually corresponds to the telescope type (e.g., "LST", "MST", "SST").
|
|
67
|
+
|
|
68
|
+
Returns
|
|
69
|
+
-------
|
|
70
|
+
dict
|
|
71
|
+
Dictionary of the form:
|
|
72
|
+
{
|
|
73
|
+
'subarray_name': {tel_id: (value, error), ...}
|
|
74
|
+
}
|
|
75
|
+
"""
|
|
76
|
+
results = {}
|
|
77
|
+
|
|
78
|
+
tel_ids = self.subarray.tel_ids
|
|
79
|
+
|
|
80
|
+
initial_values = {
|
|
81
|
+
str(tel_id): self.throughput_normalization.tel[tel_id] for tel_id in tel_ids
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
def _chi2(*values):
|
|
85
|
+
params = dict(zip(tel_ids, values))
|
|
86
|
+
chi2 = 0.0
|
|
87
|
+
for (i, j), entry in subarray_data.items():
|
|
88
|
+
a_ij = entry["mean_asymmetry"]
|
|
89
|
+
s_ij = entry["mean_uncertainty"]
|
|
90
|
+
ci = params[i]
|
|
91
|
+
cj = params[j]
|
|
92
|
+
model = (ci - cj) / (ci + cj)
|
|
93
|
+
chi2 += ((a_ij - model) ** 2) / (s_ij**2)
|
|
94
|
+
return chi2
|
|
95
|
+
|
|
96
|
+
fit = Minuit(_chi2, name=initial_values.keys(), **initial_values)
|
|
97
|
+
|
|
98
|
+
# Create a list of unique reference telescope IDs
|
|
99
|
+
unique_reference_telescopes = set(
|
|
100
|
+
self.reference_telescopes.tel[tel_id]
|
|
101
|
+
for tel_id in tel_ids
|
|
102
|
+
if self.reference_telescopes.tel[tel_id] is not None
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# Pass each unique reference telescope ID to fit.fixed
|
|
106
|
+
for ref_tel_id in unique_reference_telescopes:
|
|
107
|
+
fit.fixed[str(ref_tel_id)] = True
|
|
108
|
+
|
|
109
|
+
fit.errordef = 1
|
|
110
|
+
fit.migrad()
|
|
111
|
+
|
|
112
|
+
results[subarray_name] = {
|
|
113
|
+
tel_id: (fit.values[tel_id], fit.errors[tel_id])
|
|
114
|
+
for tel_id in initial_values.keys()
|
|
115
|
+
}
|
|
116
|
+
return results
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class PairFinder(TelescopeComponent):
|
|
120
|
+
"""Find pairs of telescopes based on their types and distances between them."""
|
|
121
|
+
|
|
122
|
+
max_impact_distance = FloatTelescopeParameter(
|
|
123
|
+
default_value=[
|
|
124
|
+
("type", "LST*", 125.0),
|
|
125
|
+
("type", "MST*", 125.0),
|
|
126
|
+
("type", "SST*", 225.0),
|
|
127
|
+
],
|
|
128
|
+
help="Maximum distance between the telescopes and a shower core in meters. "
|
|
129
|
+
"The maximum distance between the telescopes in pair "
|
|
130
|
+
"should not exceed the sum of these values for the telescopes in pair.",
|
|
131
|
+
allow_none=False,
|
|
132
|
+
).tag(config=True)
|
|
133
|
+
|
|
134
|
+
def find_pairs(self, by_tel_type=True, cross_type_only=False):
|
|
135
|
+
"""
|
|
136
|
+
Find pairs of telescope IDs.
|
|
137
|
+
|
|
138
|
+
Parameters
|
|
139
|
+
----------
|
|
140
|
+
by_tel_type : bool
|
|
141
|
+
If True, find pairs of telescopes of the same type.
|
|
142
|
+
If False, find pairs of all telescopes in the array.
|
|
143
|
+
cross_type_only : bool
|
|
144
|
+
If True, find pairs of telescopes of different types only.
|
|
145
|
+
|
|
146
|
+
Returns
|
|
147
|
+
-------
|
|
148
|
+
dict
|
|
149
|
+
A dictionary where keys are telescope types (str) and values are sets of
|
|
150
|
+
pairs of telescope IDs (int, int). If by_tel_type is False, the key will be "ALL".
|
|
151
|
+
If cross_type_only is True, the key will be "XTEL".
|
|
152
|
+
"""
|
|
153
|
+
result = defaultdict(set)
|
|
154
|
+
telescope_positions = self.subarray.positions
|
|
155
|
+
|
|
156
|
+
def check_distance(tel1, tel2):
|
|
157
|
+
"""Check if the pair of telescopes is within the maximum distance."""
|
|
158
|
+
pos1 = telescope_positions[tel1]
|
|
159
|
+
pos2 = telescope_positions[tel2]
|
|
160
|
+
distance = np.linalg.norm((pos1 - pos2).to(u.m).value)
|
|
161
|
+
return (
|
|
162
|
+
distance
|
|
163
|
+
<= self.max_impact_distance.tel[tel1]
|
|
164
|
+
+ self.max_impact_distance.tel[tel2]
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
def check_type(tel1, tel2):
|
|
168
|
+
"""Check if the pair of telescopes are of the same type."""
|
|
169
|
+
return self.subarray.tel[tel1].type == self.subarray.tel[tel2].type
|
|
170
|
+
|
|
171
|
+
telescope_types = defaultdict(list)
|
|
172
|
+
if by_tel_type:
|
|
173
|
+
for tel_id, desc in self.subarray.tel.items():
|
|
174
|
+
telescope_types[desc.type].append(tel_id)
|
|
175
|
+
else:
|
|
176
|
+
if cross_type_only:
|
|
177
|
+
telescope_types["XTEL"] = list(self.subarray.tel_ids)
|
|
178
|
+
else:
|
|
179
|
+
telescope_types["ALL"] = list(self.subarray.tel_ids)
|
|
180
|
+
|
|
181
|
+
for tel_type, tel_ids in telescope_types.items():
|
|
182
|
+
for tel1, tel2 in combinations(tel_ids, 2):
|
|
183
|
+
if cross_type_only and check_type(tel1, tel2):
|
|
184
|
+
continue
|
|
185
|
+
if check_distance(tel1, tel2):
|
|
186
|
+
result[tel_type].add((tel1, tel2))
|
|
187
|
+
|
|
188
|
+
return result
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class CalculateCrossCalibration(Tool):
|
|
192
|
+
"""Calibrator that performs cross calibration of telescopes."""
|
|
193
|
+
|
|
194
|
+
input_url = Path(
|
|
195
|
+
default_value="dl2.h5",
|
|
196
|
+
help="Path to the input file with the DL2 (merged) data.",
|
|
197
|
+
).tag(config=True)
|
|
198
|
+
output_url = Path(
|
|
199
|
+
default_value="monitoring_cross_calibration_dl2.h5",
|
|
200
|
+
help="Path to the output file where the produced calibration"
|
|
201
|
+
"products will be stored",
|
|
202
|
+
).tag(config=True)
|
|
203
|
+
overwrite = Bool(default_value=False, help="Overwrite output file.").tag(
|
|
204
|
+
config=True
|
|
205
|
+
)
|
|
206
|
+
reconstruction_algorithm = Unicode(
|
|
207
|
+
default_value="RandomForest",
|
|
208
|
+
help="Name of the reconstruction algorithm",
|
|
209
|
+
allow_none=False,
|
|
210
|
+
).tag(config=True)
|
|
211
|
+
event_filters = Dict(
|
|
212
|
+
per_key_traits={
|
|
213
|
+
"min_gammaness": Float(allow_none=True),
|
|
214
|
+
"min_energy": AstroQuantity(
|
|
215
|
+
physical_type=u.physical.energy, allow_none=True
|
|
216
|
+
),
|
|
217
|
+
"max_distance_asymmetry": Float(allow_none=True),
|
|
218
|
+
},
|
|
219
|
+
default_value={
|
|
220
|
+
"min_gammaness": None,
|
|
221
|
+
"min_energy": None,
|
|
222
|
+
"max_distance_asymmetry": None,
|
|
223
|
+
},
|
|
224
|
+
help=(
|
|
225
|
+
"Dictionary of event filters:\n"
|
|
226
|
+
" - min_gammaness is used to select the gamma-like events\n"
|
|
227
|
+
" - min_energy defines the energy range suitable for different types of telescopes\n"
|
|
228
|
+
" - max_distance_asymmetry sets the equidistance asymmetry limit\n"
|
|
229
|
+
"If a filter parameter is set to None (the default value), "
|
|
230
|
+
"it will not be applied during event selection."
|
|
231
|
+
),
|
|
232
|
+
).tag(config=True)
|
|
233
|
+
|
|
234
|
+
aliases = {
|
|
235
|
+
("i", "input_url"): "CalculateCrossCalibration.input_url",
|
|
236
|
+
("o", "output_url"): "CalculateCrossCalibration.output_url",
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
classes = [RelativeThroughputFitter, PairFinder]
|
|
240
|
+
|
|
241
|
+
def merge_tables(self, telescope_pairs):
|
|
242
|
+
"""
|
|
243
|
+
Merge telescope energy tables for specified telescope pairs.
|
|
244
|
+
|
|
245
|
+
Parameters
|
|
246
|
+
----------
|
|
247
|
+
telescope_pairs : dict
|
|
248
|
+
A dictionary where keys are telescope types (str), and values are lists of
|
|
249
|
+
tuples. Each tuple contains two telescope IDs (int) representing a pair.
|
|
250
|
+
|
|
251
|
+
Returns
|
|
252
|
+
-------
|
|
253
|
+
dict
|
|
254
|
+
A dictionary where keys are tuples of telescope IDs (int, int),
|
|
255
|
+
representing the telescope pairs. The values are lists of merged tables
|
|
256
|
+
for the specified telescope pairs. If no tables are successfully merged
|
|
257
|
+
for a telescope pair, the list will be empty.
|
|
258
|
+
"""
|
|
259
|
+
merged_tables = {}
|
|
260
|
+
unique_telescope_ids = {
|
|
261
|
+
tel_id
|
|
262
|
+
for pairs in telescope_pairs.values()
|
|
263
|
+
for pair in pairs
|
|
264
|
+
for tel_id in pair
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
telescope_tables = {}
|
|
268
|
+
for tel_id in unique_telescope_ids:
|
|
269
|
+
try:
|
|
270
|
+
path = f"dl2/event/telescope/energy/{self.reconstruction_algorithm}Regressor/tel_{tel_id:03d}"
|
|
271
|
+
telescope_tables[tel_id] = read_table(
|
|
272
|
+
self.input_url,
|
|
273
|
+
path,
|
|
274
|
+
condition=f"{self.reconstruction_algorithm}Regressor_tel_is_valid",
|
|
275
|
+
)
|
|
276
|
+
except NoSuchNodeError:
|
|
277
|
+
pass # data for one telescope is missing, will be reported in application to a particular pair later.
|
|
278
|
+
for telescope_type, pairs in telescope_pairs.items():
|
|
279
|
+
if telescope_type not in merged_tables:
|
|
280
|
+
merged_tables[telescope_type] = {}
|
|
281
|
+
for tel1, tel2 in pairs:
|
|
282
|
+
table_1 = telescope_tables.get(tel1)
|
|
283
|
+
table_2 = telescope_tables.get(tel2)
|
|
284
|
+
if table_1 is None or table_2 is None:
|
|
285
|
+
self.log.warning(
|
|
286
|
+
"Missing telescope data for tel %s in pair (%s, %s)",
|
|
287
|
+
tel1 if table_1 is None else tel2,
|
|
288
|
+
tel1,
|
|
289
|
+
tel2,
|
|
290
|
+
)
|
|
291
|
+
continue
|
|
292
|
+
|
|
293
|
+
try:
|
|
294
|
+
merged_table = join(
|
|
295
|
+
table_1[
|
|
296
|
+
[
|
|
297
|
+
"obs_id",
|
|
298
|
+
"event_id",
|
|
299
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy",
|
|
300
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert",
|
|
301
|
+
]
|
|
302
|
+
],
|
|
303
|
+
table_2[
|
|
304
|
+
[
|
|
305
|
+
"obs_id",
|
|
306
|
+
"event_id",
|
|
307
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy",
|
|
308
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert",
|
|
309
|
+
]
|
|
310
|
+
],
|
|
311
|
+
keys=["obs_id", "event_id"],
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
if len(merged_table) > 0:
|
|
315
|
+
pair = (tel1, tel2)
|
|
316
|
+
if pair not in merged_tables[telescope_type]:
|
|
317
|
+
merged_tables[telescope_type][pair] = []
|
|
318
|
+
|
|
319
|
+
merged_tables[telescope_type][pair].append(merged_table)
|
|
320
|
+
except MemoryError as e:
|
|
321
|
+
self.log.error(
|
|
322
|
+
"MemoryError: The data is too large to process: %s", e
|
|
323
|
+
)
|
|
324
|
+
raise
|
|
325
|
+
|
|
326
|
+
return merged_tables
|
|
327
|
+
|
|
328
|
+
def _apply_min_gammaness(self, tel_id1, tel_id2, threshold):
|
|
329
|
+
"""Select gamma-like events based on the gammaness threshold."""
|
|
330
|
+
events_tel1 = self._get_gamma_like_events(tel_id1, threshold)
|
|
331
|
+
events_tel2 = self._get_gamma_like_events(tel_id2, threshold)
|
|
332
|
+
return events_tel1 & events_tel2
|
|
333
|
+
|
|
334
|
+
def _apply_min_energy(self, tel_id1, tel_id2, threshold):
|
|
335
|
+
"""Select showers over an energy threshold."""
|
|
336
|
+
events_tel1 = self._set_energy_threshold(tel_id1, threshold)
|
|
337
|
+
events_tel2 = self._set_energy_threshold(tel_id2, threshold)
|
|
338
|
+
return events_tel1 & events_tel2
|
|
339
|
+
|
|
340
|
+
def _apply_max_distance_asymmetry(self, tel_id1, tel_id2, threshold):
|
|
341
|
+
"""Select showers with similar impact distances."""
|
|
342
|
+
|
|
343
|
+
def load_distance_table(tel_id):
|
|
344
|
+
path = f"dl2/event/telescope/impact/HillasReconstructor/tel_{tel_id:03d}"
|
|
345
|
+
return read_table(self.input_url, path)[
|
|
346
|
+
["obs_id", "event_id", "HillasReconstructor_tel_impact_distance"]
|
|
347
|
+
]
|
|
348
|
+
|
|
349
|
+
table1 = load_distance_table(tel_id1)
|
|
350
|
+
table2 = load_distance_table(tel_id2)
|
|
351
|
+
|
|
352
|
+
merged = join(table1, table2, keys=["obs_id", "event_id"])
|
|
353
|
+
dist1 = merged["HillasReconstructor_tel_impact_distance_1"]
|
|
354
|
+
dist2 = merged["HillasReconstructor_tel_impact_distance_2"]
|
|
355
|
+
|
|
356
|
+
asym = ((dist2 - dist1) / (dist1 + dist2)).value
|
|
357
|
+
mask = np.abs(asym) < threshold
|
|
358
|
+
return set(zip(merged["obs_id"][mask], merged["event_id"][mask]))
|
|
359
|
+
|
|
360
|
+
def _get_gamma_like_events(self, tel_id, threshold):
|
|
361
|
+
"""Select showers that are gamma like."""
|
|
362
|
+
path = f"dl2/event/telescope/classification/{self.reconstruction_algorithm}Classifier/tel_{tel_id:03d}"
|
|
363
|
+
class_table = read_table(
|
|
364
|
+
self.input_url,
|
|
365
|
+
path,
|
|
366
|
+
condition=f"{self.reconstruction_algorithm}Classifier_tel_is_valid",
|
|
367
|
+
)
|
|
368
|
+
mask = (
|
|
369
|
+
class_table[f"{self.reconstruction_algorithm}Classifier_tel_prediction"]
|
|
370
|
+
> threshold
|
|
371
|
+
)
|
|
372
|
+
return set(zip(class_table["obs_id"][mask], class_table["event_id"][mask]))
|
|
373
|
+
|
|
374
|
+
def _set_energy_threshold(self, tel_id, threshold):
|
|
375
|
+
"""Select showers over an energy threshold."""
|
|
376
|
+
path = f"dl2/event/telescope/energy/{self.reconstruction_algorithm}Regressor/tel_{tel_id:03d}"
|
|
377
|
+
energy_table = read_table(
|
|
378
|
+
self.input_url,
|
|
379
|
+
path,
|
|
380
|
+
condition=f"{self.reconstruction_algorithm}Regressor_tel_is_valid",
|
|
381
|
+
)
|
|
382
|
+
mask = (
|
|
383
|
+
energy_table[f"{self.reconstruction_algorithm}Regressor_tel_energy"]
|
|
384
|
+
> threshold
|
|
385
|
+
)
|
|
386
|
+
return set(zip(energy_table["obs_id"][mask], energy_table["event_id"][mask]))
|
|
387
|
+
|
|
388
|
+
def event_selection(self, merged_table, tel1, tel2):
|
|
389
|
+
"""
|
|
390
|
+
Filter merged table based on energy, classification, and optionally equidistant impact criteria.
|
|
391
|
+
|
|
392
|
+
Parameters
|
|
393
|
+
----------
|
|
394
|
+
merged_table : Table
|
|
395
|
+
Merged table from two telescopes.
|
|
396
|
+
tel1, tel2 : int
|
|
397
|
+
Telescope IDs.
|
|
398
|
+
|
|
399
|
+
Returns
|
|
400
|
+
-------
|
|
401
|
+
Table
|
|
402
|
+
Filtered table.
|
|
403
|
+
"""
|
|
404
|
+
selected_events = self._get_valid_events(tel1, tel2)
|
|
405
|
+
|
|
406
|
+
event_ids = set(zip(merged_table["obs_id"], merged_table["event_id"]))
|
|
407
|
+
valid_mask = np.array(
|
|
408
|
+
[(obs_id, event_id) in selected_events for obs_id, event_id in event_ids]
|
|
409
|
+
)
|
|
410
|
+
return merged_table[valid_mask]
|
|
411
|
+
|
|
412
|
+
def _get_valid_events(self, tel_id1, tel_id2):
|
|
413
|
+
"""Apply all event filters for a given telescope."""
|
|
414
|
+
event_sets = []
|
|
415
|
+
for filter_name, threshold in self.event_filters.items():
|
|
416
|
+
if threshold is None:
|
|
417
|
+
self.log.debug("Skipping filter %s", filter_name)
|
|
418
|
+
continue
|
|
419
|
+
try:
|
|
420
|
+
filter_func = getattr(self, f"_apply_{filter_name}")
|
|
421
|
+
except AttributeError:
|
|
422
|
+
self.log.error(
|
|
423
|
+
"Filter function _apply_%s not found for filter %s",
|
|
424
|
+
filter_name,
|
|
425
|
+
filter_name,
|
|
426
|
+
)
|
|
427
|
+
raise ValueError(
|
|
428
|
+
f"Filter {filter_name} is not implemented or not recognized."
|
|
429
|
+
)
|
|
430
|
+
event_sets.append(filter_func(tel_id1, tel_id2, threshold))
|
|
431
|
+
|
|
432
|
+
return set.intersection(*event_sets) if event_sets else set()
|
|
433
|
+
|
|
434
|
+
def calculate_energy_asymmetry(self, merged_tables):
|
|
435
|
+
"""
|
|
436
|
+
Calculate the mean energy asymmetry and its uncertainty for each telescope pair from the merged tables.
|
|
437
|
+
|
|
438
|
+
Parameters
|
|
439
|
+
----------
|
|
440
|
+
merged_tables : dict
|
|
441
|
+
A dictionary where keys are telescope types. Each value is another dictionary where keys are tuples
|
|
442
|
+
of telescope IDs (int, int), representing telescope pairs, and the values are lists of merged tables
|
|
443
|
+
containing energy data.
|
|
444
|
+
|
|
445
|
+
Returns
|
|
446
|
+
-------
|
|
447
|
+
dict
|
|
448
|
+
A nested dictionary with telescope types as the first level of keys.
|
|
449
|
+
Each value is a dictionary with telescope pairs as keys and a dictionary as value containing:
|
|
450
|
+
- "mean_asymmetry": the average energy asymmetry
|
|
451
|
+
- "mean_uncertainty": the average uncertainty of the asymmetry
|
|
452
|
+
"""
|
|
453
|
+
energy_asymmetry_results = {}
|
|
454
|
+
|
|
455
|
+
for telescope_type, pairs in merged_tables.items():
|
|
456
|
+
energy_asymmetry_results[telescope_type] = {}
|
|
457
|
+
for pair, merged_tables_list in pairs.items():
|
|
458
|
+
asymmetry_values = []
|
|
459
|
+
uncertainty_values = []
|
|
460
|
+
for merged_table_unfiltered in merged_tables_list:
|
|
461
|
+
merged_table = self.event_selection(
|
|
462
|
+
merged_table_unfiltered,
|
|
463
|
+
pair[0],
|
|
464
|
+
pair[1],
|
|
465
|
+
)
|
|
466
|
+
energy_tel1 = merged_table[
|
|
467
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_1"
|
|
468
|
+
]
|
|
469
|
+
energy_tel2 = merged_table[
|
|
470
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_2"
|
|
471
|
+
]
|
|
472
|
+
energy_uncertainty_tel1 = merged_table[
|
|
473
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert_1"
|
|
474
|
+
]
|
|
475
|
+
energy_uncertainty_tel2 = merged_table[
|
|
476
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert_2"
|
|
477
|
+
]
|
|
478
|
+
|
|
479
|
+
energy_sum = energy_tel1 + energy_tel2
|
|
480
|
+
energy_asymmetry = ((energy_tel1 - energy_tel2) / energy_sum).value
|
|
481
|
+
mean_asymmetry = energy_asymmetry.mean()
|
|
482
|
+
|
|
483
|
+
energy_asymmetry_uncertainty = (2.0 / energy_sum**2) * np.sqrt(
|
|
484
|
+
(energy_tel1 * energy_uncertainty_tel2) ** 2
|
|
485
|
+
+ (energy_tel2 * energy_uncertainty_tel1) ** 2
|
|
486
|
+
)
|
|
487
|
+
asymmetry_uncertainty = energy_asymmetry_uncertainty.mean()
|
|
488
|
+
mean_asymmetry_uncertainty = asymmetry_uncertainty / len(
|
|
489
|
+
energy_asymmetry
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
asymmetry_values.append(mean_asymmetry)
|
|
493
|
+
uncertainty_values.append(mean_asymmetry_uncertainty)
|
|
494
|
+
|
|
495
|
+
if asymmetry_values:
|
|
496
|
+
energy_asymmetry_results[telescope_type][pair] = {
|
|
497
|
+
"mean_asymmetry": sum(asymmetry_values) / len(asymmetry_values),
|
|
498
|
+
"mean_uncertainty": sum(uncertainty_values)
|
|
499
|
+
/ len(uncertainty_values),
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
return energy_asymmetry_results
|
|
503
|
+
|
|
504
|
+
def _load_telescope_energy_tables(self, pairs):
|
|
505
|
+
"""Load energy tables for unique telescopes in the given pairs."""
|
|
506
|
+
telescope_tables = {}
|
|
507
|
+
unique_tel_ids = {tel for pair in pairs for tel in pair}
|
|
508
|
+
for tel_id in unique_tel_ids:
|
|
509
|
+
try:
|
|
510
|
+
path = f"dl2/event/telescope/energy/{self.reconstruction_algorithm}Regressor/tel_{tel_id:03d}"
|
|
511
|
+
telescope_tables[tel_id] = read_table(self.input_url, path)
|
|
512
|
+
except NoSuchNodeError:
|
|
513
|
+
continue
|
|
514
|
+
return telescope_tables
|
|
515
|
+
|
|
516
|
+
def _merge_valid_energy_tables(self, tel1, table1, tel2, table2):
|
|
517
|
+
"""Merge valid energy tables for two telescopes based on common events."""
|
|
518
|
+
valid1 = table1[
|
|
519
|
+
table1[f"{self.reconstruction_algorithm}Regressor_tel_is_valid"]
|
|
520
|
+
]
|
|
521
|
+
valid2 = table2[
|
|
522
|
+
table2[f"{self.reconstruction_algorithm}Regressor_tel_is_valid"]
|
|
523
|
+
]
|
|
524
|
+
|
|
525
|
+
merged = join(
|
|
526
|
+
valid1[
|
|
527
|
+
[
|
|
528
|
+
"obs_id",
|
|
529
|
+
"event_id",
|
|
530
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy",
|
|
531
|
+
]
|
|
532
|
+
],
|
|
533
|
+
valid2[
|
|
534
|
+
[
|
|
535
|
+
"obs_id",
|
|
536
|
+
"event_id",
|
|
537
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy",
|
|
538
|
+
]
|
|
539
|
+
],
|
|
540
|
+
keys=["obs_id", "event_id"],
|
|
541
|
+
table_names=["1", "2"],
|
|
542
|
+
uniq_col_name="{col_name}_{table_name}",
|
|
543
|
+
)
|
|
544
|
+
return merged
|
|
545
|
+
|
|
546
|
+
def compute_cross_type_energy_ratios(self, cross_type_pairs, fit_results):
|
|
547
|
+
"""
|
|
548
|
+
Compute energy ratios grouped by telescope type pairs using throughput-corrected energies.
|
|
549
|
+
|
|
550
|
+
Parameters
|
|
551
|
+
----------
|
|
552
|
+
telescope_pairs : dict
|
|
553
|
+
A dictionary where keys are telescope types and values are lists of
|
|
554
|
+
tuples. Each tuple contains two telescope IDs (int) representing a pair.
|
|
555
|
+
fit_results : dict
|
|
556
|
+
Dictionary from `fit()` with format {subarray_name: {tel_id: (throughput, error)}}
|
|
557
|
+
|
|
558
|
+
Returns
|
|
559
|
+
-------
|
|
560
|
+
dict
|
|
561
|
+
Dictionary of format {(type1, type2): [corrected energy ratios]}
|
|
562
|
+
"""
|
|
563
|
+
energy_ratios_by_type = {}
|
|
564
|
+
mean_ratios = {}
|
|
565
|
+
telescope_tables = self._load_telescope_energy_tables(cross_type_pairs)
|
|
566
|
+
|
|
567
|
+
# Build lookup tables
|
|
568
|
+
throughput_map = {
|
|
569
|
+
tel_id: value[0]
|
|
570
|
+
for subarray in fit_results.values()
|
|
571
|
+
for tel_id, value in subarray.items()
|
|
572
|
+
}
|
|
573
|
+
type_map = {
|
|
574
|
+
tel_id: self.subarray.tel[tel_id].type for tel_id in self.subarray.tel_ids
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
self.relative_throughput_fitter.subarray = self.subarray
|
|
578
|
+
for tel1, tel2 in cross_type_pairs:
|
|
579
|
+
table1 = telescope_tables.get(tel1)
|
|
580
|
+
table2 = telescope_tables.get(tel2)
|
|
581
|
+
if table1 is None or table2 is None:
|
|
582
|
+
continue
|
|
583
|
+
merged = self._merge_valid_energy_tables(tel1, table1, tel2, table2)
|
|
584
|
+
if (
|
|
585
|
+
len(merged) > 0
|
|
586
|
+
and str(tel1) in throughput_map
|
|
587
|
+
and str(tel2) in throughput_map
|
|
588
|
+
):
|
|
589
|
+
throughput1 = throughput_map[str(tel1)]
|
|
590
|
+
throughput2 = throughput_map[str(tel2)]
|
|
591
|
+
normalisation1 = (
|
|
592
|
+
self.relative_throughput_fitter.throughput_normalization.tel[tel1]
|
|
593
|
+
)
|
|
594
|
+
normalisation2 = (
|
|
595
|
+
self.relative_throughput_fitter.throughput_normalization.tel[tel2]
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
corrected_energy1 = merged[
|
|
599
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_1"
|
|
600
|
+
] * (throughput1 / normalisation1)
|
|
601
|
+
corrected_energy2 = merged[
|
|
602
|
+
f"{self.reconstruction_algorithm}Regressor_tel_energy_2"
|
|
603
|
+
] * (throughput2 / normalisation2)
|
|
604
|
+
ratios = corrected_energy1 / corrected_energy2
|
|
605
|
+
|
|
606
|
+
type1 = type_map[tel1]
|
|
607
|
+
type2 = type_map[tel2]
|
|
608
|
+
type_pair = tuple(sorted((type1, type2)))
|
|
609
|
+
|
|
610
|
+
if type_pair not in energy_ratios_by_type:
|
|
611
|
+
energy_ratios_by_type[type_pair] = []
|
|
612
|
+
|
|
613
|
+
energy_ratios_by_type[type_pair].extend(ratios.tolist())
|
|
614
|
+
|
|
615
|
+
for type_pair, ratios in energy_ratios_by_type.items():
|
|
616
|
+
if ratios:
|
|
617
|
+
mu, std = norm.fit(ratios)
|
|
618
|
+
err = std / np.sqrt(len(ratios))
|
|
619
|
+
mean_ratios[type_pair] = (mu, err)
|
|
620
|
+
|
|
621
|
+
return mean_ratios
|
|
622
|
+
|
|
623
|
+
def save_monitoring_data(self, intercalibration_results, cross_calibration_results):
|
|
624
|
+
"""
|
|
625
|
+
Save the calibration products in DL2 monitoring data.
|
|
626
|
+
|
|
627
|
+
Parameters
|
|
628
|
+
----------
|
|
629
|
+
intercalibration_results : dict
|
|
630
|
+
Dictionary of the form:
|
|
631
|
+
{
|
|
632
|
+
'subarray_name': {tel_id: (value, error), ...}
|
|
633
|
+
}
|
|
634
|
+
that stores the relative throughput coefficients for
|
|
635
|
+
each telescope.
|
|
636
|
+
cross_calibration_results : dict
|
|
637
|
+
Dictionary of format {(type1, type2): [corrected energy ratios]}
|
|
638
|
+
"""
|
|
639
|
+
tel_rows = []
|
|
640
|
+
for size_type, tel_data in intercalibration_results.items():
|
|
641
|
+
size_type_str = str(size_type.value)
|
|
642
|
+
for tel_id, (value, error) in tel_data.items():
|
|
643
|
+
tel_rows.append((size_type_str, int(tel_id), value, error))
|
|
644
|
+
|
|
645
|
+
tel_table = Table(
|
|
646
|
+
rows=tel_rows, names=("size_type", "tel_id", "value", "error")
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
ratio_rows = []
|
|
650
|
+
for (type_a, type_b), (mean, error) in cross_calibration_results.items():
|
|
651
|
+
ratio_rows.append((str(type_a.value), str(type_b.value), mean, error))
|
|
652
|
+
|
|
653
|
+
ratio_table = Table(
|
|
654
|
+
rows=ratio_rows, names=("type_a", "type_b", "ratio", "error")
|
|
655
|
+
)
|
|
656
|
+
|
|
657
|
+
write_table(
|
|
658
|
+
tel_table,
|
|
659
|
+
self.output_url,
|
|
660
|
+
"/dl2/monitoring/inter_calibration",
|
|
661
|
+
overwrite=self.overwrite,
|
|
662
|
+
)
|
|
663
|
+
write_table(
|
|
664
|
+
ratio_table,
|
|
665
|
+
self.output_url,
|
|
666
|
+
"/dl2/monitoring/cross_calibration",
|
|
667
|
+
overwrite=self.overwrite,
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
def setup(self):
|
|
671
|
+
"""Set up the logic."""
|
|
672
|
+
# Read subarray description
|
|
673
|
+
self.subarray = SubarrayDescription.read(self.input_url)
|
|
674
|
+
self.relative_throughput_fitter = RelativeThroughputFitter(
|
|
675
|
+
subarray=self.subarray,
|
|
676
|
+
parent=self,
|
|
677
|
+
)
|
|
678
|
+
self.pair_finder = PairFinder(
|
|
679
|
+
subarray=self.subarray,
|
|
680
|
+
parent=self,
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
def start(self):
|
|
684
|
+
"""Perform cross-calibration per telescope subsystem."""
|
|
685
|
+
# Intercalibration
|
|
686
|
+
tel_pairs = self.pair_finder.find_pairs()
|
|
687
|
+
merged_tables = self.merge_tables(tel_pairs)
|
|
688
|
+
energy_asymmetry_results = self.calculate_energy_asymmetry(merged_tables)
|
|
689
|
+
intercalibration_results = {}
|
|
690
|
+
|
|
691
|
+
for subarray_name, subarray_data in energy_asymmetry_results.items():
|
|
692
|
+
measured_telescopes = set(
|
|
693
|
+
tel_id for (i, j), entry in subarray_data.items() for tel_id in (i, j)
|
|
694
|
+
)
|
|
695
|
+
self.relative_throughput_fitter.subarray = self.subarray.select_subarray(
|
|
696
|
+
tel_ids=measured_telescopes, name=subarray_name
|
|
697
|
+
)
|
|
698
|
+
intercalibration_results.update(
|
|
699
|
+
self.relative_throughput_fitter.fit(subarray_name, subarray_data)
|
|
700
|
+
)
|
|
701
|
+
self.log.debug("intercalibration minimization", intercalibration_results)
|
|
702
|
+
|
|
703
|
+
# Cross-calibration
|
|
704
|
+
cross_type_pairs = self.pair_finder.find_pairs(
|
|
705
|
+
by_tel_type=False, cross_type_only=True
|
|
706
|
+
)
|
|
707
|
+
cross_calibration_results = self.compute_cross_type_energy_ratios(
|
|
708
|
+
cross_type_pairs["XTEL"], intercalibration_results
|
|
709
|
+
)
|
|
710
|
+
# Save calibration products to dl2/monitoring
|
|
711
|
+
self.save_monitoring_data(intercalibration_results, cross_calibration_results)
|
|
712
|
+
|
|
713
|
+
def finish(self):
|
|
714
|
+
"""Store the results."""
|
|
715
|
+
self.log.info("Shutting down.")
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def main():
|
|
719
|
+
"""Run the app."""
|
|
720
|
+
tool = CalculateCrossCalibration()
|
|
721
|
+
tool.run()
|