ctao-calibpipe 0.3.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. calibpipe/__init__.py +5 -0
  2. calibpipe/_dev_version/__init__.py +9 -0
  3. calibpipe/_version.py +34 -0
  4. calibpipe/atmosphere/__init__.py +1 -0
  5. calibpipe/atmosphere/atmosphere_containers.py +109 -0
  6. calibpipe/atmosphere/meteo_data_handlers.py +485 -0
  7. calibpipe/atmosphere/models/README.md +14 -0
  8. calibpipe/atmosphere/models/__init__.py +1 -0
  9. calibpipe/atmosphere/models/macobac.ecsv +23 -0
  10. calibpipe/atmosphere/models/reference_MDPs/__init__.py +1 -0
  11. calibpipe/atmosphere/models/reference_MDPs/ref_density_at_15km_ctao-north_intermediate.ecsv +8 -0
  12. calibpipe/atmosphere/models/reference_MDPs/ref_density_at_15km_ctao-north_summer.ecsv +8 -0
  13. calibpipe/atmosphere/models/reference_MDPs/ref_density_at_15km_ctao-north_winter.ecsv +8 -0
  14. calibpipe/atmosphere/models/reference_MDPs/ref_density_at_15km_ctao-south_summer.ecsv +8 -0
  15. calibpipe/atmosphere/models/reference_MDPs/ref_density_at_15km_ctao-south_winter.ecsv +8 -0
  16. calibpipe/atmosphere/models/reference_atmospheres/__init__.py +1 -0
  17. calibpipe/atmosphere/models/reference_atmospheres/reference_atmo_model_v0_ctao-north_intermediate.ecsv +73 -0
  18. calibpipe/atmosphere/models/reference_atmospheres/reference_atmo_model_v0_ctao-north_summer.ecsv +73 -0
  19. calibpipe/atmosphere/models/reference_atmospheres/reference_atmo_model_v0_ctao-north_winter.ecsv +73 -0
  20. calibpipe/atmosphere/models/reference_atmospheres/reference_atmo_model_v0_ctao-south_summer.ecsv +73 -0
  21. calibpipe/atmosphere/models/reference_atmospheres/reference_atmo_model_v0_ctao-south_winter.ecsv +73 -0
  22. calibpipe/atmosphere/models/reference_rayleigh_scattering_profiles/__init__.py +1 -0
  23. calibpipe/atmosphere/models/reference_rayleigh_scattering_profiles/reference_rayleigh_extinction_profile_v0_ctao-north_intermediate.ecsv +857 -0
  24. calibpipe/atmosphere/models/reference_rayleigh_scattering_profiles/reference_rayleigh_extinction_profile_v0_ctao-north_summer.ecsv +857 -0
  25. calibpipe/atmosphere/models/reference_rayleigh_scattering_profiles/reference_rayleigh_extinction_profile_v0_ctao-north_winter.ecsv +857 -0
  26. calibpipe/atmosphere/models/reference_rayleigh_scattering_profiles/reference_rayleigh_extinction_profile_v0_ctao-south_summer.ecsv +857 -0
  27. calibpipe/atmosphere/models/reference_rayleigh_scattering_profiles/reference_rayleigh_extinction_profile_v0_ctao-south_winter.ecsv +857 -0
  28. calibpipe/atmosphere/templates/request_templates/__init__.py +1 -0
  29. calibpipe/atmosphere/templates/request_templates/copernicus.json +11 -0
  30. calibpipe/atmosphere/templates/request_templates/gdas.json +12 -0
  31. calibpipe/core/__init__.py +39 -0
  32. calibpipe/core/common_metadata_containers.py +198 -0
  33. calibpipe/core/exceptions.py +87 -0
  34. calibpipe/database/__init__.py +24 -0
  35. calibpipe/database/adapter/__init__.py +23 -0
  36. calibpipe/database/adapter/adapter.py +80 -0
  37. calibpipe/database/adapter/database_containers/__init__.py +63 -0
  38. calibpipe/database/adapter/database_containers/atmosphere.py +199 -0
  39. calibpipe/database/adapter/database_containers/common_metadata.py +150 -0
  40. calibpipe/database/adapter/database_containers/container_map.py +59 -0
  41. calibpipe/database/adapter/database_containers/observatory.py +61 -0
  42. calibpipe/database/adapter/database_containers/table_version_manager.py +39 -0
  43. calibpipe/database/adapter/database_containers/throughput.py +30 -0
  44. calibpipe/database/adapter/database_containers/version_control.py +17 -0
  45. calibpipe/database/connections/__init__.py +28 -0
  46. calibpipe/database/connections/calibpipe_database.py +60 -0
  47. calibpipe/database/connections/postgres_utils.py +97 -0
  48. calibpipe/database/connections/sql_connection.py +103 -0
  49. calibpipe/database/connections/user_confirmation.py +19 -0
  50. calibpipe/database/interfaces/__init__.py +71 -0
  51. calibpipe/database/interfaces/hashable_row_data.py +54 -0
  52. calibpipe/database/interfaces/queries.py +180 -0
  53. calibpipe/database/interfaces/sql_column_info.py +67 -0
  54. calibpipe/database/interfaces/sql_metadata.py +6 -0
  55. calibpipe/database/interfaces/sql_table_info.py +131 -0
  56. calibpipe/database/interfaces/table_handler.py +333 -0
  57. calibpipe/database/interfaces/types.py +96 -0
  58. calibpipe/telescope/throughput/containers.py +66 -0
  59. calibpipe/tests/conftest.py +274 -0
  60. calibpipe/tests/data/atmosphere/molecular_atmosphere/__init__.py +0 -0
  61. calibpipe/tests/data/atmosphere/molecular_atmosphere/contemporary_MDP.ecsv +34 -0
  62. calibpipe/tests/data/atmosphere/molecular_atmosphere/macobac.csv +852 -0
  63. calibpipe/tests/data/atmosphere/molecular_atmosphere/macobac.ecsv +23 -0
  64. calibpipe/tests/data/atmosphere/molecular_atmosphere/merged_file.ecsv +1082 -0
  65. calibpipe/tests/data/atmosphere/molecular_atmosphere/meteo_data_copernicus.ecsv +1082 -0
  66. calibpipe/tests/data/atmosphere/molecular_atmosphere/meteo_data_gdas.ecsv +66 -0
  67. calibpipe/tests/data/atmosphere/molecular_atmosphere/observatory_configurations.json +71 -0
  68. calibpipe/tests/data/utils/__init__.py +0 -0
  69. calibpipe/tests/data/utils/meteo_data_winter_and_summer.ecsv +12992 -0
  70. calibpipe/tests/test_conftest_data.py +200 -0
  71. calibpipe/tests/unittests/array/test_cross_calibration.py +412 -0
  72. calibpipe/tests/unittests/atmosphere/astral_testing.py +107 -0
  73. calibpipe/tests/unittests/atmosphere/test_meteo_data_handler.py +775 -0
  74. calibpipe/tests/unittests/atmosphere/test_molecular_atmosphere.py +327 -0
  75. calibpipe/tests/unittests/database/test_table_handler.py +163 -0
  76. calibpipe/tests/unittests/database/test_types.py +38 -0
  77. calibpipe/tests/unittests/telescope/camera/test_calculate_camcalib_coefficients.py +456 -0
  78. calibpipe/tests/unittests/telescope/camera/test_produce_camcalib_test_data.py +37 -0
  79. calibpipe/tests/unittests/telescope/throughput/test_muon_throughput_calibrator.py +693 -0
  80. calibpipe/tests/unittests/test_bootstrap_db.py +79 -0
  81. calibpipe/tests/unittests/utils/test_observatory.py +309 -0
  82. calibpipe/tools/atmospheric_base_tool.py +78 -0
  83. calibpipe/tools/atmospheric_model_db_loader.py +181 -0
  84. calibpipe/tools/basic_tool_with_db.py +38 -0
  85. calibpipe/tools/camcalib_test_data.py +374 -0
  86. calibpipe/tools/camera_calibrator.py +462 -0
  87. calibpipe/tools/contemporary_mdp_producer.py +87 -0
  88. calibpipe/tools/init_db.py +37 -0
  89. calibpipe/tools/macobac_calculator.py +82 -0
  90. calibpipe/tools/molecular_atmospheric_model_producer.py +197 -0
  91. calibpipe/tools/muon_throughput_calculator.py +219 -0
  92. calibpipe/tools/observatory_data_db_loader.py +71 -0
  93. calibpipe/tools/reference_atmospheric_model_selector.py +201 -0
  94. calibpipe/tools/telescope_cross_calibration_calculator.py +721 -0
  95. calibpipe/utils/__init__.py +10 -0
  96. calibpipe/utils/observatory.py +486 -0
  97. calibpipe/utils/observatory_containers.py +26 -0
  98. calibpipe/version.py +24 -0
  99. ctao_calibpipe-0.3.0rc2.dist-info/METADATA +92 -0
  100. ctao_calibpipe-0.3.0rc2.dist-info/RECORD +105 -0
  101. ctao_calibpipe-0.3.0rc2.dist-info/WHEEL +5 -0
  102. ctao_calibpipe-0.3.0rc2.dist-info/entry_points.txt +12 -0
  103. ctao_calibpipe-0.3.0rc2.dist-info/licenses/AUTHORS.md +13 -0
  104. ctao_calibpipe-0.3.0rc2.dist-info/licenses/LICENSE +21 -0
  105. ctao_calibpipe-0.3.0rc2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,721 @@
1
+ """Calculate the relative throughput of the telescopes."""
2
+
3
+ from collections import defaultdict
4
+ from itertools import combinations # noqa: D100
5
+
6
+ import astropy.units as u
7
+ import numpy as np
8
+ from astropy.table import Table, join
9
+ from ctapipe.core import (
10
+ TelescopeComponent,
11
+ Tool,
12
+ )
13
+ from ctapipe.core.traits import (
14
+ AstroQuantity,
15
+ Bool,
16
+ Dict,
17
+ Float,
18
+ FloatTelescopeParameter,
19
+ IntTelescopeParameter,
20
+ Path,
21
+ Unicode,
22
+ )
23
+ from ctapipe.instrument import SubarrayDescription
24
+ from ctapipe.io import read_table, write_table
25
+ from iminuit import Minuit
26
+ from scipy.stats import norm
27
+ from tables.exceptions import NoSuchNodeError
28
+
29
+
30
+ class RelativeThroughputFitter(TelescopeComponent):
31
+ """Perform relative throughput fitting for telescopes.
32
+
33
+ This component is used to fit the relative throughput of telescopes based on the
34
+ energy asymmetry of telescope pairs.
35
+ """
36
+
37
+ reference_telescopes = IntTelescopeParameter(
38
+ default_value=None,
39
+ help="ID of the telescopes whose throughput kept fixed during the intercalibration minimization",
40
+ allow_none=True,
41
+ ).tag(config=True)
42
+ throughput_normalization = FloatTelescopeParameter(
43
+ default_value=1.0,
44
+ help="Setting the overall telescope throughput normalization. "
45
+ "Depending the use case, it could reflect the "
46
+ "absolute optical throughput measured by the muon rings / illuminator, "
47
+ "if we want to compare or complement these methods. "
48
+ "Alternatively it could get an arbitrary number, e.g. one that sets the average throughput to 1, "
49
+ "if the user wants to 'flat-field' the array. "
50
+ "Finally it could be set to 1, if we want to identify outlier telescopes or study the aging.",
51
+ allow_none=False,
52
+ ).tag(config=True)
53
+
54
+ def fit(self, subarray_name, subarray_data):
55
+ """
56
+ Perform minimization for each telescope subsystem to compute optical throughput factors.
57
+
58
+ Parameters
59
+ ----------
60
+ subarray_data : dict
61
+ A dictionary where keys are telescope pairs and values are dictionaries containing
62
+ "mean_asymmetry": the average energy asymmetry and "mean_uncertainty": the average
63
+ uncertainty of the asymmetry for each telescope pair.
64
+ subarray_name : str
65
+ The name of the intercalibration subarray, used to identify the telescopes.
66
+ Usually corresponds to the telescope type (e.g., "LST", "MST", "SST").
67
+
68
+ Returns
69
+ -------
70
+ dict
71
+ Dictionary of the form:
72
+ {
73
+ 'subarray_name': {tel_id: (value, error), ...}
74
+ }
75
+ """
76
+ results = {}
77
+
78
+ tel_ids = self.subarray.tel_ids
79
+
80
+ initial_values = {
81
+ str(tel_id): self.throughput_normalization.tel[tel_id] for tel_id in tel_ids
82
+ }
83
+
84
+ def _chi2(*values):
85
+ params = dict(zip(tel_ids, values))
86
+ chi2 = 0.0
87
+ for (i, j), entry in subarray_data.items():
88
+ a_ij = entry["mean_asymmetry"]
89
+ s_ij = entry["mean_uncertainty"]
90
+ ci = params[i]
91
+ cj = params[j]
92
+ model = (ci - cj) / (ci + cj)
93
+ chi2 += ((a_ij - model) ** 2) / (s_ij**2)
94
+ return chi2
95
+
96
+ fit = Minuit(_chi2, name=initial_values.keys(), **initial_values)
97
+
98
+ # Create a list of unique reference telescope IDs
99
+ unique_reference_telescopes = set(
100
+ self.reference_telescopes.tel[tel_id]
101
+ for tel_id in tel_ids
102
+ if self.reference_telescopes.tel[tel_id] is not None
103
+ )
104
+
105
+ # Pass each unique reference telescope ID to fit.fixed
106
+ for ref_tel_id in unique_reference_telescopes:
107
+ fit.fixed[str(ref_tel_id)] = True
108
+
109
+ fit.errordef = 1
110
+ fit.migrad()
111
+
112
+ results[subarray_name] = {
113
+ tel_id: (fit.values[tel_id], fit.errors[tel_id])
114
+ for tel_id in initial_values.keys()
115
+ }
116
+ return results
117
+
118
+
119
+ class PairFinder(TelescopeComponent):
120
+ """Find pairs of telescopes based on their types and distances between them."""
121
+
122
+ max_impact_distance = FloatTelescopeParameter(
123
+ default_value=[
124
+ ("type", "LST*", 125.0),
125
+ ("type", "MST*", 125.0),
126
+ ("type", "SST*", 225.0),
127
+ ],
128
+ help="Maximum distance between the telescopes and a shower core in meters. "
129
+ "The maximum distance between the telescopes in pair "
130
+ "should not exceed the sum of these values for the telescopes in pair.",
131
+ allow_none=False,
132
+ ).tag(config=True)
133
+
134
+ def find_pairs(self, by_tel_type=True, cross_type_only=False):
135
+ """
136
+ Find pairs of telescope IDs.
137
+
138
+ Parameters
139
+ ----------
140
+ by_tel_type : bool
141
+ If True, find pairs of telescopes of the same type.
142
+ If False, find pairs of all telescopes in the array.
143
+ cross_type_only : bool
144
+ If True, find pairs of telescopes of different types only.
145
+
146
+ Returns
147
+ -------
148
+ dict
149
+ A dictionary where keys are telescope types (str) and values are sets of
150
+ pairs of telescope IDs (int, int). If by_tel_type is False, the key will be "ALL".
151
+ If cross_type_only is True, the key will be "XTEL".
152
+ """
153
+ result = defaultdict(set)
154
+ telescope_positions = self.subarray.positions
155
+
156
+ def check_distance(tel1, tel2):
157
+ """Check if the pair of telescopes is within the maximum distance."""
158
+ pos1 = telescope_positions[tel1]
159
+ pos2 = telescope_positions[tel2]
160
+ distance = np.linalg.norm((pos1 - pos2).to(u.m).value)
161
+ return (
162
+ distance
163
+ <= self.max_impact_distance.tel[tel1]
164
+ + self.max_impact_distance.tel[tel2]
165
+ )
166
+
167
+ def check_type(tel1, tel2):
168
+ """Check if the pair of telescopes are of the same type."""
169
+ return self.subarray.tel[tel1].type == self.subarray.tel[tel2].type
170
+
171
+ telescope_types = defaultdict(list)
172
+ if by_tel_type:
173
+ for tel_id, desc in self.subarray.tel.items():
174
+ telescope_types[desc.type].append(tel_id)
175
+ else:
176
+ if cross_type_only:
177
+ telescope_types["XTEL"] = list(self.subarray.tel_ids)
178
+ else:
179
+ telescope_types["ALL"] = list(self.subarray.tel_ids)
180
+
181
+ for tel_type, tel_ids in telescope_types.items():
182
+ for tel1, tel2 in combinations(tel_ids, 2):
183
+ if cross_type_only and check_type(tel1, tel2):
184
+ continue
185
+ if check_distance(tel1, tel2):
186
+ result[tel_type].add((tel1, tel2))
187
+
188
+ return result
189
+
190
+
191
+ class CalculateCrossCalibration(Tool):
192
+ """Calibrator that performs cross calibration of telescopes."""
193
+
194
+ input_url = Path(
195
+ default_value="dl2.h5",
196
+ help="Path to the input file with the DL2 (merged) data.",
197
+ ).tag(config=True)
198
+ output_url = Path(
199
+ default_value="monitoring_cross_calibration_dl2.h5",
200
+ help="Path to the output file where the produced calibration"
201
+ "products will be stored",
202
+ ).tag(config=True)
203
+ overwrite = Bool(default_value=False, help="Overwrite output file.").tag(
204
+ config=True
205
+ )
206
+ reconstruction_algorithm = Unicode(
207
+ default_value="RandomForest",
208
+ help="Name of the reconstruction algorithm",
209
+ allow_none=False,
210
+ ).tag(config=True)
211
+ event_filters = Dict(
212
+ per_key_traits={
213
+ "min_gammaness": Float(allow_none=True),
214
+ "min_energy": AstroQuantity(
215
+ physical_type=u.physical.energy, allow_none=True
216
+ ),
217
+ "max_distance_asymmetry": Float(allow_none=True),
218
+ },
219
+ default_value={
220
+ "min_gammaness": None,
221
+ "min_energy": None,
222
+ "max_distance_asymmetry": None,
223
+ },
224
+ help=(
225
+ "Dictionary of event filters:\n"
226
+ " - min_gammaness is used to select the gamma-like events\n"
227
+ " - min_energy defines the energy range suitable for different types of telescopes\n"
228
+ " - max_distance_asymmetry sets the equidistance asymmetry limit\n"
229
+ "If a filter parameter is set to None (the default value), "
230
+ "it will not be applied during event selection."
231
+ ),
232
+ ).tag(config=True)
233
+
234
+ aliases = {
235
+ ("i", "input_url"): "CalculateCrossCalibration.input_url",
236
+ ("o", "output_url"): "CalculateCrossCalibration.output_url",
237
+ }
238
+
239
+ classes = [RelativeThroughputFitter, PairFinder]
240
+
241
+ def merge_tables(self, telescope_pairs):
242
+ """
243
+ Merge telescope energy tables for specified telescope pairs.
244
+
245
+ Parameters
246
+ ----------
247
+ telescope_pairs : dict
248
+ A dictionary where keys are telescope types (str), and values are lists of
249
+ tuples. Each tuple contains two telescope IDs (int) representing a pair.
250
+
251
+ Returns
252
+ -------
253
+ dict
254
+ A dictionary where keys are tuples of telescope IDs (int, int),
255
+ representing the telescope pairs. The values are lists of merged tables
256
+ for the specified telescope pairs. If no tables are successfully merged
257
+ for a telescope pair, the list will be empty.
258
+ """
259
+ merged_tables = {}
260
+ unique_telescope_ids = {
261
+ tel_id
262
+ for pairs in telescope_pairs.values()
263
+ for pair in pairs
264
+ for tel_id in pair
265
+ }
266
+
267
+ telescope_tables = {}
268
+ for tel_id in unique_telescope_ids:
269
+ try:
270
+ path = f"dl2/event/telescope/energy/{self.reconstruction_algorithm}Regressor/tel_{tel_id:03d}"
271
+ telescope_tables[tel_id] = read_table(
272
+ self.input_url,
273
+ path,
274
+ condition=f"{self.reconstruction_algorithm}Regressor_tel_is_valid",
275
+ )
276
+ except NoSuchNodeError:
277
+ pass # data for one telescope is missing, will be reported in application to a particular pair later.
278
+ for telescope_type, pairs in telescope_pairs.items():
279
+ if telescope_type not in merged_tables:
280
+ merged_tables[telescope_type] = {}
281
+ for tel1, tel2 in pairs:
282
+ table_1 = telescope_tables.get(tel1)
283
+ table_2 = telescope_tables.get(tel2)
284
+ if table_1 is None or table_2 is None:
285
+ self.log.warning(
286
+ "Missing telescope data for tel %s in pair (%s, %s)",
287
+ tel1 if table_1 is None else tel2,
288
+ tel1,
289
+ tel2,
290
+ )
291
+ continue
292
+
293
+ try:
294
+ merged_table = join(
295
+ table_1[
296
+ [
297
+ "obs_id",
298
+ "event_id",
299
+ f"{self.reconstruction_algorithm}Regressor_tel_energy",
300
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert",
301
+ ]
302
+ ],
303
+ table_2[
304
+ [
305
+ "obs_id",
306
+ "event_id",
307
+ f"{self.reconstruction_algorithm}Regressor_tel_energy",
308
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert",
309
+ ]
310
+ ],
311
+ keys=["obs_id", "event_id"],
312
+ )
313
+
314
+ if len(merged_table) > 0:
315
+ pair = (tel1, tel2)
316
+ if pair not in merged_tables[telescope_type]:
317
+ merged_tables[telescope_type][pair] = []
318
+
319
+ merged_tables[telescope_type][pair].append(merged_table)
320
+ except MemoryError as e:
321
+ self.log.error(
322
+ "MemoryError: The data is too large to process: %s", e
323
+ )
324
+ raise
325
+
326
+ return merged_tables
327
+
328
+ def _apply_min_gammaness(self, tel_id1, tel_id2, threshold):
329
+ """Select gamma-like events based on the gammaness threshold."""
330
+ events_tel1 = self._get_gamma_like_events(tel_id1, threshold)
331
+ events_tel2 = self._get_gamma_like_events(tel_id2, threshold)
332
+ return events_tel1 & events_tel2
333
+
334
+ def _apply_min_energy(self, tel_id1, tel_id2, threshold):
335
+ """Select showers over an energy threshold."""
336
+ events_tel1 = self._set_energy_threshold(tel_id1, threshold)
337
+ events_tel2 = self._set_energy_threshold(tel_id2, threshold)
338
+ return events_tel1 & events_tel2
339
+
340
+ def _apply_max_distance_asymmetry(self, tel_id1, tel_id2, threshold):
341
+ """Select showers with similar impact distances."""
342
+
343
+ def load_distance_table(tel_id):
344
+ path = f"dl2/event/telescope/impact/HillasReconstructor/tel_{tel_id:03d}"
345
+ return read_table(self.input_url, path)[
346
+ ["obs_id", "event_id", "HillasReconstructor_tel_impact_distance"]
347
+ ]
348
+
349
+ table1 = load_distance_table(tel_id1)
350
+ table2 = load_distance_table(tel_id2)
351
+
352
+ merged = join(table1, table2, keys=["obs_id", "event_id"])
353
+ dist1 = merged["HillasReconstructor_tel_impact_distance_1"]
354
+ dist2 = merged["HillasReconstructor_tel_impact_distance_2"]
355
+
356
+ asym = ((dist2 - dist1) / (dist1 + dist2)).value
357
+ mask = np.abs(asym) < threshold
358
+ return set(zip(merged["obs_id"][mask], merged["event_id"][mask]))
359
+
360
+ def _get_gamma_like_events(self, tel_id, threshold):
361
+ """Select showers that are gamma like."""
362
+ path = f"dl2/event/telescope/classification/{self.reconstruction_algorithm}Classifier/tel_{tel_id:03d}"
363
+ class_table = read_table(
364
+ self.input_url,
365
+ path,
366
+ condition=f"{self.reconstruction_algorithm}Classifier_tel_is_valid",
367
+ )
368
+ mask = (
369
+ class_table[f"{self.reconstruction_algorithm}Classifier_tel_prediction"]
370
+ > threshold
371
+ )
372
+ return set(zip(class_table["obs_id"][mask], class_table["event_id"][mask]))
373
+
374
+ def _set_energy_threshold(self, tel_id, threshold):
375
+ """Select showers over an energy threshold."""
376
+ path = f"dl2/event/telescope/energy/{self.reconstruction_algorithm}Regressor/tel_{tel_id:03d}"
377
+ energy_table = read_table(
378
+ self.input_url,
379
+ path,
380
+ condition=f"{self.reconstruction_algorithm}Regressor_tel_is_valid",
381
+ )
382
+ mask = (
383
+ energy_table[f"{self.reconstruction_algorithm}Regressor_tel_energy"]
384
+ > threshold
385
+ )
386
+ return set(zip(energy_table["obs_id"][mask], energy_table["event_id"][mask]))
387
+
388
+ def event_selection(self, merged_table, tel1, tel2):
389
+ """
390
+ Filter merged table based on energy, classification, and optionally equidistant impact criteria.
391
+
392
+ Parameters
393
+ ----------
394
+ merged_table : Table
395
+ Merged table from two telescopes.
396
+ tel1, tel2 : int
397
+ Telescope IDs.
398
+
399
+ Returns
400
+ -------
401
+ Table
402
+ Filtered table.
403
+ """
404
+ selected_events = self._get_valid_events(tel1, tel2)
405
+
406
+ event_ids = set(zip(merged_table["obs_id"], merged_table["event_id"]))
407
+ valid_mask = np.array(
408
+ [(obs_id, event_id) in selected_events for obs_id, event_id in event_ids]
409
+ )
410
+ return merged_table[valid_mask]
411
+
412
+ def _get_valid_events(self, tel_id1, tel_id2):
413
+ """Apply all event filters for a given telescope."""
414
+ event_sets = []
415
+ for filter_name, threshold in self.event_filters.items():
416
+ if threshold is None:
417
+ self.log.debug("Skipping filter %s", filter_name)
418
+ continue
419
+ try:
420
+ filter_func = getattr(self, f"_apply_{filter_name}")
421
+ except AttributeError:
422
+ self.log.error(
423
+ "Filter function _apply_%s not found for filter %s",
424
+ filter_name,
425
+ filter_name,
426
+ )
427
+ raise ValueError(
428
+ f"Filter {filter_name} is not implemented or not recognized."
429
+ )
430
+ event_sets.append(filter_func(tel_id1, tel_id2, threshold))
431
+
432
+ return set.intersection(*event_sets) if event_sets else set()
433
+
434
+ def calculate_energy_asymmetry(self, merged_tables):
435
+ """
436
+ Calculate the mean energy asymmetry and its uncertainty for each telescope pair from the merged tables.
437
+
438
+ Parameters
439
+ ----------
440
+ merged_tables : dict
441
+ A dictionary where keys are telescope types. Each value is another dictionary where keys are tuples
442
+ of telescope IDs (int, int), representing telescope pairs, and the values are lists of merged tables
443
+ containing energy data.
444
+
445
+ Returns
446
+ -------
447
+ dict
448
+ A nested dictionary with telescope types as the first level of keys.
449
+ Each value is a dictionary with telescope pairs as keys and a dictionary as value containing:
450
+ - "mean_asymmetry": the average energy asymmetry
451
+ - "mean_uncertainty": the average uncertainty of the asymmetry
452
+ """
453
+ energy_asymmetry_results = {}
454
+
455
+ for telescope_type, pairs in merged_tables.items():
456
+ energy_asymmetry_results[telescope_type] = {}
457
+ for pair, merged_tables_list in pairs.items():
458
+ asymmetry_values = []
459
+ uncertainty_values = []
460
+ for merged_table_unfiltered in merged_tables_list:
461
+ merged_table = self.event_selection(
462
+ merged_table_unfiltered,
463
+ pair[0],
464
+ pair[1],
465
+ )
466
+ energy_tel1 = merged_table[
467
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_1"
468
+ ]
469
+ energy_tel2 = merged_table[
470
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_2"
471
+ ]
472
+ energy_uncertainty_tel1 = merged_table[
473
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert_1"
474
+ ]
475
+ energy_uncertainty_tel2 = merged_table[
476
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_uncert_2"
477
+ ]
478
+
479
+ energy_sum = energy_tel1 + energy_tel2
480
+ energy_asymmetry = ((energy_tel1 - energy_tel2) / energy_sum).value
481
+ mean_asymmetry = energy_asymmetry.mean()
482
+
483
+ energy_asymmetry_uncertainty = (2.0 / energy_sum**2) * np.sqrt(
484
+ (energy_tel1 * energy_uncertainty_tel2) ** 2
485
+ + (energy_tel2 * energy_uncertainty_tel1) ** 2
486
+ )
487
+ asymmetry_uncertainty = energy_asymmetry_uncertainty.mean()
488
+ mean_asymmetry_uncertainty = asymmetry_uncertainty / len(
489
+ energy_asymmetry
490
+ )
491
+
492
+ asymmetry_values.append(mean_asymmetry)
493
+ uncertainty_values.append(mean_asymmetry_uncertainty)
494
+
495
+ if asymmetry_values:
496
+ energy_asymmetry_results[telescope_type][pair] = {
497
+ "mean_asymmetry": sum(asymmetry_values) / len(asymmetry_values),
498
+ "mean_uncertainty": sum(uncertainty_values)
499
+ / len(uncertainty_values),
500
+ }
501
+
502
+ return energy_asymmetry_results
503
+
504
+ def _load_telescope_energy_tables(self, pairs):
505
+ """Load energy tables for unique telescopes in the given pairs."""
506
+ telescope_tables = {}
507
+ unique_tel_ids = {tel for pair in pairs for tel in pair}
508
+ for tel_id in unique_tel_ids:
509
+ try:
510
+ path = f"dl2/event/telescope/energy/{self.reconstruction_algorithm}Regressor/tel_{tel_id:03d}"
511
+ telescope_tables[tel_id] = read_table(self.input_url, path)
512
+ except NoSuchNodeError:
513
+ continue
514
+ return telescope_tables
515
+
516
+ def _merge_valid_energy_tables(self, tel1, table1, tel2, table2):
517
+ """Merge valid energy tables for two telescopes based on common events."""
518
+ valid1 = table1[
519
+ table1[f"{self.reconstruction_algorithm}Regressor_tel_is_valid"]
520
+ ]
521
+ valid2 = table2[
522
+ table2[f"{self.reconstruction_algorithm}Regressor_tel_is_valid"]
523
+ ]
524
+
525
+ merged = join(
526
+ valid1[
527
+ [
528
+ "obs_id",
529
+ "event_id",
530
+ f"{self.reconstruction_algorithm}Regressor_tel_energy",
531
+ ]
532
+ ],
533
+ valid2[
534
+ [
535
+ "obs_id",
536
+ "event_id",
537
+ f"{self.reconstruction_algorithm}Regressor_tel_energy",
538
+ ]
539
+ ],
540
+ keys=["obs_id", "event_id"],
541
+ table_names=["1", "2"],
542
+ uniq_col_name="{col_name}_{table_name}",
543
+ )
544
+ return merged
545
+
546
+ def compute_cross_type_energy_ratios(self, cross_type_pairs, fit_results):
547
+ """
548
+ Compute energy ratios grouped by telescope type pairs using throughput-corrected energies.
549
+
550
+ Parameters
551
+ ----------
552
+ telescope_pairs : dict
553
+ A dictionary where keys are telescope types and values are lists of
554
+ tuples. Each tuple contains two telescope IDs (int) representing a pair.
555
+ fit_results : dict
556
+ Dictionary from `fit()` with format {subarray_name: {tel_id: (throughput, error)}}
557
+
558
+ Returns
559
+ -------
560
+ dict
561
+ Dictionary of format {(type1, type2): [corrected energy ratios]}
562
+ """
563
+ energy_ratios_by_type = {}
564
+ mean_ratios = {}
565
+ telescope_tables = self._load_telescope_energy_tables(cross_type_pairs)
566
+
567
+ # Build lookup tables
568
+ throughput_map = {
569
+ tel_id: value[0]
570
+ for subarray in fit_results.values()
571
+ for tel_id, value in subarray.items()
572
+ }
573
+ type_map = {
574
+ tel_id: self.subarray.tel[tel_id].type for tel_id in self.subarray.tel_ids
575
+ }
576
+
577
+ self.relative_throughput_fitter.subarray = self.subarray
578
+ for tel1, tel2 in cross_type_pairs:
579
+ table1 = telescope_tables.get(tel1)
580
+ table2 = telescope_tables.get(tel2)
581
+ if table1 is None or table2 is None:
582
+ continue
583
+ merged = self._merge_valid_energy_tables(tel1, table1, tel2, table2)
584
+ if (
585
+ len(merged) > 0
586
+ and str(tel1) in throughput_map
587
+ and str(tel2) in throughput_map
588
+ ):
589
+ throughput1 = throughput_map[str(tel1)]
590
+ throughput2 = throughput_map[str(tel2)]
591
+ normalisation1 = (
592
+ self.relative_throughput_fitter.throughput_normalization.tel[tel1]
593
+ )
594
+ normalisation2 = (
595
+ self.relative_throughput_fitter.throughput_normalization.tel[tel2]
596
+ )
597
+
598
+ corrected_energy1 = merged[
599
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_1"
600
+ ] * (throughput1 / normalisation1)
601
+ corrected_energy2 = merged[
602
+ f"{self.reconstruction_algorithm}Regressor_tel_energy_2"
603
+ ] * (throughput2 / normalisation2)
604
+ ratios = corrected_energy1 / corrected_energy2
605
+
606
+ type1 = type_map[tel1]
607
+ type2 = type_map[tel2]
608
+ type_pair = tuple(sorted((type1, type2)))
609
+
610
+ if type_pair not in energy_ratios_by_type:
611
+ energy_ratios_by_type[type_pair] = []
612
+
613
+ energy_ratios_by_type[type_pair].extend(ratios.tolist())
614
+
615
+ for type_pair, ratios in energy_ratios_by_type.items():
616
+ if ratios:
617
+ mu, std = norm.fit(ratios)
618
+ err = std / np.sqrt(len(ratios))
619
+ mean_ratios[type_pair] = (mu, err)
620
+
621
+ return mean_ratios
622
+
623
+ def save_monitoring_data(self, intercalibration_results, cross_calibration_results):
624
+ """
625
+ Save the calibration products in DL2 monitoring data.
626
+
627
+ Parameters
628
+ ----------
629
+ intercalibration_results : dict
630
+ Dictionary of the form:
631
+ {
632
+ 'subarray_name': {tel_id: (value, error), ...}
633
+ }
634
+ that stores the relative throughput coefficients for
635
+ each telescope.
636
+ cross_calibration_results : dict
637
+ Dictionary of format {(type1, type2): [corrected energy ratios]}
638
+ """
639
+ tel_rows = []
640
+ for size_type, tel_data in intercalibration_results.items():
641
+ size_type_str = str(size_type.value)
642
+ for tel_id, (value, error) in tel_data.items():
643
+ tel_rows.append((size_type_str, int(tel_id), value, error))
644
+
645
+ tel_table = Table(
646
+ rows=tel_rows, names=("size_type", "tel_id", "value", "error")
647
+ )
648
+
649
+ ratio_rows = []
650
+ for (type_a, type_b), (mean, error) in cross_calibration_results.items():
651
+ ratio_rows.append((str(type_a.value), str(type_b.value), mean, error))
652
+
653
+ ratio_table = Table(
654
+ rows=ratio_rows, names=("type_a", "type_b", "ratio", "error")
655
+ )
656
+
657
+ write_table(
658
+ tel_table,
659
+ self.output_url,
660
+ "/dl2/monitoring/inter_calibration",
661
+ overwrite=self.overwrite,
662
+ )
663
+ write_table(
664
+ ratio_table,
665
+ self.output_url,
666
+ "/dl2/monitoring/cross_calibration",
667
+ overwrite=self.overwrite,
668
+ )
669
+
670
+ def setup(self):
671
+ """Set up the logic."""
672
+ # Read subarray description
673
+ self.subarray = SubarrayDescription.read(self.input_url)
674
+ self.relative_throughput_fitter = RelativeThroughputFitter(
675
+ subarray=self.subarray,
676
+ parent=self,
677
+ )
678
+ self.pair_finder = PairFinder(
679
+ subarray=self.subarray,
680
+ parent=self,
681
+ )
682
+
683
+ def start(self):
684
+ """Perform cross-calibration per telescope subsystem."""
685
+ # Intercalibration
686
+ tel_pairs = self.pair_finder.find_pairs()
687
+ merged_tables = self.merge_tables(tel_pairs)
688
+ energy_asymmetry_results = self.calculate_energy_asymmetry(merged_tables)
689
+ intercalibration_results = {}
690
+
691
+ for subarray_name, subarray_data in energy_asymmetry_results.items():
692
+ measured_telescopes = set(
693
+ tel_id for (i, j), entry in subarray_data.items() for tel_id in (i, j)
694
+ )
695
+ self.relative_throughput_fitter.subarray = self.subarray.select_subarray(
696
+ tel_ids=measured_telescopes, name=subarray_name
697
+ )
698
+ intercalibration_results.update(
699
+ self.relative_throughput_fitter.fit(subarray_name, subarray_data)
700
+ )
701
+ self.log.debug("intercalibration minimization", intercalibration_results)
702
+
703
+ # Cross-calibration
704
+ cross_type_pairs = self.pair_finder.find_pairs(
705
+ by_tel_type=False, cross_type_only=True
706
+ )
707
+ cross_calibration_results = self.compute_cross_type_energy_ratios(
708
+ cross_type_pairs["XTEL"], intercalibration_results
709
+ )
710
+ # Save calibration products to dl2/monitoring
711
+ self.save_monitoring_data(intercalibration_results, cross_calibration_results)
712
+
713
+ def finish(self):
714
+ """Store the results."""
715
+ self.log.info("Shutting down.")
716
+
717
+
718
+ def main():
719
+ """Run the app."""
720
+ tool = CalculateCrossCalibration()
721
+ tool.run()