gammasimtools 0.16.0__py3-none-any.whl → 0.17.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/METADATA +4 -2
- {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/RECORD +60 -54
- {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/WHEEL +1 -1
- {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/entry_points.txt +3 -1
- simtools/_version.py +2 -2
- simtools/applications/derive_ctao_array_layouts.py +5 -5
- simtools/applications/generate_simtel_event_data.py +36 -46
- simtools/applications/merge_tables.py +104 -0
- simtools/applications/plot_array_layout.py +145 -258
- simtools/applications/production_derive_corsika_limits.py +35 -220
- simtools/applications/production_derive_statistics.py +77 -43
- simtools/applications/simulate_light_emission.py +1 -0
- simtools/applications/simulate_prod.py +30 -18
- simtools/applications/simulate_prod_htcondor_generator.py +0 -1
- simtools/applications/submit_array_layouts.py +93 -0
- simtools/applications/verify_simulation_model_production_tables.py +52 -0
- simtools/camera/camera_efficiency.py +3 -3
- simtools/configuration/commandline_parser.py +28 -34
- simtools/configuration/configurator.py +0 -4
- simtools/corsika/corsika_config.py +17 -12
- simtools/corsika/primary_particle.py +46 -13
- simtools/data_model/metadata_collector.py +7 -3
- simtools/db/db_handler.py +11 -11
- simtools/db/db_model_upload.py +2 -2
- simtools/io_operations/io_handler.py +2 -2
- simtools/io_operations/io_table_handler.py +345 -0
- simtools/job_execution/htcondor_script_generator.py +2 -2
- simtools/job_execution/job_manager.py +7 -121
- simtools/layout/array_layout_utils.py +385 -0
- simtools/model/array_model.py +5 -0
- simtools/model/model_repository.py +134 -0
- simtools/production_configuration/{calculate_statistical_errors_grid_point.py → calculate_statistical_uncertainties_grid_point.py} +101 -112
- simtools/production_configuration/derive_corsika_limits.py +239 -111
- simtools/production_configuration/derive_corsika_limits_grid.py +189 -0
- simtools/production_configuration/derive_production_statistics.py +57 -26
- simtools/production_configuration/derive_production_statistics_handler.py +70 -37
- simtools/production_configuration/interpolation_handler.py +296 -94
- simtools/ray_tracing/ray_tracing.py +7 -6
- simtools/reporting/docs_read_parameters.py +104 -62
- simtools/runners/corsika_simtel_runner.py +4 -1
- simtools/runners/runner_services.py +5 -4
- simtools/schemas/model_parameters/dsum_threshold.schema.yml +41 -0
- simtools/schemas/production_configuration_metrics.schema.yml +2 -2
- simtools/simtel/simtel_config_writer.py +34 -14
- simtools/simtel/simtel_io_event_reader.py +301 -194
- simtools/simtel/simtel_io_event_writer.py +207 -227
- simtools/simtel/simtel_io_file_info.py +9 -4
- simtools/simtel/simtel_io_metadata.py +20 -5
- simtools/simtel/simulator_array.py +2 -2
- simtools/simtel/simulator_light_emission.py +79 -34
- simtools/simtel/simulator_ray_tracing.py +2 -2
- simtools/simulator.py +101 -68
- simtools/testing/validate_output.py +4 -1
- simtools/utils/general.py +1 -1
- simtools/utils/names.py +5 -5
- simtools/visualization/plot_array_layout.py +242 -0
- simtools/visualization/plot_pixels.py +681 -0
- simtools/visualization/visualize.py +3 -219
- simtools/applications/production_generate_simulation_config.py +0 -152
- simtools/layout/ctao_array_layouts.py +0 -172
- simtools/production_configuration/generate_simulation_config.py +0 -158
- {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/licenses/LICENSE +0 -0
- {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/top_level.txt +0 -0
|
@@ -46,6 +46,7 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
46
46
|
self._telescope_model = telescope_model
|
|
47
47
|
|
|
48
48
|
self.label = label if label is not None else self._telescope_model.label
|
|
49
|
+
self.test = test
|
|
49
50
|
|
|
50
51
|
self._calibration_model = calibration_model
|
|
51
52
|
self._site_model = site_model
|
|
@@ -56,7 +57,9 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
56
57
|
self._rep_number = 0
|
|
57
58
|
self.runs = 1
|
|
58
59
|
self.photons_per_run = (
|
|
59
|
-
self._calibration_model.get_parameter_value("photons_per_run")
|
|
60
|
+
(self._calibration_model.get_parameter_value("photons_per_run"))
|
|
61
|
+
if not self.test
|
|
62
|
+
else 1e8
|
|
60
63
|
)
|
|
61
64
|
|
|
62
65
|
self.le_application = le_application
|
|
@@ -64,7 +67,6 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
64
67
|
self.distance = None
|
|
65
68
|
self.light_source_type = light_source_type
|
|
66
69
|
self._telescope_model.write_sim_telarray_config_file(additional_model=site_model)
|
|
67
|
-
self.test = test
|
|
68
70
|
|
|
69
71
|
@staticmethod
|
|
70
72
|
def light_emission_default_configuration():
|
|
@@ -119,14 +121,15 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
119
121
|
list
|
|
120
122
|
The pointing vector from the calibration device to the telescope.
|
|
121
123
|
"""
|
|
122
|
-
|
|
123
|
-
x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value(
|
|
124
|
+
x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value_with_unit(
|
|
124
125
|
"array_element_position_ground"
|
|
125
126
|
)
|
|
127
|
+
x_cal, y_cal, z_cal = [coord.to(u.m).value for coord in (x_cal, y_cal, z_cal)]
|
|
126
128
|
cal_vect = np.array([x_cal, y_cal, z_cal])
|
|
127
|
-
x_tel, y_tel, z_tel = self._telescope_model.
|
|
129
|
+
x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
|
|
128
130
|
"array_element_position_ground"
|
|
129
131
|
)
|
|
132
|
+
x_tel, y_tel, z_tel = [coord.to(u.m).value for coord in (x_tel, y_tel, z_tel)]
|
|
130
133
|
|
|
131
134
|
tel_vect = np.array([x_tel, y_tel, z_tel])
|
|
132
135
|
|
|
@@ -153,6 +156,30 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
153
156
|
)
|
|
154
157
|
return pointing_vector.tolist(), [tel_theta, tel_phi, laser_theta, laser_phi]
|
|
155
158
|
|
|
159
|
+
def _write_telpos_file(self):
|
|
160
|
+
"""
|
|
161
|
+
Write the telescope positions to a telpos file.
|
|
162
|
+
|
|
163
|
+
The file will contain lines in the format: x y z r in cm
|
|
164
|
+
|
|
165
|
+
Returns
|
|
166
|
+
-------
|
|
167
|
+
Path
|
|
168
|
+
The path to the generated telpos file.
|
|
169
|
+
"""
|
|
170
|
+
telpos_file = self.output_directory.joinpath("telpos.dat")
|
|
171
|
+
x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
|
|
172
|
+
"array_element_position_ground"
|
|
173
|
+
)
|
|
174
|
+
x_tel, y_tel, z_tel = [coord.to(u.cm).value for coord in (x_tel, y_tel, z_tel)]
|
|
175
|
+
|
|
176
|
+
radius = self._telescope_model.get_parameter_value_with_unit("telescope_sphere_radius")
|
|
177
|
+
radius = radius.to(u.cm).value # Convert radius to cm
|
|
178
|
+
with telpos_file.open("w", encoding="utf-8") as file:
|
|
179
|
+
file.write(f"{x_tel} {y_tel} {z_tel} {radius}\n")
|
|
180
|
+
|
|
181
|
+
return telpos_file
|
|
182
|
+
|
|
156
183
|
def _make_light_emission_script(self):
|
|
157
184
|
"""
|
|
158
185
|
Create the light emission script to run the light emission package.
|
|
@@ -165,17 +192,28 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
165
192
|
str
|
|
166
193
|
The commands to run the Light Emission package
|
|
167
194
|
"""
|
|
168
|
-
x_cal, y_cal, z_cal = (
|
|
169
|
-
|
|
195
|
+
x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value_with_unit(
|
|
196
|
+
"array_element_position_ground"
|
|
197
|
+
)
|
|
198
|
+
x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
|
|
199
|
+
"array_element_position_ground"
|
|
170
200
|
)
|
|
171
|
-
|
|
172
|
-
|
|
201
|
+
|
|
202
|
+
config_directory = self.io_handler.get_output_directory(
|
|
203
|
+
label=self.label, sub_dir=f"model/{self._site_model.model_version}"
|
|
173
204
|
)
|
|
174
|
-
|
|
175
|
-
|
|
205
|
+
|
|
206
|
+
telpos_file = self._write_telpos_file()
|
|
207
|
+
|
|
208
|
+
command = f"rm {self.output_directory}/"
|
|
176
209
|
command += f"{self.le_application[0]}_{self.le_application[1]}.simtel.gz\n"
|
|
177
210
|
command += str(self._simtel_path.joinpath("sim_telarray/LightEmission/"))
|
|
178
211
|
command += f"/{self.le_application[0]}"
|
|
212
|
+
corsika_observation_level = self._site_model.get_parameter_value_with_unit(
|
|
213
|
+
"corsika_observation_level"
|
|
214
|
+
)
|
|
215
|
+
command += f" -h {corsika_observation_level.to(u.m).value}"
|
|
216
|
+
command += f" --telpos-file {telpos_file}"
|
|
179
217
|
|
|
180
218
|
if self.light_source_type == "led":
|
|
181
219
|
if self.le_application[1] == "variable":
|
|
@@ -188,28 +226,27 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
188
226
|
command += f" -n {self.photons_per_run}"
|
|
189
227
|
|
|
190
228
|
elif self.le_application[1] == "layout":
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
# light_source coordinates relative to telescope
|
|
195
|
-
command += f" -x {x_origin.to(u.cm).value}"
|
|
196
|
-
command += f" -y {y_origin.to(u.cm).value}"
|
|
197
|
-
command += f" -z {z_origin.to(u.cm).value}"
|
|
229
|
+
command += f" -x {x_cal.to(u.cm).value}"
|
|
230
|
+
command += f" -y {y_cal.to(u.cm).value}"
|
|
231
|
+
command += f" -z {z_cal.to(u.cm).value}"
|
|
198
232
|
pointing_vector = self.calibration_pointing_direction()[0]
|
|
199
233
|
command += f" -d {','.join(map(str, pointing_vector))}"
|
|
200
234
|
|
|
201
235
|
command += f" -n {self.photons_per_run}"
|
|
236
|
+
self._logger.info(f"Photons per run: {self.photons_per_run} ")
|
|
202
237
|
|
|
203
|
-
|
|
204
|
-
|
|
238
|
+
laser_wavelength = self._calibration_model.get_parameter_value_with_unit(
|
|
239
|
+
"laser_wavelength"
|
|
240
|
+
)
|
|
241
|
+
command += f" -s {int(laser_wavelength.to(u.nm).value)}"
|
|
205
242
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
f" -p Gauss:{self._calibration_model.get_parameter_value('led_pulse_sigtime')}"
|
|
243
|
+
led_pulse_sigtime = self._calibration_model.get_parameter_value_with_unit(
|
|
244
|
+
"led_pulse_sigtime"
|
|
209
245
|
)
|
|
210
|
-
command += " -
|
|
246
|
+
command += f" -p Gauss:{led_pulse_sigtime.to(u.ns).value}"
|
|
247
|
+
command += " -a isotropic"
|
|
211
248
|
|
|
212
|
-
command += f" -A {
|
|
249
|
+
command += f" -A {config_directory}/"
|
|
213
250
|
command += f"{self._telescope_model.get_parameter_value('atmospheric_profile')}"
|
|
214
251
|
|
|
215
252
|
elif self.light_source_type == "laser":
|
|
@@ -217,11 +254,13 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
217
254
|
command += " --bunches 2500000"
|
|
218
255
|
command += " --step 0.1"
|
|
219
256
|
command += " --bunchsize 1"
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
)
|
|
257
|
+
spectrum = self._calibration_model.get_parameter_value_with_unit("laser_wavelength")
|
|
258
|
+
command += f" --spectrum {int(spectrum.to(u.nm).value)}"
|
|
223
259
|
command += " --lightpulse Gauss:"
|
|
224
|
-
|
|
260
|
+
pulse_sigtime = self._calibration_model.get_parameter_value_with_unit(
|
|
261
|
+
"laser_pulse_sigtime"
|
|
262
|
+
)
|
|
263
|
+
command += f"{pulse_sigtime.to(u.ns).value}"
|
|
225
264
|
x_origin = x_cal - x_tel
|
|
226
265
|
y_origin = y_cal - y_tel
|
|
227
266
|
z_origin = z_cal - z_tel
|
|
@@ -234,8 +273,8 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
234
273
|
command += f" --telescope-theta {angle_theta}"
|
|
235
274
|
command += f" --telescope-phi {angle_phi}"
|
|
236
275
|
command += f" --laser-theta {90 - angles[2]}"
|
|
237
|
-
command += f" --laser-phi {angles[3]}"
|
|
238
|
-
command += f" --atmosphere {
|
|
276
|
+
command += f" --laser-phi {angles[3]}"
|
|
277
|
+
command += f" --atmosphere {config_directory}/"
|
|
239
278
|
command += f"{self._telescope_model.get_parameter_value('atmospheric_profile')}"
|
|
240
279
|
command += f" -o {self.output_directory}/{self.le_application[0]}.iact.gz"
|
|
241
280
|
command += "\n"
|
|
@@ -264,7 +303,10 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
264
303
|
command += " -DNUM_TELESCOPES=1"
|
|
265
304
|
|
|
266
305
|
command += super().get_config_option(
|
|
267
|
-
"altitude",
|
|
306
|
+
"altitude",
|
|
307
|
+
self._site_model.get_parameter_value_with_unit("corsika_observation_level")
|
|
308
|
+
.to(u.m)
|
|
309
|
+
.value,
|
|
268
310
|
)
|
|
269
311
|
command += super().get_config_option(
|
|
270
312
|
"atmospheric_transmission",
|
|
@@ -526,15 +568,18 @@ class SimulatorLightEmission(SimtelRunner):
|
|
|
526
568
|
"""
|
|
527
569
|
if not self.light_emission_config:
|
|
528
570
|
# Layout positions: Use DB coordinates
|
|
529
|
-
x_cal, y_cal, z_cal = self._calibration_model.
|
|
571
|
+
x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value_with_unit(
|
|
530
572
|
"array_element_position_ground"
|
|
531
573
|
)
|
|
532
|
-
|
|
574
|
+
x_cal, y_cal, z_cal = [coord.to(u.m).value for coord in (x_cal, y_cal, z_cal)]
|
|
575
|
+
x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
|
|
533
576
|
"array_element_position_ground"
|
|
534
577
|
)
|
|
578
|
+
x_tel, y_tel, z_tel = [coord.to(u.m).value for coord in (x_tel, y_tel, z_tel)]
|
|
535
579
|
tel_vect = np.array([x_tel, y_tel, z_tel])
|
|
536
580
|
cal_vect = np.array([x_cal, y_cal, z_cal])
|
|
537
581
|
distance = np.linalg.norm(cal_vect - tel_vect)
|
|
582
|
+
print("Distance between telescope and calibration device:", distance * u.m)
|
|
538
583
|
return [distance * u.m]
|
|
539
584
|
|
|
540
585
|
# Variable positions: Calculate distances for all positions
|
|
@@ -61,7 +61,7 @@ class SimulatorRayTracing(SimtelRunner):
|
|
|
61
61
|
self.label = label if label is not None else self.telescope_model.label
|
|
62
62
|
|
|
63
63
|
self.io_handler = io_handler.IOHandler()
|
|
64
|
-
self._base_directory = self.io_handler.get_output_directory(self.label, "
|
|
64
|
+
self._base_directory = self.io_handler.get_output_directory(self.label, "ray_tracing")
|
|
65
65
|
|
|
66
66
|
self.config = (
|
|
67
67
|
self._config_to_namedtuple(config_data)
|
|
@@ -93,7 +93,7 @@ class SimulatorRayTracing(SimtelRunner):
|
|
|
93
93
|
# Files will be named _base_file = self.__dict__['_' + base + 'File']
|
|
94
94
|
for base_name in ["stars", "photons", "log"]:
|
|
95
95
|
file_name = names.generate_file_name(
|
|
96
|
-
file_type=base_name,
|
|
96
|
+
file_type=f"ray_tracing_{base_name}",
|
|
97
97
|
suffix=".log" if base_name == "log" else ".lis",
|
|
98
98
|
site=self.telescope_model.site,
|
|
99
99
|
telescope_model_name=self.telescope_model.name,
|
simtools/simulator.py
CHANGED
|
@@ -12,11 +12,12 @@ import numpy as np
|
|
|
12
12
|
|
|
13
13
|
import simtools.utils.general as gen
|
|
14
14
|
from simtools.corsika.corsika_config import CorsikaConfig
|
|
15
|
-
from simtools.io_operations import io_handler
|
|
15
|
+
from simtools.io_operations import io_handler, io_table_handler
|
|
16
16
|
from simtools.job_execution.job_manager import JobManager
|
|
17
17
|
from simtools.model.array_model import ArrayModel
|
|
18
18
|
from simtools.runners.corsika_runner import CorsikaRunner
|
|
19
19
|
from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
|
|
20
|
+
from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
|
|
20
21
|
from simtools.simtel.simulator_array import SimulatorArray
|
|
21
22
|
from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
|
|
22
23
|
|
|
@@ -72,8 +73,6 @@ class Simulator:
|
|
|
72
73
|
self.runs = self._initialize_run_list()
|
|
73
74
|
self._results = defaultdict(list)
|
|
74
75
|
self._test = self.args_dict.get("test", False)
|
|
75
|
-
self.submit_engine = self.args_dict.get("submit_engine", "local")
|
|
76
|
-
self._submit_options = self.args_dict.get("submit_options", None)
|
|
77
76
|
self._extra_commands = extra_commands
|
|
78
77
|
|
|
79
78
|
self.sim_telarray_seeds = {
|
|
@@ -145,6 +144,7 @@ class Simulator:
|
|
|
145
144
|
],
|
|
146
145
|
"seed_file_name": self.sim_telarray_seeds["seed_file_name"],
|
|
147
146
|
},
|
|
147
|
+
simtel_path=self.args_dict.get("simtel_path", None),
|
|
148
148
|
)
|
|
149
149
|
for version in versions
|
|
150
150
|
]
|
|
@@ -179,7 +179,10 @@ class Simulator:
|
|
|
179
179
|
|
|
180
180
|
def _initialize_run_list(self):
|
|
181
181
|
"""
|
|
182
|
-
Initialize run list using the configuration values
|
|
182
|
+
Initialize run list using the configuration values.
|
|
183
|
+
|
|
184
|
+
Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
|
|
185
|
+
to create a list of run numbers.
|
|
183
186
|
|
|
184
187
|
Returns
|
|
185
188
|
-------
|
|
@@ -189,19 +192,27 @@ class Simulator:
|
|
|
189
192
|
Raises
|
|
190
193
|
------
|
|
191
194
|
KeyError
|
|
192
|
-
If '
|
|
195
|
+
If 'run_number', 'run_number_offset' and 'number_of_runs' are
|
|
196
|
+
not found in the configuration.
|
|
193
197
|
"""
|
|
194
198
|
try:
|
|
199
|
+
offset_run_number = self.args_dict["run_number_offset"] + self.args_dict["run_number"]
|
|
200
|
+
if self.args_dict["number_of_runs"] <= 1:
|
|
201
|
+
return self._validate_run_list_and_range(
|
|
202
|
+
run_list=offset_run_number,
|
|
203
|
+
run_range=None,
|
|
204
|
+
)
|
|
195
205
|
return self._validate_run_list_and_range(
|
|
196
206
|
run_list=None,
|
|
197
207
|
run_range=[
|
|
198
|
-
|
|
199
|
-
|
|
208
|
+
offset_run_number,
|
|
209
|
+
offset_run_number + self.args_dict["number_of_runs"],
|
|
200
210
|
],
|
|
201
211
|
)
|
|
202
212
|
except KeyError as exc:
|
|
203
213
|
self._logger.error(
|
|
204
|
-
"Error in initializing run list
|
|
214
|
+
"Error in initializing run list "
|
|
215
|
+
"(missing 'run_number', 'run_number_offset' or 'number_of_runs')."
|
|
205
216
|
)
|
|
206
217
|
raise exc
|
|
207
218
|
|
|
@@ -301,6 +312,8 @@ class Simulator:
|
|
|
301
312
|
runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
|
|
302
313
|
if runner_class is not CorsikaRunner:
|
|
303
314
|
runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
|
|
315
|
+
if runner_class is CorsikaSimtelRunner:
|
|
316
|
+
runner_args["sequential"] = self.args_dict.get("sequential", False)
|
|
304
317
|
|
|
305
318
|
return runner_class(**runner_args)
|
|
306
319
|
|
|
@@ -330,8 +343,6 @@ class Simulator:
|
|
|
330
343
|
input_file_list: str or list of str
|
|
331
344
|
Single file or list of files of shower simulations.
|
|
332
345
|
"""
|
|
333
|
-
self._logger.info(f"Submission command: {self.submit_engine}")
|
|
334
|
-
|
|
335
346
|
runs_and_files_to_submit = self._get_runs_and_files_to_submit(
|
|
336
347
|
input_file_list=input_file_list
|
|
337
348
|
)
|
|
@@ -345,11 +356,7 @@ class Simulator:
|
|
|
345
356
|
run_number=run_number, input_file=input_file, extra_commands=self._extra_commands
|
|
346
357
|
)
|
|
347
358
|
|
|
348
|
-
job_manager = JobManager(
|
|
349
|
-
submit_engine=self.submit_engine,
|
|
350
|
-
submit_options=self._submit_options,
|
|
351
|
-
test=self._test,
|
|
352
|
-
)
|
|
359
|
+
job_manager = JobManager(test=self._test)
|
|
353
360
|
job_manager.submit(
|
|
354
361
|
run_script=run_script,
|
|
355
362
|
run_out_file=self._simulation_runner.get_file_name(
|
|
@@ -454,73 +461,66 @@ class Simulator:
|
|
|
454
461
|
run number
|
|
455
462
|
|
|
456
463
|
"""
|
|
457
|
-
keys = ["
|
|
464
|
+
keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
|
|
458
465
|
results = {key: [] for key in keys}
|
|
459
466
|
|
|
467
|
+
def get_file_name(name, **kwargs):
|
|
468
|
+
return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
|
|
469
|
+
|
|
460
470
|
if "sim_telarray" in self.simulation_software:
|
|
461
471
|
results["input"].append(str(file))
|
|
462
472
|
|
|
463
|
-
results["sub_out"].append(
|
|
464
|
-
str(
|
|
465
|
-
self._simulation_runner.get_file_name(
|
|
466
|
-
file_type="sub_log",
|
|
467
|
-
mode="out",
|
|
468
|
-
run_number=run_number,
|
|
469
|
-
)
|
|
470
|
-
)
|
|
471
|
-
)
|
|
473
|
+
results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
|
|
472
474
|
|
|
473
|
-
for
|
|
474
|
-
results["
|
|
475
|
-
|
|
476
|
-
self._simulation_runner.get_file_name(
|
|
477
|
-
file_type="output",
|
|
478
|
-
run_number=run_number,
|
|
479
|
-
model_version_index=model_version_index,
|
|
480
|
-
)
|
|
481
|
-
)
|
|
475
|
+
for i in range(len(self.array_models)):
|
|
476
|
+
results["simtel_output"].append(
|
|
477
|
+
get_file_name("simtel_output", run_number=run_number, model_version_index=i)
|
|
482
478
|
)
|
|
479
|
+
|
|
483
480
|
if "sim_telarray" in self.simulation_software:
|
|
484
481
|
results["log"].append(
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
model_version_index=model_version_index,
|
|
491
|
-
)
|
|
482
|
+
get_file_name(
|
|
483
|
+
"log",
|
|
484
|
+
simulation_software="sim_telarray",
|
|
485
|
+
run_number=run_number,
|
|
486
|
+
model_version_index=i,
|
|
492
487
|
)
|
|
493
488
|
)
|
|
494
489
|
results["hist"].append(
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
model_version_index=model_version_index,
|
|
501
|
-
)
|
|
490
|
+
get_file_name(
|
|
491
|
+
"histogram",
|
|
492
|
+
simulation_software="sim_telarray",
|
|
493
|
+
run_number=run_number,
|
|
494
|
+
model_version_index=i,
|
|
502
495
|
)
|
|
503
496
|
)
|
|
497
|
+
results["event_data"].append(
|
|
498
|
+
get_file_name(
|
|
499
|
+
"event_data",
|
|
500
|
+
simulation_software="sim_telarray",
|
|
501
|
+
run_number=run_number,
|
|
502
|
+
model_version_index=i,
|
|
503
|
+
)
|
|
504
|
+
)
|
|
505
|
+
|
|
504
506
|
if "corsika" in self.simulation_software:
|
|
505
507
|
results["corsika_log"].append(
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
model_version_index=model_version_index,
|
|
512
|
-
)
|
|
508
|
+
get_file_name(
|
|
509
|
+
"corsika_log",
|
|
510
|
+
simulation_software="corsika",
|
|
511
|
+
run_number=run_number,
|
|
512
|
+
model_version_index=i,
|
|
513
513
|
)
|
|
514
514
|
)
|
|
515
515
|
|
|
516
516
|
for key in keys:
|
|
517
517
|
self._results[key].extend(results[key])
|
|
518
518
|
|
|
519
|
-
def get_file_list(self, file_type="
|
|
519
|
+
def get_file_list(self, file_type="simtel_output"):
|
|
520
520
|
"""
|
|
521
521
|
Get list of files generated by simulations.
|
|
522
522
|
|
|
523
|
-
Options are "input", "
|
|
523
|
+
Options are "input", "simtel_output", "hist", "log", "corsika_log".
|
|
524
524
|
Not all file types are available for all simulation types.
|
|
525
525
|
Returns an empty list for an unknown file type.
|
|
526
526
|
|
|
@@ -538,11 +538,11 @@ class Simulator:
|
|
|
538
538
|
self._logger.info(f"Getting list of {file_type} files")
|
|
539
539
|
return self._results[file_type]
|
|
540
540
|
|
|
541
|
-
def print_list_of_files(self, file_type="
|
|
541
|
+
def print_list_of_files(self, file_type="simtel_output"):
|
|
542
542
|
"""
|
|
543
543
|
Print list of output files generated by simulations.
|
|
544
544
|
|
|
545
|
-
Options are "input", "
|
|
545
|
+
Options are "input", "simtel_output", "hist", "log".
|
|
546
546
|
|
|
547
547
|
Parameters
|
|
548
548
|
----------
|
|
@@ -636,7 +636,7 @@ class Simulator:
|
|
|
636
636
|
|
|
637
637
|
def save_file_lists(self):
|
|
638
638
|
"""Save files lists for output and log files."""
|
|
639
|
-
for file_type in ["
|
|
639
|
+
for file_type in ["simtel_output", "log", "corsika_log", "hist"]:
|
|
640
640
|
file_name = self.io_handler.get_output_directory(label=self.label).joinpath(
|
|
641
641
|
f"{file_type}_files.txt"
|
|
642
642
|
)
|
|
@@ -649,6 +649,29 @@ class Simulator:
|
|
|
649
649
|
else:
|
|
650
650
|
self._logger.debug(f"No files to save for {file_type} files.")
|
|
651
651
|
|
|
652
|
+
def save_reduced_event_lists(self):
|
|
653
|
+
"""
|
|
654
|
+
Save reduced event lists with event data on simulated and triggered events.
|
|
655
|
+
|
|
656
|
+
The files are saved with the same name as the sim_telarray output file
|
|
657
|
+
but with a 'hdf5' extension.
|
|
658
|
+
"""
|
|
659
|
+
if "sim_telarray" not in self.simulation_software:
|
|
660
|
+
self._logger.warning(
|
|
661
|
+
"Reduced event lists can only be saved for sim_telarray simulations."
|
|
662
|
+
)
|
|
663
|
+
return
|
|
664
|
+
|
|
665
|
+
input_files = self.get_file_list(file_type="simtel_output")
|
|
666
|
+
output_files = self.get_file_list(file_type="event_data")
|
|
667
|
+
for input_file, output_file in zip(input_files, output_files):
|
|
668
|
+
generator = SimtelIOEventDataWriter([input_file])
|
|
669
|
+
io_table_handler.write_tables(
|
|
670
|
+
tables=generator.process_files(),
|
|
671
|
+
output_file=Path(output_file),
|
|
672
|
+
overwrite_existing=True,
|
|
673
|
+
)
|
|
674
|
+
|
|
652
675
|
def pack_for_register(self, directory_for_grid_upload=None):
|
|
653
676
|
"""
|
|
654
677
|
Pack simulation output files for registering on the grid.
|
|
@@ -664,10 +687,15 @@ class Simulator:
|
|
|
664
687
|
self._logger.info(
|
|
665
688
|
f"Packing the output files for registering on the grid ({directory_for_grid_upload})"
|
|
666
689
|
)
|
|
667
|
-
output_files = self.get_file_list(file_type="
|
|
690
|
+
output_files = self.get_file_list(file_type="simtel_output")
|
|
668
691
|
log_files = self.get_file_list(file_type="log")
|
|
669
692
|
corsika_log_files = self.get_file_list(file_type="corsika_log")
|
|
670
693
|
histogram_files = self.get_file_list(file_type="hist")
|
|
694
|
+
reduced_event_files = (
|
|
695
|
+
self.get_file_list(file_type="event_data")
|
|
696
|
+
if self.args_dict.get("save_reduced_event_lists")
|
|
697
|
+
else []
|
|
698
|
+
)
|
|
671
699
|
|
|
672
700
|
directory_for_grid_upload = (
|
|
673
701
|
Path(directory_for_grid_upload)
|
|
@@ -697,11 +725,12 @@ class Simulator:
|
|
|
697
725
|
tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
|
|
698
726
|
|
|
699
727
|
with tarfile.open(tar_file_path, "w:gz") as tar:
|
|
728
|
+
# Add all relevant log, histogram, and CORSIKA log files to the tarball
|
|
700
729
|
files_to_tar = model_logs + model_hists + model_corsika_logs
|
|
701
730
|
for file_to_tar in files_to_tar:
|
|
702
731
|
tar.add(file_to_tar, arcname=Path(file_to_tar).name)
|
|
703
732
|
|
|
704
|
-
for file_to_move in output_files:
|
|
733
|
+
for file_to_move in output_files + reduced_event_files:
|
|
705
734
|
source_file = Path(file_to_move)
|
|
706
735
|
destination_file = directory_for_grid_upload / source_file.name
|
|
707
736
|
if destination_file.exists():
|
|
@@ -717,7 +746,7 @@ class Simulator:
|
|
|
717
746
|
return
|
|
718
747
|
|
|
719
748
|
for model in self.array_models:
|
|
720
|
-
files = self.get_file_list(file_type="
|
|
749
|
+
files = self.get_file_list(file_type="simtel_output")
|
|
721
750
|
output_file = next((f for f in files if model.model_version in f), None)
|
|
722
751
|
if output_file:
|
|
723
752
|
self._logger.info(f"Validating metadata for {output_file}")
|
|
@@ -738,7 +767,6 @@ class Simulator:
|
|
|
738
767
|
----------
|
|
739
768
|
corsika_log_files: list
|
|
740
769
|
List containing the original CORSIKA log file path.
|
|
741
|
-
|
|
742
770
|
"""
|
|
743
771
|
original_log = Path(corsika_log_files[0])
|
|
744
772
|
# Find which model version the original log belongs to
|
|
@@ -759,15 +787,20 @@ class Simulator:
|
|
|
759
787
|
original_version, model.model_version
|
|
760
788
|
)
|
|
761
789
|
|
|
762
|
-
with gzip.open(new_log, "wt") as new_file:
|
|
763
|
-
|
|
790
|
+
with gzip.open(new_log, "wt", encoding="utf-8") as new_file:
|
|
791
|
+
# Write the header to the new file
|
|
792
|
+
header = (
|
|
764
793
|
f"###############################################################\n"
|
|
765
794
|
f"Copy of CORSIKA log file from model version {original_version}.\n"
|
|
766
795
|
f"Applicable also for {model.model_version} (same CORSIKA configuration,\n"
|
|
767
796
|
f"different sim_telarray model versions in the same run).\n"
|
|
768
797
|
f"###############################################################\n\n"
|
|
769
798
|
)
|
|
770
|
-
|
|
771
|
-
|
|
799
|
+
new_file.write(header)
|
|
800
|
+
|
|
801
|
+
# Copy the content of the original log file, ignoring invalid characters
|
|
802
|
+
with gzip.open(original_log, "rt", encoding="utf-8", errors="ignore") as orig_file:
|
|
803
|
+
for line in orig_file:
|
|
804
|
+
new_file.write(line)
|
|
772
805
|
|
|
773
806
|
corsika_log_files.append(str(new_log))
|
|
@@ -102,7 +102,10 @@ def _validate_output_path_and_file(config, integration_file_tests):
|
|
|
102
102
|
|
|
103
103
|
output_file_path = Path(output_path) / file_test["FILE"]
|
|
104
104
|
_logger.info(f"Checking path: {output_file_path}")
|
|
105
|
-
|
|
105
|
+
try:
|
|
106
|
+
assert output_file_path.exists()
|
|
107
|
+
except AssertionError as exc:
|
|
108
|
+
raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
|
|
106
109
|
|
|
107
110
|
if "EXPECTED_OUTPUT" in file_test:
|
|
108
111
|
assert assertions.check_output_from_sim_telarray(
|
simtools/utils/general.py
CHANGED
|
@@ -178,7 +178,7 @@ def _collect_data_from_different_file_types(file, file_name, suffix, yaml_docume
|
|
|
178
178
|
"""Collect data from different file types."""
|
|
179
179
|
if suffix == ".json":
|
|
180
180
|
return json.load(file)
|
|
181
|
-
if suffix
|
|
181
|
+
if suffix in (".list", ".txt"):
|
|
182
182
|
return [line.strip() for line in file.readlines()]
|
|
183
183
|
if suffix in [".yml", ".yaml"]:
|
|
184
184
|
return _collect_data_from_yaml_file(file, file_name, yaml_document)
|
simtools/utils/names.py
CHANGED
|
@@ -658,7 +658,7 @@ def generate_file_name(
|
|
|
658
658
|
"""
|
|
659
659
|
Generate a file name for output, config, or plotting.
|
|
660
660
|
|
|
661
|
-
Used e.g., to generate
|
|
661
|
+
Used e.g., to generate camera_efficiency and ray_tracing output files.
|
|
662
662
|
|
|
663
663
|
Parameters
|
|
664
664
|
----------
|
|
@@ -690,10 +690,10 @@ def generate_file_name(
|
|
|
690
690
|
str
|
|
691
691
|
File name.
|
|
692
692
|
"""
|
|
693
|
-
name = f"{file_type}
|
|
694
|
-
name += f"
|
|
695
|
-
name += f"
|
|
696
|
-
name += f"
|
|
693
|
+
name = f"{file_type}_{site}_{telescope_model_name}"
|
|
694
|
+
name += f"_d{source_distance:.1f}km" if source_distance is not None else ""
|
|
695
|
+
name += f"_za{float(zenith_angle):.1f}deg"
|
|
696
|
+
name += f"_off{off_axis_angle:.3f}deg" if off_axis_angle is not None else ""
|
|
697
697
|
name += f"_azm{round(azimuth_angle):03}deg" if azimuth_angle is not None else ""
|
|
698
698
|
name += f"_mirror{mirror_number}" if mirror_number is not None else ""
|
|
699
699
|
name += f"_{label}" if label is not None else ""
|