gammasimtools 0.16.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/METADATA +4 -2
  2. {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/RECORD +60 -54
  3. {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/WHEEL +1 -1
  4. {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/entry_points.txt +3 -1
  5. simtools/_version.py +2 -2
  6. simtools/applications/derive_ctao_array_layouts.py +5 -5
  7. simtools/applications/generate_simtel_event_data.py +36 -46
  8. simtools/applications/merge_tables.py +104 -0
  9. simtools/applications/plot_array_layout.py +145 -258
  10. simtools/applications/production_derive_corsika_limits.py +35 -220
  11. simtools/applications/production_derive_statistics.py +77 -43
  12. simtools/applications/simulate_light_emission.py +1 -0
  13. simtools/applications/simulate_prod.py +30 -18
  14. simtools/applications/simulate_prod_htcondor_generator.py +0 -1
  15. simtools/applications/submit_array_layouts.py +93 -0
  16. simtools/applications/verify_simulation_model_production_tables.py +52 -0
  17. simtools/camera/camera_efficiency.py +3 -3
  18. simtools/configuration/commandline_parser.py +28 -34
  19. simtools/configuration/configurator.py +0 -4
  20. simtools/corsika/corsika_config.py +17 -12
  21. simtools/corsika/primary_particle.py +46 -13
  22. simtools/data_model/metadata_collector.py +7 -3
  23. simtools/db/db_handler.py +11 -11
  24. simtools/db/db_model_upload.py +2 -2
  25. simtools/io_operations/io_handler.py +2 -2
  26. simtools/io_operations/io_table_handler.py +345 -0
  27. simtools/job_execution/htcondor_script_generator.py +2 -2
  28. simtools/job_execution/job_manager.py +7 -121
  29. simtools/layout/array_layout_utils.py +385 -0
  30. simtools/model/array_model.py +5 -0
  31. simtools/model/model_repository.py +134 -0
  32. simtools/production_configuration/{calculate_statistical_errors_grid_point.py → calculate_statistical_uncertainties_grid_point.py} +101 -112
  33. simtools/production_configuration/derive_corsika_limits.py +239 -111
  34. simtools/production_configuration/derive_corsika_limits_grid.py +189 -0
  35. simtools/production_configuration/derive_production_statistics.py +57 -26
  36. simtools/production_configuration/derive_production_statistics_handler.py +70 -37
  37. simtools/production_configuration/interpolation_handler.py +296 -94
  38. simtools/ray_tracing/ray_tracing.py +7 -6
  39. simtools/reporting/docs_read_parameters.py +104 -62
  40. simtools/runners/corsika_simtel_runner.py +4 -1
  41. simtools/runners/runner_services.py +5 -4
  42. simtools/schemas/model_parameters/dsum_threshold.schema.yml +41 -0
  43. simtools/schemas/production_configuration_metrics.schema.yml +2 -2
  44. simtools/simtel/simtel_config_writer.py +34 -14
  45. simtools/simtel/simtel_io_event_reader.py +301 -194
  46. simtools/simtel/simtel_io_event_writer.py +207 -227
  47. simtools/simtel/simtel_io_file_info.py +9 -4
  48. simtools/simtel/simtel_io_metadata.py +20 -5
  49. simtools/simtel/simulator_array.py +2 -2
  50. simtools/simtel/simulator_light_emission.py +79 -34
  51. simtools/simtel/simulator_ray_tracing.py +2 -2
  52. simtools/simulator.py +101 -68
  53. simtools/testing/validate_output.py +4 -1
  54. simtools/utils/general.py +1 -1
  55. simtools/utils/names.py +5 -5
  56. simtools/visualization/plot_array_layout.py +242 -0
  57. simtools/visualization/plot_pixels.py +681 -0
  58. simtools/visualization/visualize.py +3 -219
  59. simtools/applications/production_generate_simulation_config.py +0 -152
  60. simtools/layout/ctao_array_layouts.py +0 -172
  61. simtools/production_configuration/generate_simulation_config.py +0 -158
  62. {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/licenses/LICENSE +0 -0
  63. {gammasimtools-0.16.0.dist-info → gammasimtools-0.17.0.dist-info}/top_level.txt +0 -0
@@ -46,6 +46,7 @@ class SimulatorLightEmission(SimtelRunner):
46
46
  self._telescope_model = telescope_model
47
47
 
48
48
  self.label = label if label is not None else self._telescope_model.label
49
+ self.test = test
49
50
 
50
51
  self._calibration_model = calibration_model
51
52
  self._site_model = site_model
@@ -56,7 +57,9 @@ class SimulatorLightEmission(SimtelRunner):
56
57
  self._rep_number = 0
57
58
  self.runs = 1
58
59
  self.photons_per_run = (
59
- self._calibration_model.get_parameter_value("photons_per_run") if not test else 1e7
60
+ (self._calibration_model.get_parameter_value("photons_per_run"))
61
+ if not self.test
62
+ else 1e8
60
63
  )
61
64
 
62
65
  self.le_application = le_application
@@ -64,7 +67,6 @@ class SimulatorLightEmission(SimtelRunner):
64
67
  self.distance = None
65
68
  self.light_source_type = light_source_type
66
69
  self._telescope_model.write_sim_telarray_config_file(additional_model=site_model)
67
- self.test = test
68
70
 
69
71
  @staticmethod
70
72
  def light_emission_default_configuration():
@@ -119,14 +121,15 @@ class SimulatorLightEmission(SimtelRunner):
119
121
  list
120
122
  The pointing vector from the calibration device to the telescope.
121
123
  """
122
- # use DB coordinates later
123
- x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value(
124
+ x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value_with_unit(
124
125
  "array_element_position_ground"
125
126
  )
127
+ x_cal, y_cal, z_cal = [coord.to(u.m).value for coord in (x_cal, y_cal, z_cal)]
126
128
  cal_vect = np.array([x_cal, y_cal, z_cal])
127
- x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value(
129
+ x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
128
130
  "array_element_position_ground"
129
131
  )
132
+ x_tel, y_tel, z_tel = [coord.to(u.m).value for coord in (x_tel, y_tel, z_tel)]
130
133
 
131
134
  tel_vect = np.array([x_tel, y_tel, z_tel])
132
135
 
@@ -153,6 +156,30 @@ class SimulatorLightEmission(SimtelRunner):
153
156
  )
154
157
  return pointing_vector.tolist(), [tel_theta, tel_phi, laser_theta, laser_phi]
155
158
 
159
+ def _write_telpos_file(self):
160
+ """
161
+ Write the telescope positions to a telpos file.
162
+
163
+ The file will contain lines in the format: x y z r in cm
164
+
165
+ Returns
166
+ -------
167
+ Path
168
+ The path to the generated telpos file.
169
+ """
170
+ telpos_file = self.output_directory.joinpath("telpos.dat")
171
+ x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
172
+ "array_element_position_ground"
173
+ )
174
+ x_tel, y_tel, z_tel = [coord.to(u.cm).value for coord in (x_tel, y_tel, z_tel)]
175
+
176
+ radius = self._telescope_model.get_parameter_value_with_unit("telescope_sphere_radius")
177
+ radius = radius.to(u.cm).value # Convert radius to cm
178
+ with telpos_file.open("w", encoding="utf-8") as file:
179
+ file.write(f"{x_tel} {y_tel} {z_tel} {radius}\n")
180
+
181
+ return telpos_file
182
+
156
183
  def _make_light_emission_script(self):
157
184
  """
158
185
  Create the light emission script to run the light emission package.
@@ -165,17 +192,28 @@ class SimulatorLightEmission(SimtelRunner):
165
192
  str
166
193
  The commands to run the Light Emission package
167
194
  """
168
- x_cal, y_cal, z_cal = (
169
- self._calibration_model.get_parameter_value("array_element_position_ground") * u.m
195
+ x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value_with_unit(
196
+ "array_element_position_ground"
197
+ )
198
+ x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
199
+ "array_element_position_ground"
170
200
  )
171
- x_tel, y_tel, z_tel = (
172
- self._telescope_model.get_parameter_value("array_element_position_ground") * u.m
201
+
202
+ config_directory = self.io_handler.get_output_directory(
203
+ label=self.label, sub_dir=f"model/{self._site_model.model_version}"
173
204
  )
174
- _model_directory = self.io_handler.get_output_directory(self.label, "model")
175
- command = f" rm {self.output_directory}/"
205
+
206
+ telpos_file = self._write_telpos_file()
207
+
208
+ command = f"rm {self.output_directory}/"
176
209
  command += f"{self.le_application[0]}_{self.le_application[1]}.simtel.gz\n"
177
210
  command += str(self._simtel_path.joinpath("sim_telarray/LightEmission/"))
178
211
  command += f"/{self.le_application[0]}"
212
+ corsika_observation_level = self._site_model.get_parameter_value_with_unit(
213
+ "corsika_observation_level"
214
+ )
215
+ command += f" -h {corsika_observation_level.to(u.m).value}"
216
+ command += f" --telpos-file {telpos_file}"
179
217
 
180
218
  if self.light_source_type == "led":
181
219
  if self.le_application[1] == "variable":
@@ -188,28 +226,27 @@ class SimulatorLightEmission(SimtelRunner):
188
226
  command += f" -n {self.photons_per_run}"
189
227
 
190
228
  elif self.le_application[1] == "layout":
191
- x_origin = x_cal - x_tel
192
- y_origin = y_cal - y_tel
193
- z_origin = z_cal - z_tel
194
- # light_source coordinates relative to telescope
195
- command += f" -x {x_origin.to(u.cm).value}"
196
- command += f" -y {y_origin.to(u.cm).value}"
197
- command += f" -z {z_origin.to(u.cm).value}"
229
+ command += f" -x {x_cal.to(u.cm).value}"
230
+ command += f" -y {y_cal.to(u.cm).value}"
231
+ command += f" -z {z_cal.to(u.cm).value}"
198
232
  pointing_vector = self.calibration_pointing_direction()[0]
199
233
  command += f" -d {','.join(map(str, pointing_vector))}"
200
234
 
201
235
  command += f" -n {self.photons_per_run}"
236
+ self._logger.info(f"Photons per run: {self.photons_per_run} ")
202
237
 
203
- # same wavelength as for laser
204
- command += f" -s {self._calibration_model.get_parameter_value('laser_wavelength')}"
238
+ laser_wavelength = self._calibration_model.get_parameter_value_with_unit(
239
+ "laser_wavelength"
240
+ )
241
+ command += f" -s {int(laser_wavelength.to(u.nm).value)}"
205
242
 
206
- # pulse
207
- command += (
208
- f" -p Gauss:{self._calibration_model.get_parameter_value('led_pulse_sigtime')}"
243
+ led_pulse_sigtime = self._calibration_model.get_parameter_value_with_unit(
244
+ "led_pulse_sigtime"
209
245
  )
210
- command += " -a isotropic" # angular distribution
246
+ command += f" -p Gauss:{led_pulse_sigtime.to(u.ns).value}"
247
+ command += " -a isotropic"
211
248
 
212
- command += f" -A {_model_directory}/"
249
+ command += f" -A {config_directory}/"
213
250
  command += f"{self._telescope_model.get_parameter_value('atmospheric_profile')}"
214
251
 
215
252
  elif self.light_source_type == "laser":
@@ -217,11 +254,13 @@ class SimulatorLightEmission(SimtelRunner):
217
254
  command += " --bunches 2500000"
218
255
  command += " --step 0.1"
219
256
  command += " --bunchsize 1"
220
- command += (
221
- f" --spectrum {self._calibration_model.get_parameter_value('laser_wavelength')}"
222
- )
257
+ spectrum = self._calibration_model.get_parameter_value_with_unit("laser_wavelength")
258
+ command += f" --spectrum {int(spectrum.to(u.nm).value)}"
223
259
  command += " --lightpulse Gauss:"
224
- command += f"{self._calibration_model.get_parameter_value('laser_pulse_sigtime')}"
260
+ pulse_sigtime = self._calibration_model.get_parameter_value_with_unit(
261
+ "laser_pulse_sigtime"
262
+ )
263
+ command += f"{pulse_sigtime.to(u.ns).value}"
225
264
  x_origin = x_cal - x_tel
226
265
  y_origin = y_cal - y_tel
227
266
  z_origin = z_cal - z_tel
@@ -234,8 +273,8 @@ class SimulatorLightEmission(SimtelRunner):
234
273
  command += f" --telescope-theta {angle_theta}"
235
274
  command += f" --telescope-phi {angle_phi}"
236
275
  command += f" --laser-theta {90 - angles[2]}"
237
- command += f" --laser-phi {angles[3]}" # convention north (x) towards east (-y)
238
- command += f" --atmosphere {_model_directory}/"
276
+ command += f" --laser-phi {angles[3]}"
277
+ command += f" --atmosphere {config_directory}/"
239
278
  command += f"{self._telescope_model.get_parameter_value('atmospheric_profile')}"
240
279
  command += f" -o {self.output_directory}/{self.le_application[0]}.iact.gz"
241
280
  command += "\n"
@@ -264,7 +303,10 @@ class SimulatorLightEmission(SimtelRunner):
264
303
  command += " -DNUM_TELESCOPES=1"
265
304
 
266
305
  command += super().get_config_option(
267
- "altitude", self._site_model.get_parameter_value("corsika_observation_level")
306
+ "altitude",
307
+ self._site_model.get_parameter_value_with_unit("corsika_observation_level")
308
+ .to(u.m)
309
+ .value,
268
310
  )
269
311
  command += super().get_config_option(
270
312
  "atmospheric_transmission",
@@ -526,15 +568,18 @@ class SimulatorLightEmission(SimtelRunner):
526
568
  """
527
569
  if not self.light_emission_config:
528
570
  # Layout positions: Use DB coordinates
529
- x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value(
571
+ x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value_with_unit(
530
572
  "array_element_position_ground"
531
573
  )
532
- x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value(
574
+ x_cal, y_cal, z_cal = [coord.to(u.m).value for coord in (x_cal, y_cal, z_cal)]
575
+ x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value_with_unit(
533
576
  "array_element_position_ground"
534
577
  )
578
+ x_tel, y_tel, z_tel = [coord.to(u.m).value for coord in (x_tel, y_tel, z_tel)]
535
579
  tel_vect = np.array([x_tel, y_tel, z_tel])
536
580
  cal_vect = np.array([x_cal, y_cal, z_cal])
537
581
  distance = np.linalg.norm(cal_vect - tel_vect)
582
+ print("Distance between telescope and calibration device:", distance * u.m)
538
583
  return [distance * u.m]
539
584
 
540
585
  # Variable positions: Calculate distances for all positions
@@ -61,7 +61,7 @@ class SimulatorRayTracing(SimtelRunner):
61
61
  self.label = label if label is not None else self.telescope_model.label
62
62
 
63
63
  self.io_handler = io_handler.IOHandler()
64
- self._base_directory = self.io_handler.get_output_directory(self.label, "ray-tracing")
64
+ self._base_directory = self.io_handler.get_output_directory(self.label, "ray_tracing")
65
65
 
66
66
  self.config = (
67
67
  self._config_to_namedtuple(config_data)
@@ -93,7 +93,7 @@ class SimulatorRayTracing(SimtelRunner):
93
93
  # Files will be named _base_file = self.__dict__['_' + base + 'File']
94
94
  for base_name in ["stars", "photons", "log"]:
95
95
  file_name = names.generate_file_name(
96
- file_type=base_name,
96
+ file_type=f"ray_tracing_{base_name}",
97
97
  suffix=".log" if base_name == "log" else ".lis",
98
98
  site=self.telescope_model.site,
99
99
  telescope_model_name=self.telescope_model.name,
simtools/simulator.py CHANGED
@@ -12,11 +12,12 @@ import numpy as np
12
12
 
13
13
  import simtools.utils.general as gen
14
14
  from simtools.corsika.corsika_config import CorsikaConfig
15
- from simtools.io_operations import io_handler
15
+ from simtools.io_operations import io_handler, io_table_handler
16
16
  from simtools.job_execution.job_manager import JobManager
17
17
  from simtools.model.array_model import ArrayModel
18
18
  from simtools.runners.corsika_runner import CorsikaRunner
19
19
  from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
20
+ from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
20
21
  from simtools.simtel.simulator_array import SimulatorArray
21
22
  from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
22
23
 
@@ -72,8 +73,6 @@ class Simulator:
72
73
  self.runs = self._initialize_run_list()
73
74
  self._results = defaultdict(list)
74
75
  self._test = self.args_dict.get("test", False)
75
- self.submit_engine = self.args_dict.get("submit_engine", "local")
76
- self._submit_options = self.args_dict.get("submit_options", None)
77
76
  self._extra_commands = extra_commands
78
77
 
79
78
  self.sim_telarray_seeds = {
@@ -145,6 +144,7 @@ class Simulator:
145
144
  ],
146
145
  "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
147
146
  },
147
+ simtel_path=self.args_dict.get("simtel_path", None),
148
148
  )
149
149
  for version in versions
150
150
  ]
@@ -179,7 +179,10 @@ class Simulator:
179
179
 
180
180
  def _initialize_run_list(self):
181
181
  """
182
- Initialize run list using the configuration values 'run_number_start' and 'number_of_runs'.
182
+ Initialize run list using the configuration values.
183
+
184
+ Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
185
+ to create a list of run numbers.
183
186
 
184
187
  Returns
185
188
  -------
@@ -189,19 +192,27 @@ class Simulator:
189
192
  Raises
190
193
  ------
191
194
  KeyError
192
- If 'run_number_start' or 'number_of_runs' are not found in the configuration.
195
+ If 'run_number', 'run_number_offset' and 'number_of_runs' are
196
+ not found in the configuration.
193
197
  """
194
198
  try:
199
+ offset_run_number = self.args_dict["run_number_offset"] + self.args_dict["run_number"]
200
+ if self.args_dict["number_of_runs"] <= 1:
201
+ return self._validate_run_list_and_range(
202
+ run_list=offset_run_number,
203
+ run_range=None,
204
+ )
195
205
  return self._validate_run_list_and_range(
196
206
  run_list=None,
197
207
  run_range=[
198
- self.args_dict["run_number_start"],
199
- self.args_dict["run_number_start"] + self.args_dict["number_of_runs"],
208
+ offset_run_number,
209
+ offset_run_number + self.args_dict["number_of_runs"],
200
210
  ],
201
211
  )
202
212
  except KeyError as exc:
203
213
  self._logger.error(
204
- "Error in initializing run list (missing 'run_number_start' or 'number_of_runs')"
214
+ "Error in initializing run list "
215
+ "(missing 'run_number', 'run_number_offset' or 'number_of_runs')."
205
216
  )
206
217
  raise exc
207
218
 
@@ -301,6 +312,8 @@ class Simulator:
301
312
  runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
302
313
  if runner_class is not CorsikaRunner:
303
314
  runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
315
+ if runner_class is CorsikaSimtelRunner:
316
+ runner_args["sequential"] = self.args_dict.get("sequential", False)
304
317
 
305
318
  return runner_class(**runner_args)
306
319
 
@@ -330,8 +343,6 @@ class Simulator:
330
343
  input_file_list: str or list of str
331
344
  Single file or list of files of shower simulations.
332
345
  """
333
- self._logger.info(f"Submission command: {self.submit_engine}")
334
-
335
346
  runs_and_files_to_submit = self._get_runs_and_files_to_submit(
336
347
  input_file_list=input_file_list
337
348
  )
@@ -345,11 +356,7 @@ class Simulator:
345
356
  run_number=run_number, input_file=input_file, extra_commands=self._extra_commands
346
357
  )
347
358
 
348
- job_manager = JobManager(
349
- submit_engine=self.submit_engine,
350
- submit_options=self._submit_options,
351
- test=self._test,
352
- )
359
+ job_manager = JobManager(test=self._test)
353
360
  job_manager.submit(
354
361
  run_script=run_script,
355
362
  run_out_file=self._simulation_runner.get_file_name(
@@ -454,73 +461,66 @@ class Simulator:
454
461
  run number
455
462
 
456
463
  """
457
- keys = ["output", "sub_out", "log", "input", "hist", "corsika_log"]
464
+ keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
458
465
  results = {key: [] for key in keys}
459
466
 
467
+ def get_file_name(name, **kwargs):
468
+ return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
469
+
460
470
  if "sim_telarray" in self.simulation_software:
461
471
  results["input"].append(str(file))
462
472
 
463
- results["sub_out"].append(
464
- str(
465
- self._simulation_runner.get_file_name(
466
- file_type="sub_log",
467
- mode="out",
468
- run_number=run_number,
469
- )
470
- )
471
- )
473
+ results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
472
474
 
473
- for model_version_index, _ in enumerate(self.array_models):
474
- results["output"].append(
475
- str(
476
- self._simulation_runner.get_file_name(
477
- file_type="output",
478
- run_number=run_number,
479
- model_version_index=model_version_index,
480
- )
481
- )
475
+ for i in range(len(self.array_models)):
476
+ results["simtel_output"].append(
477
+ get_file_name("simtel_output", run_number=run_number, model_version_index=i)
482
478
  )
479
+
483
480
  if "sim_telarray" in self.simulation_software:
484
481
  results["log"].append(
485
- str(
486
- self._simulation_runner.get_file_name(
487
- file_type="log",
488
- simulation_software="sim_telarray",
489
- run_number=run_number,
490
- model_version_index=model_version_index,
491
- )
482
+ get_file_name(
483
+ "log",
484
+ simulation_software="sim_telarray",
485
+ run_number=run_number,
486
+ model_version_index=i,
492
487
  )
493
488
  )
494
489
  results["hist"].append(
495
- str(
496
- self._simulation_runner.get_file_name(
497
- file_type="histogram",
498
- simulation_software="sim_telarray",
499
- run_number=run_number,
500
- model_version_index=model_version_index,
501
- )
490
+ get_file_name(
491
+ "histogram",
492
+ simulation_software="sim_telarray",
493
+ run_number=run_number,
494
+ model_version_index=i,
502
495
  )
503
496
  )
497
+ results["event_data"].append(
498
+ get_file_name(
499
+ "event_data",
500
+ simulation_software="sim_telarray",
501
+ run_number=run_number,
502
+ model_version_index=i,
503
+ )
504
+ )
505
+
504
506
  if "corsika" in self.simulation_software:
505
507
  results["corsika_log"].append(
506
- str(
507
- self._simulation_runner.get_file_name(
508
- file_type="corsika_log",
509
- simulation_software="corsika",
510
- run_number=run_number,
511
- model_version_index=model_version_index,
512
- )
508
+ get_file_name(
509
+ "corsika_log",
510
+ simulation_software="corsika",
511
+ run_number=run_number,
512
+ model_version_index=i,
513
513
  )
514
514
  )
515
515
 
516
516
  for key in keys:
517
517
  self._results[key].extend(results[key])
518
518
 
519
- def get_file_list(self, file_type="output"):
519
+ def get_file_list(self, file_type="simtel_output"):
520
520
  """
521
521
  Get list of files generated by simulations.
522
522
 
523
- Options are "input", "output", "hist", "log", "corsika_log".
523
+ Options are "input", "simtel_output", "hist", "log", "corsika_log".
524
524
  Not all file types are available for all simulation types.
525
525
  Returns an empty list for an unknown file type.
526
526
 
@@ -538,11 +538,11 @@ class Simulator:
538
538
  self._logger.info(f"Getting list of {file_type} files")
539
539
  return self._results[file_type]
540
540
 
541
- def print_list_of_files(self, file_type="output"):
541
+ def print_list_of_files(self, file_type="simtel_output"):
542
542
  """
543
543
  Print list of output files generated by simulations.
544
544
 
545
- Options are "input", "output", "hist", "log".
545
+ Options are "input", "simtel_output", "hist", "log".
546
546
 
547
547
  Parameters
548
548
  ----------
@@ -636,7 +636,7 @@ class Simulator:
636
636
 
637
637
  def save_file_lists(self):
638
638
  """Save files lists for output and log files."""
639
- for file_type in ["output", "log", "corsika_log", "hist"]:
639
+ for file_type in ["simtel_output", "log", "corsika_log", "hist"]:
640
640
  file_name = self.io_handler.get_output_directory(label=self.label).joinpath(
641
641
  f"{file_type}_files.txt"
642
642
  )
@@ -649,6 +649,29 @@ class Simulator:
649
649
  else:
650
650
  self._logger.debug(f"No files to save for {file_type} files.")
651
651
 
652
+ def save_reduced_event_lists(self):
653
+ """
654
+ Save reduced event lists with event data on simulated and triggered events.
655
+
656
+ The files are saved with the same name as the sim_telarray output file
657
+ but with a 'hdf5' extension.
658
+ """
659
+ if "sim_telarray" not in self.simulation_software:
660
+ self._logger.warning(
661
+ "Reduced event lists can only be saved for sim_telarray simulations."
662
+ )
663
+ return
664
+
665
+ input_files = self.get_file_list(file_type="simtel_output")
666
+ output_files = self.get_file_list(file_type="event_data")
667
+ for input_file, output_file in zip(input_files, output_files):
668
+ generator = SimtelIOEventDataWriter([input_file])
669
+ io_table_handler.write_tables(
670
+ tables=generator.process_files(),
671
+ output_file=Path(output_file),
672
+ overwrite_existing=True,
673
+ )
674
+
652
675
  def pack_for_register(self, directory_for_grid_upload=None):
653
676
  """
654
677
  Pack simulation output files for registering on the grid.
@@ -664,10 +687,15 @@ class Simulator:
664
687
  self._logger.info(
665
688
  f"Packing the output files for registering on the grid ({directory_for_grid_upload})"
666
689
  )
667
- output_files = self.get_file_list(file_type="output")
690
+ output_files = self.get_file_list(file_type="simtel_output")
668
691
  log_files = self.get_file_list(file_type="log")
669
692
  corsika_log_files = self.get_file_list(file_type="corsika_log")
670
693
  histogram_files = self.get_file_list(file_type="hist")
694
+ reduced_event_files = (
695
+ self.get_file_list(file_type="event_data")
696
+ if self.args_dict.get("save_reduced_event_lists")
697
+ else []
698
+ )
671
699
 
672
700
  directory_for_grid_upload = (
673
701
  Path(directory_for_grid_upload)
@@ -697,11 +725,12 @@ class Simulator:
697
725
  tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
698
726
 
699
727
  with tarfile.open(tar_file_path, "w:gz") as tar:
728
+ # Add all relevant log, histogram, and CORSIKA log files to the tarball
700
729
  files_to_tar = model_logs + model_hists + model_corsika_logs
701
730
  for file_to_tar in files_to_tar:
702
731
  tar.add(file_to_tar, arcname=Path(file_to_tar).name)
703
732
 
704
- for file_to_move in output_files:
733
+ for file_to_move in output_files + reduced_event_files:
705
734
  source_file = Path(file_to_move)
706
735
  destination_file = directory_for_grid_upload / source_file.name
707
736
  if destination_file.exists():
@@ -717,7 +746,7 @@ class Simulator:
717
746
  return
718
747
 
719
748
  for model in self.array_models:
720
- files = self.get_file_list(file_type="output")
749
+ files = self.get_file_list(file_type="simtel_output")
721
750
  output_file = next((f for f in files if model.model_version in f), None)
722
751
  if output_file:
723
752
  self._logger.info(f"Validating metadata for {output_file}")
@@ -738,7 +767,6 @@ class Simulator:
738
767
  ----------
739
768
  corsika_log_files: list
740
769
  List containing the original CORSIKA log file path.
741
-
742
770
  """
743
771
  original_log = Path(corsika_log_files[0])
744
772
  # Find which model version the original log belongs to
@@ -759,15 +787,20 @@ class Simulator:
759
787
  original_version, model.model_version
760
788
  )
761
789
 
762
- with gzip.open(new_log, "wt") as new_file:
763
- new_file.write(
790
+ with gzip.open(new_log, "wt", encoding="utf-8") as new_file:
791
+ # Write the header to the new file
792
+ header = (
764
793
  f"###############################################################\n"
765
794
  f"Copy of CORSIKA log file from model version {original_version}.\n"
766
795
  f"Applicable also for {model.model_version} (same CORSIKA configuration,\n"
767
796
  f"different sim_telarray model versions in the same run).\n"
768
797
  f"###############################################################\n\n"
769
798
  )
770
- with gzip.open(original_log, "rt") as orig_file:
771
- shutil.copyfileobj(orig_file, new_file)
799
+ new_file.write(header)
800
+
801
+ # Copy the content of the original log file, ignoring invalid characters
802
+ with gzip.open(original_log, "rt", encoding="utf-8", errors="ignore") as orig_file:
803
+ for line in orig_file:
804
+ new_file.write(line)
772
805
 
773
806
  corsika_log_files.append(str(new_log))
@@ -102,7 +102,10 @@ def _validate_output_path_and_file(config, integration_file_tests):
102
102
 
103
103
  output_file_path = Path(output_path) / file_test["FILE"]
104
104
  _logger.info(f"Checking path: {output_file_path}")
105
- assert output_file_path.exists()
105
+ try:
106
+ assert output_file_path.exists()
107
+ except AssertionError as exc:
108
+ raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
106
109
 
107
110
  if "EXPECTED_OUTPUT" in file_test:
108
111
  assert assertions.check_output_from_sim_telarray(
simtools/utils/general.py CHANGED
@@ -178,7 +178,7 @@ def _collect_data_from_different_file_types(file, file_name, suffix, yaml_docume
178
178
  """Collect data from different file types."""
179
179
  if suffix == ".json":
180
180
  return json.load(file)
181
- if suffix == ".list":
181
+ if suffix in (".list", ".txt"):
182
182
  return [line.strip() for line in file.readlines()]
183
183
  if suffix in [".yml", ".yaml"]:
184
184
  return _collect_data_from_yaml_file(file, file_name, yaml_document)
simtools/utils/names.py CHANGED
@@ -658,7 +658,7 @@ def generate_file_name(
658
658
  """
659
659
  Generate a file name for output, config, or plotting.
660
660
 
661
- Used e.g., to generate camera-efficiency and ray-tracing output files.
661
+ Used e.g., to generate camera_efficiency and ray_tracing output files.
662
662
 
663
663
  Parameters
664
664
  ----------
@@ -690,10 +690,10 @@ def generate_file_name(
690
690
  str
691
691
  File name.
692
692
  """
693
- name = f"{file_type}-{site}-{telescope_model_name}"
694
- name += f"-d{source_distance:.1f}km" if source_distance is not None else ""
695
- name += f"-za{float(zenith_angle):.1f}deg"
696
- name += f"-off{off_axis_angle:.3f}deg" if off_axis_angle is not None else ""
693
+ name = f"{file_type}_{site}_{telescope_model_name}"
694
+ name += f"_d{source_distance:.1f}km" if source_distance is not None else ""
695
+ name += f"_za{float(zenith_angle):.1f}deg"
696
+ name += f"_off{off_axis_angle:.3f}deg" if off_axis_angle is not None else ""
697
697
  name += f"_azm{round(azimuth_angle):03}deg" if azimuth_angle is not None else ""
698
698
  name += f"_mirror{mirror_number}" if mirror_number is not None else ""
699
699
  name += f"_{label}" if label is not None else ""