gammasimtools 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/METADATA +2 -1
  2. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/RECORD +134 -130
  3. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/entry_points.txt +3 -1
  4. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/licenses/LICENSE +1 -1
  5. simtools/_version.py +2 -2
  6. simtools/application_control.py +78 -0
  7. simtools/applications/calculate_incident_angles.py +0 -2
  8. simtools/applications/convert_geo_coordinates_of_array_elements.py +1 -2
  9. simtools/applications/db_add_file_to_db.py +1 -1
  10. simtools/applications/db_add_simulation_model_from_repository_to_db.py +1 -1
  11. simtools/applications/db_add_value_from_json_to_db.py +1 -1
  12. simtools/applications/db_generate_compound_indexes.py +1 -1
  13. simtools/applications/db_get_array_layouts_from_db.py +2 -6
  14. simtools/applications/db_get_file_from_db.py +1 -1
  15. simtools/applications/db_get_parameter_from_db.py +1 -1
  16. simtools/applications/db_inspect_databases.py +1 -1
  17. simtools/applications/db_upload_model_repository.py +1 -1
  18. simtools/applications/derive_ctao_array_layouts.py +1 -2
  19. simtools/applications/derive_mirror_rnda.py +1 -3
  20. simtools/applications/derive_psf_parameters.py +5 -1
  21. simtools/applications/derive_pulse_shape_parameters.py +194 -0
  22. simtools/applications/derive_trigger_rates.py +1 -1
  23. simtools/applications/docs_produce_array_element_report.py +2 -8
  24. simtools/applications/docs_produce_calibration_reports.py +1 -3
  25. simtools/applications/docs_produce_model_parameter_reports.py +0 -2
  26. simtools/applications/docs_produce_simulation_configuration_report.py +1 -3
  27. simtools/applications/generate_array_config.py +0 -1
  28. simtools/applications/generate_corsika_histograms.py +48 -235
  29. simtools/applications/generate_regular_arrays.py +5 -35
  30. simtools/applications/generate_simtel_event_data.py +2 -2
  31. simtools/applications/maintain_simulation_model_add_production.py +2 -2
  32. simtools/applications/maintain_simulation_model_write_array_element_positions.py +87 -0
  33. simtools/applications/plot_array_layout.py +64 -108
  34. simtools/applications/plot_simulated_event_distributions.py +57 -0
  35. simtools/applications/plot_tabular_data.py +0 -1
  36. simtools/applications/plot_tabular_data_for_model_parameter.py +1 -6
  37. simtools/applications/production_derive_corsika_limits.py +1 -1
  38. simtools/applications/production_generate_grid.py +0 -1
  39. simtools/applications/run_application.py +1 -1
  40. simtools/applications/simulate_flasher.py +3 -4
  41. simtools/applications/simulate_illuminator.py +0 -1
  42. simtools/applications/simulate_pedestals.py +2 -6
  43. simtools/applications/simulate_prod.py +9 -28
  44. simtools/applications/simulate_prod_htcondor_generator.py +8 -1
  45. simtools/applications/submit_array_layouts.py +7 -7
  46. simtools/applications/submit_model_parameter_from_external.py +1 -3
  47. simtools/applications/validate_camera_efficiency.py +0 -1
  48. simtools/applications/validate_camera_fov.py +0 -1
  49. simtools/applications/validate_cumulative_psf.py +0 -2
  50. simtools/applications/validate_file_using_schema.py +49 -123
  51. simtools/applications/validate_optics.py +0 -13
  52. simtools/camera/camera_efficiency.py +1 -6
  53. simtools/camera/single_photon_electron_spectrum.py +2 -1
  54. simtools/configuration/commandline_parser.py +43 -8
  55. simtools/configuration/configurator.py +6 -11
  56. simtools/corsika/corsika_config.py +204 -99
  57. simtools/corsika/corsika_histograms.py +411 -1735
  58. simtools/corsika/primary_particle.py +1 -1
  59. simtools/data_model/metadata_collector.py +5 -2
  60. simtools/data_model/metadata_model.py +0 -4
  61. simtools/data_model/model_data_writer.py +27 -17
  62. simtools/data_model/schema.py +112 -5
  63. simtools/data_model/validate_data.py +80 -48
  64. simtools/db/db_handler.py +19 -8
  65. simtools/db/db_model_upload.py +2 -1
  66. simtools/db/mongo_db.py +133 -42
  67. simtools/dependencies.py +83 -44
  68. simtools/io/ascii_handler.py +4 -2
  69. simtools/io/table_handler.py +1 -1
  70. simtools/job_execution/htcondor_script_generator.py +0 -2
  71. simtools/layout/array_layout.py +4 -12
  72. simtools/layout/array_layout_utils.py +227 -58
  73. simtools/model/array_model.py +37 -18
  74. simtools/model/calibration_model.py +0 -4
  75. simtools/model/legacy_model_parameter.py +134 -0
  76. simtools/model/model_parameter.py +24 -14
  77. simtools/model/model_repository.py +18 -5
  78. simtools/model/model_utils.py +1 -6
  79. simtools/model/site_model.py +0 -4
  80. simtools/model/telescope_model.py +6 -11
  81. simtools/production_configuration/derive_corsika_limits.py +6 -11
  82. simtools/production_configuration/interpolation_handler.py +16 -16
  83. simtools/ray_tracing/incident_angles.py +5 -11
  84. simtools/ray_tracing/mirror_panel_psf.py +3 -7
  85. simtools/ray_tracing/psf_analysis.py +29 -27
  86. simtools/ray_tracing/psf_parameter_optimisation.py +822 -680
  87. simtools/ray_tracing/ray_tracing.py +6 -15
  88. simtools/reporting/docs_auto_report_generator.py +8 -13
  89. simtools/reporting/docs_read_parameters.py +70 -16
  90. simtools/runners/corsika_runner.py +15 -10
  91. simtools/runners/corsika_simtel_runner.py +9 -8
  92. simtools/runners/runner_services.py +17 -7
  93. simtools/runners/simtel_runner.py +11 -58
  94. simtools/runners/simtools_runner.py +2 -4
  95. simtools/schemas/model_parameters/flasher_pulse_exp_decay.schema.yml +2 -0
  96. simtools/schemas/model_parameters/flasher_pulse_shape.schema.yml +50 -0
  97. simtools/schemas/model_parameters/flasher_pulse_width.schema.yml +2 -0
  98. simtools/schemas/simulation_models_info.schema.yml +2 -0
  99. simtools/settings.py +154 -0
  100. simtools/sim_events/file_info.py +128 -0
  101. simtools/{simtel/simtel_io_event_histograms.py → sim_events/histograms.py} +25 -15
  102. simtools/{simtel/simtel_io_event_reader.py → sim_events/reader.py} +20 -17
  103. simtools/{simtel/simtel_io_event_writer.py → sim_events/writer.py} +84 -25
  104. simtools/simtel/pulse_shapes.py +273 -0
  105. simtools/simtel/simtel_config_writer.py +146 -22
  106. simtools/simtel/simtel_table_reader.py +6 -4
  107. simtools/simtel/simulator_array.py +62 -23
  108. simtools/simtel/simulator_camera_efficiency.py +4 -6
  109. simtools/simtel/simulator_light_emission.py +101 -19
  110. simtools/simtel/simulator_ray_tracing.py +4 -10
  111. simtools/simulator.py +360 -353
  112. simtools/telescope_trigger_rates.py +3 -4
  113. simtools/testing/assertions.py +115 -8
  114. simtools/testing/configuration.py +2 -3
  115. simtools/testing/helpers.py +2 -3
  116. simtools/testing/log_inspector.py +5 -1
  117. simtools/testing/sim_telarray_metadata.py +1 -1
  118. simtools/testing/validate_output.py +69 -23
  119. simtools/utils/general.py +37 -0
  120. simtools/utils/geometry.py +0 -77
  121. simtools/utils/names.py +7 -9
  122. simtools/version.py +37 -0
  123. simtools/visualization/legend_handlers.py +21 -10
  124. simtools/visualization/plot_array_layout.py +312 -41
  125. simtools/visualization/plot_corsika_histograms.py +143 -605
  126. simtools/visualization/plot_mirrors.py +834 -0
  127. simtools/visualization/plot_pixels.py +2 -4
  128. simtools/visualization/plot_psf.py +0 -1
  129. simtools/visualization/plot_simtel_event_histograms.py +4 -4
  130. simtools/visualization/plot_simtel_events.py +6 -11
  131. simtools/visualization/plot_tables.py +8 -19
  132. simtools/visualization/visualize.py +22 -2
  133. simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +0 -160
  134. simtools/applications/print_version.py +0 -53
  135. simtools/io/hdf5_handler.py +0 -139
  136. simtools/simtel/simtel_io_file_info.py +0 -62
  137. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/WHEEL +0 -0
  138. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/top_level.txt +0 -0
simtools/simulator.py CHANGED
@@ -10,28 +10,26 @@ from pathlib import Path
10
10
  import numpy as np
11
11
  from astropy import units as u
12
12
 
13
- import simtools.utils.general as gen
14
13
  from simtools.corsika.corsika_config import CorsikaConfig
15
14
  from simtools.io import io_handler, table_handler
16
15
  from simtools.job_execution.job_manager import JobManager
17
16
  from simtools.model.array_model import ArrayModel
18
17
  from simtools.runners.corsika_runner import CorsikaRunner
19
18
  from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
20
- from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
19
+ from simtools.sim_events import file_info
20
+ from simtools.sim_events.writer import EventDataWriter
21
21
  from simtools.simtel.simulator_array import SimulatorArray
22
22
  from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
23
+ from simtools.utils import general, names
23
24
  from simtools.version import semver_to_int
24
25
 
25
26
 
26
- class InvalidRunsToSimulateError(Exception):
27
- """Exception for invalid runs to simulate."""
28
-
29
-
30
27
  class Simulator:
31
28
  """
32
29
  Simulator is managing the simulation of showers and of the array of telescopes.
33
30
 
34
31
  It interfaces with simulation software packages (e.g., CORSIKA or sim_telarray).
32
+ A single run is simulated per instance, possibly for multiple model versions.
35
33
 
36
34
  The configuration is set as a dict corresponding to the command line configuration groups
37
35
  (especially simulation_software, simulation_model, simulation_parameters).
@@ -45,8 +43,6 @@ class Simulator:
45
43
  Instance label.
46
44
  extra_commands: str or list of str
47
45
  Extra commands to be added to the run script before the run command.
48
- db_config: dict
49
- Database configuration.
50
46
  """
51
47
 
52
48
  def __init__(
@@ -54,14 +50,14 @@ class Simulator:
54
50
  args_dict,
55
51
  label=None,
56
52
  extra_commands=None,
57
- db_config=None,
58
53
  ):
59
54
  """Initialize Simulator class."""
60
55
  self.logger = logging.getLogger(__name__)
61
56
  self.label = label
62
57
 
63
58
  self.args_dict = args_dict
64
- self.db_config = db_config
59
+ self.site = self.args_dict.get("site", None)
60
+ self.model_version = self.args_dict.get("model_version", None)
65
61
 
66
62
  self.simulation_software = self.args_dict.get("simulation_software", "corsika_sim_telarray")
67
63
  self.logger.debug(f"Init Simulator {self.simulation_software}")
@@ -69,19 +65,13 @@ class Simulator:
69
65
 
70
66
  self.io_handler = io_handler.IOHandler()
71
67
 
72
- self.runs = self._initialize_run_list()
68
+ self.run_number = None
73
69
  self._results = defaultdict(list)
74
- self._test = self.args_dict.get("test", False)
70
+ self._test = None
75
71
  self._extra_commands = extra_commands
76
-
77
- self.sim_telarray_seeds = {
78
- "seed": self.args_dict.get("sim_telarray_instrument_seeds"),
79
- "random_instrument_instances": self.args_dict.get(
80
- "sim_telarray_random_instrument_instances"
81
- ),
82
- "seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
83
- }
84
- self.array_models = self._initialize_array_models()
72
+ self.sim_telarray_seeds = None
73
+ self._initialize_from_tool_configuration()
74
+ self.array_models, self.corsika_configurations = self._initialize_array_models()
85
75
  self._simulation_runner = self._initialize_simulation_runner()
86
76
 
87
77
  @property
@@ -109,43 +99,82 @@ class Simulator:
109
99
  raise ValueError(f"Invalid simulation software: {simulation_software}")
110
100
  self._simulation_software = simulation_software.lower()
111
101
 
102
+ def _initialize_from_tool_configuration(self):
103
+ """Initialize simulator from tool configuration."""
104
+ self._test = self.args_dict.get("test", False)
105
+ self.sim_telarray_seeds = {
106
+ "seed": self.args_dict.get("sim_telarray_instrument_seeds"),
107
+ "random_instrument_instances": self.args_dict.get(
108
+ "sim_telarray_random_instrument_instances"
109
+ ),
110
+ "seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
111
+ }
112
+
113
+ if self.args_dict.get("corsika_file"):
114
+ self.run_number = file_info.get_corsika_run_number(self.args_dict["corsika_file"])
115
+ else:
116
+ self.run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
117
+ "run_number", 1
118
+ )
119
+
112
120
  def _initialize_array_models(self):
113
121
  """
114
- Initialize array simulation models.
122
+ Initialize array simulation models and CORSIKA config (one per model version).
115
123
 
116
124
  Returns
117
125
  -------
118
126
  list
119
127
  List of ArrayModel objects.
120
128
  """
121
- versions = gen.ensure_iterable(self.args_dict.get("model_version", []))
122
-
123
- return [
124
- ArrayModel(
125
- label=self.label,
126
- site=self.args_dict.get("site"),
127
- layout_name=self.args_dict.get("array_layout_name"),
128
- db_config=self.db_config,
129
- model_version=version,
130
- sim_telarray_seeds={
131
- "seed": self._get_seed_for_random_instrument_instances(
132
- self.sim_telarray_seeds["seed"], version
133
- ),
134
- "random_instrument_instances": self.sim_telarray_seeds[
135
- "random_instrument_instances"
136
- ],
137
- "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
138
- },
139
- simtel_path=self.args_dict.get("simtel_path", None),
140
- calibration_device_types=self._get_calibration_device_types(
141
- self.args_dict.get("run_mode")
142
- ),
143
- overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters", None),
129
+ versions = general.ensure_iterable(self.model_version)
130
+
131
+ array_model = []
132
+ corsika_configurations = []
133
+
134
+ for version in versions:
135
+ array_model.append(
136
+ ArrayModel(
137
+ label=self.label,
138
+ site=self.site,
139
+ layout_name=self.args_dict.get("array_layout_name"),
140
+ model_version=version,
141
+ calibration_device_types=self._get_calibration_device_types(self.run_mode),
142
+ overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters"),
143
+ )
144
144
  )
145
- for version in versions
146
- ]
145
+ corsika_configurations.append(
146
+ CorsikaConfig(
147
+ array_model=array_model[-1],
148
+ label=self.label,
149
+ args_dict=self.args_dict,
150
+ dummy_simulations=self._is_calibration_run(self.run_mode),
151
+ )
152
+ )
153
+ array_model[-1].sim_telarray_seeds = {
154
+ "seed": self._get_seed_for_random_instrument_instances(
155
+ self.sim_telarray_seeds["seed"],
156
+ version,
157
+ corsika_configurations[-1].zenith_angle,
158
+ corsika_configurations[-1].azimuth_angle,
159
+ ),
160
+ "random_instrument_instances": self.sim_telarray_seeds[
161
+ "random_instrument_instances"
162
+ ],
163
+ "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
164
+ }
165
+
166
+ # 'corsika_sim_telarray' allows for multiple model versions (multipipe option)
167
+ corsika_configurations = (
168
+ corsika_configurations
169
+ if self.simulation_software == "corsika_sim_telarray"
170
+ else corsika_configurations[0]
171
+ )
147
172
 
148
- def _get_seed_for_random_instrument_instances(self, seed, model_version):
173
+ return array_model, corsika_configurations
174
+
175
+ def _get_seed_for_random_instrument_instances(
176
+ self, seed, model_version, zenith_angle, azimuth_angle
177
+ ):
149
178
  """
150
179
  Generate seed for random instances of the instrument.
151
180
 
@@ -155,6 +184,10 @@ class Simulator:
155
184
  Seed string given through configuration.
156
185
  model_version: str
157
186
  Model version.
187
+ zenith_angle: float
188
+ Zenith angle of the observation (in degrees).
189
+ azimuth_angle: float
190
+ Azimuth angle of the observation (in degrees).
158
191
 
159
192
  Returns
160
193
  -------
@@ -164,107 +197,16 @@ class Simulator:
164
197
  if seed:
165
198
  return int(seed.split(",")[0].strip())
166
199
 
167
- seed = semver_to_int(model_version) * 10000000
168
- seed = seed + 1000000 if self.args_dict.get("site") != "North" else seed + 2000000
169
- seed = seed + (int)(self.args_dict.get("zenith_angle", 0.0 * u.deg).value) * 1000
170
- return seed + (int)(self.args_dict.get("azimuth_angle", 0.0 * u.deg).value)
171
-
172
- def _initialize_run_list(self):
173
- """
174
- Initialize run list using the configuration values.
175
-
176
- Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
177
- to create a list of run numbers.
178
-
179
- Returns
180
- -------
181
- list
182
- List of run numbers.
183
- """
184
- offset_run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
185
- "run_number", 1
186
- )
187
- if self.args_dict.get("number_of_runs", 1) <= 1:
188
- return self._prepare_run_list_and_range(
189
- run_list=offset_run_number,
190
- run_range=None,
191
- )
192
- return self._prepare_run_list_and_range(
193
- run_list=None,
194
- run_range=[
195
- offset_run_number,
196
- offset_run_number + self.args_dict["number_of_runs"],
197
- ],
198
- )
199
-
200
- def _prepare_run_list_and_range(self, run_list, run_range):
201
- """
202
- Prepare list of run numbers from a list or from a range.
203
-
204
- If both arguments are given, they will be merged into a single list.
200
+ def key_index(key):
201
+ try:
202
+ return list(names.site_names()).index(key) + 1
203
+ except ValueError:
204
+ return 1
205
205
 
206
- Attributes
207
- ----------
208
- run_list: list
209
- list of runs (integers)
210
- run_range:list
211
- min and max of range of runs to be simulated (two list entries)
212
-
213
- Returns
214
- -------
215
- list
216
- list of unique run numbers (integers)
217
- """
218
- if run_list is None and run_range is None:
219
- self.logger.debug("Nothing to prepare - run_list and run_range not given.")
220
- return None
221
-
222
- validated_runs = []
223
- if run_list is not None:
224
- validated_runs = gen.ensure_iterable(run_list)
225
- if not all(isinstance(r, int) for r in validated_runs):
226
- raise InvalidRunsToSimulateError(f"Run list must contain only integers: {run_list}")
227
-
228
- if run_range is not None:
229
- if not all(isinstance(r, int) for r in run_range) or len(run_range) != 2:
230
- raise InvalidRunsToSimulateError(
231
- f"Run_range must contain two integers only: {run_range}"
232
- )
233
- run_range = np.arange(run_range[0], run_range[1])
234
- validated_runs.extend(list(run_range))
235
-
236
- validated_runs_unique = sorted(set(validated_runs))
237
- self.logger.info(f"Runlist: {validated_runs_unique}")
238
- return validated_runs_unique
239
-
240
- def _corsika_configuration(self):
241
- """
242
- Define CORSIKA configurations based on the simulation model.
243
-
244
- For 'corsika_sim_telarray', this is a list since multiple configurations
245
- might be defined to run in a single job using multipipe.
246
-
247
- Returns
248
- -------
249
- CorsikaConfig or list of CorsikaConfig
250
- CORSIKA configuration(s) based on the simulation model.
251
- """
252
- corsika_configurations = []
253
- for array_model in self.array_models:
254
- corsika_configurations.append(
255
- CorsikaConfig(
256
- array_model=array_model,
257
- label=self.label,
258
- args_dict=self.args_dict,
259
- db_config=self.db_config,
260
- dummy_simulations=self._is_calibration_run(self.run_mode),
261
- )
262
- )
263
- return (
264
- corsika_configurations
265
- if self.simulation_software == "corsika_sim_telarray"
266
- else corsika_configurations[0]
267
- )
206
+ seed = semver_to_int(model_version) * 10000000
207
+ seed = seed + key_index(self.site) * 1000000
208
+ seed = seed + (int)(zenith_angle) * 1000
209
+ return seed + (int)(azimuth_angle)
268
210
 
269
211
  def _initialize_simulation_runner(self):
270
212
  """
@@ -275,8 +217,6 @@ class Simulator:
275
217
  CorsikaRunner or SimulatorArray or CorsikaSimtelRunner
276
218
  Simulation runner object.
277
219
  """
278
- corsika_configurations = self._corsika_configuration()
279
-
280
220
  runner_class = {
281
221
  "corsika": CorsikaRunner,
282
222
  "sim_telarray": SimulatorArray,
@@ -285,13 +225,15 @@ class Simulator:
285
225
 
286
226
  runner_args = {
287
227
  "label": self.label,
288
- "corsika_config": corsika_configurations,
289
- "simtel_path": self.args_dict.get("simtel_path"),
228
+ "corsika_config": self.corsika_configurations,
290
229
  "use_multipipe": runner_class is CorsikaSimtelRunner,
291
230
  }
292
231
 
293
232
  if runner_class is not SimulatorArray:
294
233
  runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
234
+ runner_args["curved_atmosphere_min_zenith_angle"] = self.args_dict.get(
235
+ "curved_atmosphere_min_zenith_angle", 65 * u.deg
236
+ )
295
237
  if runner_class is not CorsikaRunner:
296
238
  runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
297
239
  if runner_class is CorsikaSimtelRunner:
@@ -302,179 +244,90 @@ class Simulator:
302
244
 
303
245
  return runner_class(**runner_args)
304
246
 
305
- def _fill_results_without_run(self, input_file_list):
247
+ def simulate(self):
306
248
  """
307
- Fill results dict without calling submit (e.g., for testing).
249
+ Prepare and submit a run script as a job.
308
250
 
309
- Parameters
310
- ----------
311
- input_file_list: str or list of str
312
- Single file or list of files of shower simulations.
313
- """
314
- for file in gen.ensure_iterable(input_file_list):
315
- run = self._guess_run_from_file(file)
316
- self._fill_results(file, run)
317
- if run not in self.runs:
318
- self.runs.append(run)
319
-
320
- def simulate(self, input_file_list=None):
251
+ Writes submission scripts using the simulation runners and submits the
252
+ run script to the job manager. Collects generated files.
321
253
  """
322
- Submit a run script as a job.
323
-
324
- Parameters
325
- ----------
326
- input_file_list: str or list of str
327
- Single file or list of files of shower simulations.
328
- """
329
- runs_and_files_to_submit = self._get_runs_and_files_to_submit(
330
- input_file_list=input_file_list
254
+ run_script = self._simulation_runner.prepare_run_script(
255
+ run_number=self.run_number,
256
+ input_file=self._get_corsika_file(),
257
+ extra_commands=self._extra_commands,
331
258
  )
332
- self.logger.info(
333
- f"Starting submission for {len(runs_and_files_to_submit)} "
334
- f"run{'s' if len(runs_and_files_to_submit) > 1 else ''}"
335
- )
336
-
337
- for run_number, input_file in runs_and_files_to_submit.items():
338
- run_script = self._simulation_runner.prepare_run_script(
339
- run_number=run_number, input_file=input_file, extra_commands=self._extra_commands
340
- )
341
259
 
342
- job_manager = JobManager(test=self._test)
343
- job_manager.submit(
344
- run_script=run_script,
345
- run_out_file=self._simulation_runner.get_file_name(
346
- file_type="sub_log", run_number=run_number
347
- ),
348
- log_file=self._simulation_runner.get_file_name(
349
- file_type=("log"), run_number=run_number
350
- ),
351
- )
352
-
353
- self._fill_results(input_file, run_number)
354
-
355
- def _get_runs_and_files_to_submit(self, input_file_list=None):
356
- """
357
- Return a dictionary with run numbers and simulation files.
358
-
359
- The latter are expected to be given for the sim_telarray simulator.
360
-
361
- Parameters
362
- ----------
363
- input_file_list: str or list of str
364
- Single file or list of files of shower simulations.
365
-
366
- Returns
367
- -------
368
- runs_and_files: dict
369
- dictionary with run number as key and (if available) simulation
370
- file name as value
371
-
372
- Raises
373
- ------
374
- ValueError
375
- If no runs are to be submitted.
376
- """
377
- _runs_and_files = {}
378
- self.logger.debug(f"Getting runs and files to submit ({input_file_list})")
260
+ job_manager = JobManager(test=self._test)
261
+ job_manager.submit(
262
+ run_script=run_script,
263
+ run_out_file=self._simulation_runner.get_file_name(
264
+ file_type="sub_log", run_number=self.run_number
265
+ ),
266
+ log_file=self._simulation_runner.get_file_name(
267
+ file_type=("log"), run_number=self.run_number
268
+ ),
269
+ )
379
270
 
380
- if self.simulation_software == "sim_telarray" and self.run_mode is None:
381
- input_file_list = gen.ensure_iterable(input_file_list)
382
- _runs_and_files = {self._guess_run_from_file(file): file for file in input_file_list}
383
- else:
384
- _runs_and_files = dict.fromkeys(self._get_runs_to_simulate())
385
- if len(_runs_and_files) == 0:
386
- raise ValueError("No runs to submit.")
387
- return _runs_and_files
271
+ self._fill_list_of_generated_files()
388
272
 
389
- def _guess_run_from_file(self, file):
273
+ def _get_corsika_file(self):
390
274
  """
391
- Extract the run number from the given file name.
392
-
393
- Input file names can follow any pattern with the
394
- string 'run' followed by the run number.
395
-
396
- Parameters
397
- ----------
398
- file: Path
399
- Simulation file name
275
+ Get the CORSIKA input file if applicable (for sim_telarray simulations).
400
276
 
401
277
  Returns
402
278
  -------
403
- int
404
- The extracted run number. If extraction fails, returns 1 and logs a warning.
405
- """
406
- file_name = str(Path(file).name)
407
-
408
- try:
409
- run_str = re.search(r"run\d*", file_name).group()
410
- return int(run_str[3:])
411
- except (ValueError, AttributeError):
412
- self.logger.warning(f"Run number could not be guessed from {file_name} using run = 1")
413
- return 1
414
-
415
- def _fill_results(self, file, run_number):
416
- """
417
- Fill the results dict with input, output, hist, and log files.
418
-
419
- Parameters
420
- ----------
421
- file: str
422
- input file name
423
- run_number: int
424
- run number
425
-
426
- """
427
- keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
279
+ Path, None
280
+ Path to the CORSIKA input file.
281
+ """
282
+ if self.simulation_software == "sim_telarray":
283
+ return self.args_dict.get("corsika_file", None)
284
+ return None
285
+
286
+ def _fill_list_of_generated_files(self):
287
+ """Fill a dictionary with lists of generated files."""
288
+ keys = [
289
+ "simtel_output",
290
+ "sub_out",
291
+ "log",
292
+ "input",
293
+ "histogram",
294
+ "corsika_log",
295
+ "corsika_output",
296
+ "event_data",
297
+ ]
428
298
  results = {key: [] for key in keys}
429
299
 
430
300
  def get_file_name(name, **kwargs):
431
301
  return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
432
302
 
433
- if "sim_telarray" in self.simulation_software:
434
- results["input"].append(str(file))
435
-
436
- results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
303
+ results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=self.run_number))
437
304
 
438
305
  for i in range(len(self.array_models)):
439
306
  results["simtel_output"].append(
440
- get_file_name("simtel_output", run_number=run_number, model_version_index=i)
307
+ get_file_name("simtel_output", run_number=self.run_number, model_version_index=i)
441
308
  )
442
309
 
443
310
  if "sim_telarray" in self.simulation_software:
444
- results["log"].append(
445
- get_file_name(
446
- "log",
447
- simulation_software="sim_telarray",
448
- run_number=run_number,
449
- model_version_index=i,
311
+ for file_type in ("log", "histogram", "event_data"):
312
+ results[file_type].append(
313
+ get_file_name(
314
+ file_type,
315
+ simulation_software="sim_telarray",
316
+ run_number=self.run_number,
317
+ model_version_index=i,
318
+ )
450
319
  )
451
- )
452
- results["hist"].append(
453
- get_file_name(
454
- "histogram",
455
- simulation_software="sim_telarray",
456
- run_number=run_number,
457
- model_version_index=i,
458
- )
459
- )
460
- results["event_data"].append(
461
- get_file_name(
462
- "event_data",
463
- simulation_software="sim_telarray",
464
- run_number=run_number,
465
- model_version_index=i,
466
- )
467
- )
468
320
 
469
321
  if "corsika" in self.simulation_software:
470
- results["corsika_log"].append(
471
- get_file_name(
472
- "corsika_log",
473
- simulation_software="corsika",
474
- run_number=run_number,
475
- model_version_index=i,
322
+ for file_type in ("corsika_output", "corsika_log"):
323
+ results[file_type].append(
324
+ get_file_name(
325
+ file_type,
326
+ simulation_software="corsika",
327
+ run_number=self.run_number,
328
+ model_version_index=i,
329
+ )
476
330
  )
477
- )
478
331
 
479
332
  for key in keys:
480
333
  self._results[key].extend(results[key])
@@ -483,7 +336,6 @@ class Simulator:
483
336
  """
484
337
  Get list of files generated by simulations.
485
338
 
486
- Options are "input", "simtel_output", "hist", "log", "corsika_log".
487
339
  Not all file types are available for all simulation types.
488
340
  Returns an empty list for an unknown file type.
489
341
 
@@ -498,7 +350,6 @@ class Simulator:
498
350
  List with the full path of all output files.
499
351
 
500
352
  """
501
- self.logger.info(f"Getting list of {file_type} files")
502
353
  return self._results[file_type]
503
354
 
504
355
  def _make_resources_report(self, input_file_list):
@@ -512,38 +363,31 @@ class Simulator:
512
363
 
513
364
  Returns
514
365
  -------
515
- dict
516
- Dictionary with reports on computing resources
366
+ str
367
+ string reporting on computing resources
517
368
 
518
369
  """
519
- if len(self._results["sub_out"]) == 0:
520
- if input_file_list is None:
521
- return {"Wall time/run [sec]": np.nan}
522
- self._fill_results_without_run(input_file_list)
370
+ if len(self._results["sub_out"]) == 0 and input_file_list is None:
371
+ return "Mean wall time/run [sec]: np.nan"
523
372
 
524
373
  runtime = []
525
374
 
526
375
  _resources = {}
527
- for run in self.runs:
528
- _resources = self._simulation_runner.get_resources(run_number=run)
529
- if _resources.get("runtime"):
530
- runtime.append(_resources["runtime"])
376
+ _resources = self._simulation_runner.get_resources(run_number=self.run_number)
377
+ if _resources.get("runtime"):
378
+ runtime.append(_resources["runtime"])
531
379
 
532
380
  mean_runtime = np.mean(runtime)
533
381
 
534
- resource_summary = {}
535
- resource_summary["Wall time/run [sec]"] = mean_runtime
382
+ resource_summary = f"Mean wall time/run [sec]: {mean_runtime}"
536
383
  if "n_events" in _resources and _resources["n_events"] > 0:
537
- resource_summary["#events/run"] = _resources["n_events"]
538
- resource_summary["Wall time/1000 events [sec]"] = (
539
- mean_runtime * 1000 / _resources["n_events"]
540
- )
384
+ resource_summary += f", #events/run: {_resources['n_events']}"
541
385
 
542
386
  return resource_summary
543
387
 
544
- def resources(self, input_file_list=None):
388
+ def report(self, input_file_list=None):
545
389
  """
546
- Print a simple report on computing resources used.
390
+ Report on simulations and computing resources used.
547
391
 
548
392
  Includes run time per run only at this point.
549
393
 
@@ -553,37 +397,21 @@ class Simulator:
553
397
  Single file or list of files of shower simulations.
554
398
 
555
399
  """
556
- resources = self._make_resources_report(input_file_list)
557
- print("-----------------------------")
558
- print(f"Computing Resources Report - {self.simulation_software} Simulations")
559
- for key, value in resources.items():
560
- print(f"{key} = {value:.2f}")
561
- print("-----------------------------")
562
-
563
- def _get_runs_to_simulate(self, run_list=None, run_range=None):
564
- """
565
- Process run_list and run_range and return the validated list of runs.
566
-
567
- Attributes
568
- ----------
569
- run_list: list
570
- list of runs (integers)
571
- run_range:list
572
- min and max of range of runs to be simulated (two list entries)
573
-
574
- Returns
575
- -------
576
- list
577
- list of unique run numbers (integers)
578
-
579
- """
580
- if run_list is None and run_range is None:
581
- return [] if self.runs is None else self.runs
582
- return self._prepare_run_list_and_range(run_list, run_range)
400
+ _corsika_config = self._get_first_corsika_config()
401
+ self.logger.info(
402
+ f"Production run complete for primary {_corsika_config.primary_particle} showers "
403
+ f"from {_corsika_config.azimuth_angle} azimuth and "
404
+ f"{_corsika_config.zenith_angle} zenith "
405
+ f"at {self.site} site, using {self.model_version} model."
406
+ )
407
+ self.logger.info(
408
+ f"Computing for {self.simulation_software} Simulations: "
409
+ f"{self._make_resources_report(input_file_list)}"
410
+ )
583
411
 
584
412
  def save_file_lists(self):
585
413
  """Save files lists for output and log files."""
586
- for file_type in ["simtel_output", "log", "corsika_log", "hist"]:
414
+ for file_type in ["simtel_output", "log", "corsika_log", "histogram"]:
587
415
  file_name = self.io_handler.get_output_directory().joinpath(f"{file_type}_files.txt")
588
416
  file_list = self.get_file_list(file_type=file_type)
589
417
  if all(element is not None for element in file_list) and len(file_list) > 0:
@@ -610,7 +438,7 @@ class Simulator:
610
438
  input_files = self.get_file_list(file_type="simtel_output")
611
439
  output_files = self.get_file_list(file_type="event_data")
612
440
  for input_file, output_file in zip(input_files, output_files):
613
- generator = SimtelIOEventDataWriter([input_file])
441
+ generator = EventDataWriter([input_file])
614
442
  table_handler.write_tables(
615
443
  tables=generator.process_files(),
616
444
  output_file=Path(output_file),
@@ -635,7 +463,7 @@ class Simulator:
635
463
  output_files = self.get_file_list(file_type="simtel_output")
636
464
  log_files = self.get_file_list(file_type="log")
637
465
  corsika_log_files = self.get_file_list(file_type="corsika_log")
638
- histogram_files = self.get_file_list(file_type="hist")
466
+ histogram_files = self.get_file_list(file_type="histogram")
639
467
  reduced_event_files = (
640
468
  self.get_file_list(file_type="event_data")
641
469
  if self.args_dict.get("save_reduced_event_lists")
@@ -657,7 +485,7 @@ class Simulator:
657
485
  # Group files by model version
658
486
  for model in self.array_models:
659
487
  model_version = model.model_version
660
- model_files = gen.ensure_iterable(model.pack_model_files())
488
+ model_files = general.ensure_iterable(model.pack_model_files())
661
489
 
662
490
  # Filter files for this model version
663
491
  model_logs = [f for f in log_files if model_version in f]
@@ -669,7 +497,7 @@ class Simulator:
669
497
  tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
670
498
  # Add all relevant model, log, histogram, and CORSIKA log files to the tarball
671
499
  files_to_tar = model_logs + model_hists + model_corsika_logs + model_files
672
- gen.pack_tar_file(tar_file_path, files_to_tar)
500
+ general.pack_tar_file(tar_file_path, files_to_tar)
673
501
 
674
502
  for file_to_move in output_files + reduced_event_files:
675
503
  source_file = Path(file_to_move)
@@ -763,8 +591,8 @@ class Simulator:
763
591
  """
764
592
  return run_mode in [
765
593
  "pedestals",
766
- "dark_pedestals",
767
- "nsb_only_pedestals",
594
+ "pedestals_dark",
595
+ "pedestals_nsb_only",
768
596
  "direct_injection",
769
597
  ]
770
598
 
@@ -786,3 +614,182 @@ class Simulator:
786
614
  if run_mode == "direct_injection":
787
615
  return ["flat_fielding"]
788
616
  return []
617
+
618
+ def _get_first_corsika_config(self):
619
+ """
620
+ Return first instance from list of CORSIKA configurations.
621
+
622
+ Most values stored in the CORSIKA configurations are identical,
623
+ with the exception of the simulation model version dependent parameters.
624
+
625
+ Returns
626
+ -------
627
+ CorsikaConfig
628
+ First CORSIKA configuration instance.
629
+ """
630
+ try:
631
+ return (
632
+ self.corsika_configurations[0]
633
+ if isinstance(self.corsika_configurations, list)
634
+ else self.corsika_configurations
635
+ )
636
+ except (IndexError, TypeError) as exc:
637
+ raise ValueError("CORSIKA configuration not found for verification.") from exc
638
+
639
+ def verify_simulations(self):
640
+ """
641
+ Verify simulations.
642
+
643
+ This includes checking the number of simulated events.
644
+
645
+ """
646
+ self.logger.info("Verifying simulations.")
647
+
648
+ _corsika_config = self._get_first_corsika_config()
649
+ expected_shower_events = _corsika_config.shower_events
650
+ expected_mc_events = _corsika_config.mc_events
651
+
652
+ self.logger.info(
653
+ f"Expected number of shower events: {expected_shower_events}, "
654
+ f"expected number of MC events: {expected_mc_events}"
655
+ )
656
+ if self.simulation_software in ["corsika_sim_telarray", "sim_telarray"]:
657
+ self._verify_simulated_events_in_sim_telarray(
658
+ expected_shower_events, expected_mc_events
659
+ )
660
+ if self.simulation_software == "corsika":
661
+ self._verify_simulated_events_corsika(expected_mc_events)
662
+ if self.args_dict.get("save_reduced_event_lists"):
663
+ self._verify_simulated_events_in_reduced_event_lists(expected_mc_events)
664
+
665
+ def _verify_simulated_events_corsika(self, expected_mc_events, tolerance=1.0e-3):
666
+ """
667
+ Verify the number of simulated events in CORSIKA output files.
668
+
669
+ Allow for a small mismatch in the number of requested events.
670
+
671
+ Parameters
672
+ ----------
673
+ expected_mc_events: int
674
+ Expected number of simulated MC events.
675
+
676
+ Raises
677
+ ------
678
+ ValueError
679
+ If the number of simulated events does not match the expected number.
680
+ """
681
+
682
+ def consistent(a, b, tol):
683
+ return abs(a - b) / max(a, b) <= tol
684
+
685
+ event_errors = []
686
+ for file in self.get_file_list(file_type="corsika_output"):
687
+ shower_events, _ = file_info.get_simulated_events(file)
688
+
689
+ if shower_events != expected_mc_events:
690
+ if consistent(shower_events, expected_mc_events, tol=tolerance):
691
+ self.logger.warning(
692
+ f"Small mismatch in number of events in: {file}: "
693
+ f"shower events: {shower_events} (expected: {expected_mc_events})"
694
+ )
695
+ continue
696
+ event_errors.append(
697
+ f"Number of simulated MC events ({shower_events}) does not match "
698
+ f"the expected number ({expected_mc_events}) in CORSIKA {file}."
699
+ )
700
+ else:
701
+ self.logger.info(
702
+ f"Consistent number of events in: {file}: shower events: {shower_events}"
703
+ )
704
+
705
+ if event_errors:
706
+ self.logger.error("Inconsistent event counts found in CORSIKA output:")
707
+ for error in event_errors:
708
+ self.logger.error(f" - {error}")
709
+ error_message = "Inconsistent event counts found in CORSIKA output:\n" + "\n".join(
710
+ f" - {error}" for error in event_errors
711
+ )
712
+ raise ValueError(error_message)
713
+
714
+ def _verify_simulated_events_in_sim_telarray(self, expected_shower_events, expected_mc_events):
715
+ """
716
+ Verify the number of simulated events.
717
+
718
+ Parameters
719
+ ----------
720
+ expected_shower_events: int
721
+ Expected number of simulated shower events.
722
+ expected_mc_events: int
723
+ Expected number of simulated MC events.
724
+
725
+ Raises
726
+ ------
727
+ ValueError
728
+ If the number of simulated events does not match the expected number.
729
+ """
730
+ event_errors = []
731
+ for file in self.get_file_list(file_type="simtel_output"):
732
+ shower_events, mc_events = file_info.get_simulated_events(file)
733
+
734
+ if (shower_events, mc_events) != (expected_shower_events, expected_mc_events):
735
+ event_errors.append(
736
+ f"Event mismatch: shower/MC events in {file}: {shower_events}/{mc_events}"
737
+ f" (expected: {expected_shower_events}/{expected_mc_events})"
738
+ )
739
+ else:
740
+ self.logger.info(
741
+ f"Consistent number of events in: {file}: "
742
+ f"shower events: {shower_events}, "
743
+ f"MC events: {mc_events}"
744
+ )
745
+
746
+ if event_errors:
747
+ self.logger.error("Inconsistent event counts found:")
748
+ for error in event_errors:
749
+ self.logger.error(f" - {error}")
750
+ error_message = "Inconsistent event counts found:\n" + "\n".join(
751
+ f" - {error}" for error in event_errors
752
+ )
753
+ raise ValueError(error_message)
754
+
755
+ def _verify_simulated_events_in_reduced_event_lists(self, expected_mc_events):
756
+ """
757
+ Verify the number of simulated events in reduced event lists.
758
+
759
+ Parameters
760
+ ----------
761
+ expected_mc_events: int
762
+ Expected number of simulated MC events.
763
+
764
+ Raises
765
+ ------
766
+ ValueError
767
+ If the number of simulated events does not match the expected number.
768
+ """
769
+ event_errors = []
770
+ for file in self.get_file_list(file_type="event_data"):
771
+ tables = table_handler.read_tables(file, ["SHOWERS"])
772
+ try:
773
+ mc_events = len(tables["SHOWERS"])
774
+ except KeyError as exc:
775
+ raise ValueError(f"SHOWERS table not found in reduced event list {file}.") from exc
776
+
777
+ if mc_events != expected_mc_events:
778
+ event_errors.append(
779
+ f"Number of simulated MC events ({mc_events}) does not match "
780
+ f"the expected number ({expected_mc_events}) in reduced event list {file}."
781
+ )
782
+ else:
783
+ self.logger.info(
784
+ f"Consistent number of events in reduced event list: {file}: MC events:"
785
+ f" {mc_events}"
786
+ )
787
+
788
+ if event_errors:
789
+ self.logger.error("Inconsistent event counts found in reduced event lists:")
790
+ for error in event_errors:
791
+ self.logger.error(f" - {error}")
792
+ error_message = "Inconsistent event counts found in reduced event lists:\n" + "\n".join(
793
+ f" - {error}" for error in event_errors
794
+ )
795
+ raise ValueError(error_message)