gammasimtools 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. {gammasimtools-0.23.0.dist-info → gammasimtools-0.25.0.dist-info}/METADATA +1 -1
  2. {gammasimtools-0.23.0.dist-info → gammasimtools-0.25.0.dist-info}/RECORD +89 -85
  3. {gammasimtools-0.23.0.dist-info → gammasimtools-0.25.0.dist-info}/entry_points.txt +1 -0
  4. simtools/_version.py +2 -2
  5. simtools/application_control.py +54 -4
  6. simtools/applications/convert_geo_coordinates_of_array_elements.py +1 -1
  7. simtools/applications/db_add_file_to_db.py +2 -2
  8. simtools/applications/db_add_simulation_model_from_repository_to_db.py +1 -1
  9. simtools/applications/db_add_value_from_json_to_db.py +2 -2
  10. simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +1 -1
  11. simtools/applications/db_generate_compound_indexes.py +1 -1
  12. simtools/applications/db_get_array_layouts_from_db.py +2 -2
  13. simtools/applications/db_get_file_from_db.py +1 -1
  14. simtools/applications/db_get_parameter_from_db.py +1 -1
  15. simtools/applications/db_inspect_databases.py +4 -2
  16. simtools/applications/db_upload_model_repository.py +1 -1
  17. simtools/applications/derive_ctao_array_layouts.py +1 -1
  18. simtools/applications/derive_psf_parameters.py +5 -0
  19. simtools/applications/derive_pulse_shape_parameters.py +195 -0
  20. simtools/applications/generate_array_config.py +1 -1
  21. simtools/applications/maintain_simulation_model_add_production.py +11 -21
  22. simtools/applications/plot_array_layout.py +63 -1
  23. simtools/applications/production_generate_grid.py +1 -1
  24. simtools/applications/simulate_flasher.py +3 -2
  25. simtools/applications/simulate_pedestals.py +1 -1
  26. simtools/applications/simulate_prod.py +8 -23
  27. simtools/applications/simulate_prod_htcondor_generator.py +7 -0
  28. simtools/applications/submit_array_layouts.py +7 -5
  29. simtools/applications/validate_camera_fov.py +1 -1
  30. simtools/applications/validate_cumulative_psf.py +2 -2
  31. simtools/applications/validate_file_using_schema.py +49 -123
  32. simtools/applications/validate_optics.py +1 -1
  33. simtools/configuration/commandline_parser.py +15 -15
  34. simtools/configuration/configurator.py +1 -1
  35. simtools/corsika/corsika_config.py +199 -91
  36. simtools/data_model/model_data_writer.py +15 -3
  37. simtools/data_model/schema.py +145 -36
  38. simtools/data_model/validate_data.py +82 -48
  39. simtools/db/db_handler.py +61 -294
  40. simtools/db/db_model_upload.py +3 -2
  41. simtools/db/mongo_db.py +626 -0
  42. simtools/dependencies.py +38 -17
  43. simtools/io/eventio_handler.py +128 -0
  44. simtools/job_execution/htcondor_script_generator.py +0 -2
  45. simtools/layout/array_layout.py +7 -7
  46. simtools/layout/array_layout_utils.py +4 -4
  47. simtools/model/array_model.py +72 -72
  48. simtools/model/calibration_model.py +12 -9
  49. simtools/model/model_parameter.py +196 -160
  50. simtools/model/model_repository.py +176 -39
  51. simtools/model/model_utils.py +3 -3
  52. simtools/model/site_model.py +59 -27
  53. simtools/model/telescope_model.py +21 -13
  54. simtools/ray_tracing/mirror_panel_psf.py +4 -4
  55. simtools/ray_tracing/psf_analysis.py +11 -8
  56. simtools/ray_tracing/psf_parameter_optimisation.py +823 -680
  57. simtools/reporting/docs_auto_report_generator.py +1 -1
  58. simtools/reporting/docs_read_parameters.py +72 -11
  59. simtools/runners/corsika_runner.py +12 -3
  60. simtools/runners/corsika_simtel_runner.py +6 -0
  61. simtools/runners/runner_services.py +17 -7
  62. simtools/runners/simtel_runner.py +12 -54
  63. simtools/schemas/model_parameters/flasher_pulse_exp_decay.schema.yml +2 -0
  64. simtools/schemas/model_parameters/flasher_pulse_shape.schema.yml +50 -0
  65. simtools/schemas/model_parameters/flasher_pulse_width.schema.yml +2 -0
  66. simtools/schemas/simulation_models_info.schema.yml +4 -1
  67. simtools/simtel/pulse_shapes.py +268 -0
  68. simtools/simtel/simtel_config_writer.py +179 -21
  69. simtools/simtel/simtel_io_event_writer.py +2 -2
  70. simtools/simtel/simulator_array.py +58 -12
  71. simtools/simtel/simulator_light_emission.py +45 -8
  72. simtools/simulator.py +361 -346
  73. simtools/testing/assertions.py +110 -10
  74. simtools/testing/configuration.py +1 -1
  75. simtools/testing/log_inspector.py +4 -1
  76. simtools/testing/sim_telarray_metadata.py +1 -1
  77. simtools/testing/validate_output.py +46 -15
  78. simtools/utils/names.py +2 -4
  79. simtools/utils/value_conversion.py +10 -5
  80. simtools/version.py +61 -0
  81. simtools/visualization/legend_handlers.py +14 -4
  82. simtools/visualization/plot_array_layout.py +229 -33
  83. simtools/visualization/plot_mirrors.py +837 -0
  84. simtools/visualization/plot_pixels.py +1 -1
  85. simtools/visualization/plot_psf.py +1 -1
  86. simtools/visualization/plot_tables.py +1 -1
  87. simtools/simtel/simtel_io_file_info.py +0 -62
  88. {gammasimtools-0.23.0.dist-info → gammasimtools-0.25.0.dist-info}/WHEEL +0 -0
  89. {gammasimtools-0.23.0.dist-info → gammasimtools-0.25.0.dist-info}/licenses/LICENSE +0 -0
  90. {gammasimtools-0.23.0.dist-info → gammasimtools-0.25.0.dist-info}/top_level.txt +0 -0
simtools/simulator.py CHANGED
@@ -10,9 +10,8 @@ from pathlib import Path
10
10
  import numpy as np
11
11
  from astropy import units as u
12
12
 
13
- import simtools.utils.general as gen
14
13
  from simtools.corsika.corsika_config import CorsikaConfig
15
- from simtools.io import io_handler, table_handler
14
+ from simtools.io import eventio_handler, io_handler, table_handler
16
15
  from simtools.job_execution.job_manager import JobManager
17
16
  from simtools.model.array_model import ArrayModel
18
17
  from simtools.runners.corsika_runner import CorsikaRunner
@@ -20,18 +19,16 @@ from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
20
19
  from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
21
20
  from simtools.simtel.simulator_array import SimulatorArray
22
21
  from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
22
+ from simtools.utils import general, names
23
23
  from simtools.version import semver_to_int
24
24
 
25
25
 
26
- class InvalidRunsToSimulateError(Exception):
27
- """Exception for invalid runs to simulate."""
28
-
29
-
30
26
  class Simulator:
31
27
  """
32
28
  Simulator is managing the simulation of showers and of the array of telescopes.
33
29
 
34
30
  It interfaces with simulation software packages (e.g., CORSIKA or sim_telarray).
31
+ A single run is simulated per instance, possibly for multiple model versions.
35
32
 
36
33
  The configuration is set as a dict corresponding to the command line configuration groups
37
34
  (especially simulation_software, simulation_model, simulation_parameters).
@@ -62,6 +59,8 @@ class Simulator:
62
59
 
63
60
  self.args_dict = args_dict
64
61
  self.db_config = db_config
62
+ self.site = self.args_dict.get("site", None)
63
+ self.model_version = self.args_dict.get("model_version", None)
65
64
 
66
65
  self.simulation_software = self.args_dict.get("simulation_software", "corsika_sim_telarray")
67
66
  self.logger.debug(f"Init Simulator {self.simulation_software}")
@@ -69,19 +68,13 @@ class Simulator:
69
68
 
70
69
  self.io_handler = io_handler.IOHandler()
71
70
 
72
- self.runs = self._initialize_run_list()
71
+ self.run_number = None
73
72
  self._results = defaultdict(list)
74
- self._test = self.args_dict.get("test", False)
73
+ self._test = None
75
74
  self._extra_commands = extra_commands
76
-
77
- self.sim_telarray_seeds = {
78
- "seed": self.args_dict.get("sim_telarray_instrument_seeds"),
79
- "random_instrument_instances": self.args_dict.get(
80
- "sim_telarray_random_instrument_instances"
81
- ),
82
- "seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
83
- }
84
- self.array_models = self._initialize_array_models()
75
+ self.sim_telarray_seeds = None
76
+ self._initialize_from_tool_configuration()
77
+ self.array_models, self.corsika_configurations = self._initialize_array_models()
85
78
  self._simulation_runner = self._initialize_simulation_runner()
86
79
 
87
80
  @property
@@ -109,42 +102,85 @@ class Simulator:
109
102
  raise ValueError(f"Invalid simulation software: {simulation_software}")
110
103
  self._simulation_software = simulation_software.lower()
111
104
 
105
+ def _initialize_from_tool_configuration(self):
106
+ """Initialize simulator from tool configuration."""
107
+ self._test = self.args_dict.get("test", False)
108
+ self.sim_telarray_seeds = {
109
+ "seed": self.args_dict.get("sim_telarray_instrument_seeds"),
110
+ "random_instrument_instances": self.args_dict.get(
111
+ "sim_telarray_random_instrument_instances"
112
+ ),
113
+ "seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
114
+ }
115
+
116
+ if self.args_dict.get("corsika_file"):
117
+ self.run_number = eventio_handler.get_corsika_run_number(self.args_dict["corsika_file"])
118
+ else:
119
+ self.run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
120
+ "run_number", 1
121
+ )
122
+
112
123
  def _initialize_array_models(self):
113
124
  """
114
- Initialize array simulation models.
125
+ Initialize array simulation models and CORSIKA config (one per model version).
115
126
 
116
127
  Returns
117
128
  -------
118
129
  list
119
130
  List of ArrayModel objects.
120
131
  """
121
- versions = gen.ensure_iterable(self.args_dict.get("model_version", []))
122
-
123
- return [
124
- ArrayModel(
125
- label=self.label,
126
- site=self.args_dict.get("site"),
127
- layout_name=self.args_dict.get("array_layout_name"),
128
- mongo_db_config=self.db_config,
129
- model_version=version,
130
- sim_telarray_seeds={
131
- "seed": self._get_seed_for_random_instrument_instances(
132
- self.sim_telarray_seeds["seed"], version
133
- ),
134
- "random_instrument_instances": self.sim_telarray_seeds[
135
- "random_instrument_instances"
136
- ],
137
- "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
138
- },
139
- simtel_path=self.args_dict.get("simtel_path", None),
140
- calibration_device_types=self._get_calibration_device_types(
141
- self.args_dict.get("run_mode")
142
- ),
132
+ versions = general.ensure_iterable(self.model_version)
133
+
134
+ array_model = []
135
+ corsika_configurations = []
136
+
137
+ for version in versions:
138
+ array_model.append(
139
+ ArrayModel(
140
+ label=self.label,
141
+ site=self.site,
142
+ layout_name=self.args_dict.get("array_layout_name"),
143
+ db_config=self.db_config,
144
+ model_version=version,
145
+ calibration_device_types=self._get_calibration_device_types(self.run_mode),
146
+ overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters"),
147
+ simtel_path=self.args_dict.get("simtel_path"),
148
+ )
143
149
  )
144
- for version in versions
145
- ]
150
+ corsika_configurations.append(
151
+ CorsikaConfig(
152
+ array_model=array_model[-1],
153
+ label=self.label,
154
+ args_dict=self.args_dict,
155
+ db_config=self.db_config,
156
+ dummy_simulations=self._is_calibration_run(self.run_mode),
157
+ )
158
+ )
159
+ array_model[-1].sim_telarray_seeds = {
160
+ "seed": self._get_seed_for_random_instrument_instances(
161
+ self.sim_telarray_seeds["seed"],
162
+ version,
163
+ corsika_configurations[-1].zenith_angle,
164
+ corsika_configurations[-1].azimuth_angle,
165
+ ),
166
+ "random_instrument_instances": self.sim_telarray_seeds[
167
+ "random_instrument_instances"
168
+ ],
169
+ "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
170
+ }
171
+
172
+ # 'corsika_sim_telarray' allows for multiple model versions (multipipe option)
173
+ corsika_configurations = (
174
+ corsika_configurations
175
+ if self.simulation_software == "corsika_sim_telarray"
176
+ else corsika_configurations[0]
177
+ )
178
+
179
+ return array_model, corsika_configurations
146
180
 
147
- def _get_seed_for_random_instrument_instances(self, seed, model_version):
181
+ def _get_seed_for_random_instrument_instances(
182
+ self, seed, model_version, zenith_angle, azimuth_angle
183
+ ):
148
184
  """
149
185
  Generate seed for random instances of the instrument.
150
186
 
@@ -154,6 +190,10 @@ class Simulator:
154
190
  Seed string given through configuration.
155
191
  model_version: str
156
192
  Model version.
193
+ zenith_angle: float
194
+ Zenith angle of the observation (in degrees).
195
+ azimuth_angle: float
196
+ Azimuth angle of the observation (in degrees).
157
197
 
158
198
  Returns
159
199
  -------
@@ -163,107 +203,16 @@ class Simulator:
163
203
  if seed:
164
204
  return int(seed.split(",")[0].strip())
165
205
 
166
- seed = semver_to_int(model_version) * 10000000
167
- seed = seed + 1000000 if self.args_dict.get("site") != "North" else seed + 2000000
168
- seed = seed + (int)(self.args_dict.get("zenith_angle", 0.0 * u.deg).value) * 1000
169
- return seed + (int)(self.args_dict.get("azimuth_angle", 0.0 * u.deg).value)
170
-
171
- def _initialize_run_list(self):
172
- """
173
- Initialize run list using the configuration values.
174
-
175
- Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
176
- to create a list of run numbers.
177
-
178
- Returns
179
- -------
180
- list
181
- List of run numbers.
182
- """
183
- offset_run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
184
- "run_number", 1
185
- )
186
- if self.args_dict.get("number_of_runs", 1) <= 1:
187
- return self._prepare_run_list_and_range(
188
- run_list=offset_run_number,
189
- run_range=None,
190
- )
191
- return self._prepare_run_list_and_range(
192
- run_list=None,
193
- run_range=[
194
- offset_run_number,
195
- offset_run_number + self.args_dict["number_of_runs"],
196
- ],
197
- )
198
-
199
- def _prepare_run_list_and_range(self, run_list, run_range):
200
- """
201
- Prepare list of run numbers from a list or from a range.
206
+ def key_index(key):
207
+ try:
208
+ return list(names.site_names()).index(key) + 1
209
+ except ValueError:
210
+ return 1
202
211
 
203
- If both arguments are given, they will be merged into a single list.
204
-
205
- Attributes
206
- ----------
207
- run_list: list
208
- list of runs (integers)
209
- run_range:list
210
- min and max of range of runs to be simulated (two list entries)
211
-
212
- Returns
213
- -------
214
- list
215
- list of unique run numbers (integers)
216
- """
217
- if run_list is None and run_range is None:
218
- self.logger.debug("Nothing to prepare - run_list and run_range not given.")
219
- return None
220
-
221
- validated_runs = []
222
- if run_list is not None:
223
- validated_runs = gen.ensure_iterable(run_list)
224
- if not all(isinstance(r, int) for r in validated_runs):
225
- raise InvalidRunsToSimulateError(f"Run list must contain only integers: {run_list}")
226
-
227
- if run_range is not None:
228
- if not all(isinstance(r, int) for r in run_range) or len(run_range) != 2:
229
- raise InvalidRunsToSimulateError(
230
- f"Run_range must contain two integers only: {run_range}"
231
- )
232
- run_range = np.arange(run_range[0], run_range[1])
233
- validated_runs.extend(list(run_range))
234
-
235
- validated_runs_unique = sorted(set(validated_runs))
236
- self.logger.info(f"Runlist: {validated_runs_unique}")
237
- return validated_runs_unique
238
-
239
- def _corsika_configuration(self):
240
- """
241
- Define CORSIKA configurations based on the simulation model.
242
-
243
- For 'corsika_sim_telarray', this is a list since multiple configurations
244
- might be defined to run in a single job using multipipe.
245
-
246
- Returns
247
- -------
248
- CorsikaConfig or list of CorsikaConfig
249
- CORSIKA configuration(s) based on the simulation model.
250
- """
251
- corsika_configurations = []
252
- for array_model in self.array_models:
253
- corsika_configurations.append(
254
- CorsikaConfig(
255
- array_model=array_model,
256
- label=self.label,
257
- args_dict=self.args_dict,
258
- db_config=self.db_config,
259
- dummy_simulations=self._is_calibration_run(self.run_mode),
260
- )
261
- )
262
- return (
263
- corsika_configurations
264
- if self.simulation_software == "corsika_sim_telarray"
265
- else corsika_configurations[0]
266
- )
212
+ seed = semver_to_int(model_version) * 10000000
213
+ seed = seed + key_index(self.site) * 1000000
214
+ seed = seed + (int)(zenith_angle) * 1000
215
+ return seed + (int)(azimuth_angle)
267
216
 
268
217
  def _initialize_simulation_runner(self):
269
218
  """
@@ -274,8 +223,6 @@ class Simulator:
274
223
  CorsikaRunner or SimulatorArray or CorsikaSimtelRunner
275
224
  Simulation runner object.
276
225
  """
277
- corsika_configurations = self._corsika_configuration()
278
-
279
226
  runner_class = {
280
227
  "corsika": CorsikaRunner,
281
228
  "sim_telarray": SimulatorArray,
@@ -284,13 +231,16 @@ class Simulator:
284
231
 
285
232
  runner_args = {
286
233
  "label": self.label,
287
- "corsika_config": corsika_configurations,
234
+ "corsika_config": self.corsika_configurations,
288
235
  "simtel_path": self.args_dict.get("simtel_path"),
289
236
  "use_multipipe": runner_class is CorsikaSimtelRunner,
290
237
  }
291
238
 
292
239
  if runner_class is not SimulatorArray:
293
240
  runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
241
+ runner_args["curved_atmosphere_min_zenith_angle"] = self.args_dict.get(
242
+ "curved_atmosphere_min_zenith_angle", 65 * u.deg
243
+ )
294
244
  if runner_class is not CorsikaRunner:
295
245
  runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
296
246
  if runner_class is CorsikaSimtelRunner:
@@ -301,179 +251,90 @@ class Simulator:
301
251
 
302
252
  return runner_class(**runner_args)
303
253
 
304
- def _fill_results_without_run(self, input_file_list):
305
- """
306
- Fill results dict without calling submit (e.g., for testing).
307
-
308
- Parameters
309
- ----------
310
- input_file_list: str or list of str
311
- Single file or list of files of shower simulations.
254
+ def simulate(self):
312
255
  """
313
- for file in gen.ensure_iterable(input_file_list):
314
- run = self._guess_run_from_file(file)
315
- self._fill_results(file, run)
316
- if run not in self.runs:
317
- self.runs.append(run)
256
+ Prepare and submit a run script as a job.
318
257
 
319
- def simulate(self, input_file_list=None):
258
+ Writes submission scripts using the simulation runners and submits the
259
+ run script to the job manager. Collects generated files.
320
260
  """
321
- Submit a run script as a job.
322
-
323
- Parameters
324
- ----------
325
- input_file_list: str or list of str
326
- Single file or list of files of shower simulations.
327
- """
328
- runs_and_files_to_submit = self._get_runs_and_files_to_submit(
329
- input_file_list=input_file_list
330
- )
331
- self.logger.info(
332
- f"Starting submission for {len(runs_and_files_to_submit)} "
333
- f"run{'s' if len(runs_and_files_to_submit) > 1 else ''}"
261
+ run_script = self._simulation_runner.prepare_run_script(
262
+ run_number=self.run_number,
263
+ input_file=self._get_corsika_file(),
264
+ extra_commands=self._extra_commands,
334
265
  )
335
266
 
336
- for run_number, input_file in runs_and_files_to_submit.items():
337
- run_script = self._simulation_runner.prepare_run_script(
338
- run_number=run_number, input_file=input_file, extra_commands=self._extra_commands
339
- )
340
-
341
- job_manager = JobManager(test=self._test)
342
- job_manager.submit(
343
- run_script=run_script,
344
- run_out_file=self._simulation_runner.get_file_name(
345
- file_type="sub_log", run_number=run_number
346
- ),
347
- log_file=self._simulation_runner.get_file_name(
348
- file_type=("log"), run_number=run_number
349
- ),
350
- )
351
-
352
- self._fill_results(input_file, run_number)
353
-
354
- def _get_runs_and_files_to_submit(self, input_file_list=None):
355
- """
356
- Return a dictionary with run numbers and simulation files.
357
-
358
- The latter are expected to be given for the sim_telarray simulator.
359
-
360
- Parameters
361
- ----------
362
- input_file_list: str or list of str
363
- Single file or list of files of shower simulations.
364
-
365
- Returns
366
- -------
367
- runs_and_files: dict
368
- dictionary with run number as key and (if available) simulation
369
- file name as value
370
-
371
- Raises
372
- ------
373
- ValueError
374
- If no runs are to be submitted.
375
- """
376
- _runs_and_files = {}
377
- self.logger.debug(f"Getting runs and files to submit ({input_file_list})")
267
+ job_manager = JobManager(test=self._test)
268
+ job_manager.submit(
269
+ run_script=run_script,
270
+ run_out_file=self._simulation_runner.get_file_name(
271
+ file_type="sub_log", run_number=self.run_number
272
+ ),
273
+ log_file=self._simulation_runner.get_file_name(
274
+ file_type=("log"), run_number=self.run_number
275
+ ),
276
+ )
378
277
 
379
- if self.simulation_software == "sim_telarray" and self.run_mode is None:
380
- input_file_list = gen.ensure_iterable(input_file_list)
381
- _runs_and_files = {self._guess_run_from_file(file): file for file in input_file_list}
382
- else:
383
- _runs_and_files = dict.fromkeys(self._get_runs_to_simulate())
384
- if len(_runs_and_files) == 0:
385
- raise ValueError("No runs to submit.")
386
- return _runs_and_files
278
+ self._fill_list_of_generated_files()
387
279
 
388
- def _guess_run_from_file(self, file):
280
+ def _get_corsika_file(self):
389
281
  """
390
- Extract the run number from the given file name.
391
-
392
- Input file names can follow any pattern with the
393
- string 'run' followed by the run number.
394
-
395
- Parameters
396
- ----------
397
- file: Path
398
- Simulation file name
282
+ Get the CORSIKA input file if applicable (for sim_telarray simulations).
399
283
 
400
284
  Returns
401
285
  -------
402
- int
403
- The extracted run number. If extraction fails, returns 1 and logs a warning.
404
- """
405
- file_name = str(Path(file).name)
406
-
407
- try:
408
- run_str = re.search(r"run\d*", file_name).group()
409
- return int(run_str[3:])
410
- except (ValueError, AttributeError):
411
- self.logger.warning(f"Run number could not be guessed from {file_name} using run = 1")
412
- return 1
413
-
414
- def _fill_results(self, file, run_number):
415
- """
416
- Fill the results dict with input, output, hist, and log files.
417
-
418
- Parameters
419
- ----------
420
- file: str
421
- input file name
422
- run_number: int
423
- run number
424
-
425
- """
426
- keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
286
+ Path, None
287
+ Path to the CORSIKA input file.
288
+ """
289
+ if self.simulation_software == "sim_telarray":
290
+ return self.args_dict.get("corsika_file", None)
291
+ return None
292
+
293
+ def _fill_list_of_generated_files(self):
294
+ """Fill a dictionary with lists of generated files."""
295
+ keys = [
296
+ "simtel_output",
297
+ "sub_out",
298
+ "log",
299
+ "input",
300
+ "histogram",
301
+ "corsika_log",
302
+ "corsika_output",
303
+ "event_data",
304
+ ]
427
305
  results = {key: [] for key in keys}
428
306
 
429
307
  def get_file_name(name, **kwargs):
430
308
  return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
431
309
 
432
- if "sim_telarray" in self.simulation_software:
433
- results["input"].append(str(file))
434
-
435
- results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
310
+ results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=self.run_number))
436
311
 
437
312
  for i in range(len(self.array_models)):
438
313
  results["simtel_output"].append(
439
- get_file_name("simtel_output", run_number=run_number, model_version_index=i)
314
+ get_file_name("simtel_output", run_number=self.run_number, model_version_index=i)
440
315
  )
441
316
 
442
317
  if "sim_telarray" in self.simulation_software:
443
- results["log"].append(
444
- get_file_name(
445
- "log",
446
- simulation_software="sim_telarray",
447
- run_number=run_number,
448
- model_version_index=i,
449
- )
450
- )
451
- results["hist"].append(
452
- get_file_name(
453
- "histogram",
454
- simulation_software="sim_telarray",
455
- run_number=run_number,
456
- model_version_index=i,
457
- )
458
- )
459
- results["event_data"].append(
460
- get_file_name(
461
- "event_data",
462
- simulation_software="sim_telarray",
463
- run_number=run_number,
464
- model_version_index=i,
318
+ for file_type in ("log", "histogram", "event_data"):
319
+ results[file_type].append(
320
+ get_file_name(
321
+ file_type,
322
+ simulation_software="sim_telarray",
323
+ run_number=self.run_number,
324
+ model_version_index=i,
325
+ )
465
326
  )
466
- )
467
327
 
468
328
  if "corsika" in self.simulation_software:
469
- results["corsika_log"].append(
470
- get_file_name(
471
- "corsika_log",
472
- simulation_software="corsika",
473
- run_number=run_number,
474
- model_version_index=i,
329
+ for file_type in ("corsika_output", "corsika_log"):
330
+ results[file_type].append(
331
+ get_file_name(
332
+ file_type,
333
+ simulation_software="corsika",
334
+ run_number=self.run_number,
335
+ model_version_index=i,
336
+ )
475
337
  )
476
- )
477
338
 
478
339
  for key in keys:
479
340
  self._results[key].extend(results[key])
@@ -482,7 +343,6 @@ class Simulator:
482
343
  """
483
344
  Get list of files generated by simulations.
484
345
 
485
- Options are "input", "simtel_output", "hist", "log", "corsika_log".
486
346
  Not all file types are available for all simulation types.
487
347
  Returns an empty list for an unknown file type.
488
348
 
@@ -497,7 +357,6 @@ class Simulator:
497
357
  List with the full path of all output files.
498
358
 
499
359
  """
500
- self.logger.info(f"Getting list of {file_type} files")
501
360
  return self._results[file_type]
502
361
 
503
362
  def _make_resources_report(self, input_file_list):
@@ -511,38 +370,31 @@ class Simulator:
511
370
 
512
371
  Returns
513
372
  -------
514
- dict
515
- Dictionary with reports on computing resources
373
+ str
374
+ string reporting on computing resources
516
375
 
517
376
  """
518
- if len(self._results["sub_out"]) == 0:
519
- if input_file_list is None:
520
- return {"Wall time/run [sec]": np.nan}
521
- self._fill_results_without_run(input_file_list)
377
+ if len(self._results["sub_out"]) == 0 and input_file_list is None:
378
+ return "Mean wall time/run [sec]: np.nan"
522
379
 
523
380
  runtime = []
524
381
 
525
382
  _resources = {}
526
- for run in self.runs:
527
- _resources = self._simulation_runner.get_resources(run_number=run)
528
- if _resources.get("runtime"):
529
- runtime.append(_resources["runtime"])
383
+ _resources = self._simulation_runner.get_resources(run_number=self.run_number)
384
+ if _resources.get("runtime"):
385
+ runtime.append(_resources["runtime"])
530
386
 
531
387
  mean_runtime = np.mean(runtime)
532
388
 
533
- resource_summary = {}
534
- resource_summary["Wall time/run [sec]"] = mean_runtime
389
+ resource_summary = f"Mean wall time/run [sec]: {mean_runtime}"
535
390
  if "n_events" in _resources and _resources["n_events"] > 0:
536
- resource_summary["#events/run"] = _resources["n_events"]
537
- resource_summary["Wall time/1000 events [sec]"] = (
538
- mean_runtime * 1000 / _resources["n_events"]
539
- )
391
+ resource_summary += f", #events/run: {_resources['n_events']}"
540
392
 
541
393
  return resource_summary
542
394
 
543
- def resources(self, input_file_list=None):
395
+ def report(self, input_file_list=None):
544
396
  """
545
- Print a simple report on computing resources used.
397
+ Report on simulations and computing resources used.
546
398
 
547
399
  Includes run time per run only at this point.
548
400
 
@@ -552,37 +404,21 @@ class Simulator:
552
404
  Single file or list of files of shower simulations.
553
405
 
554
406
  """
555
- resources = self._make_resources_report(input_file_list)
556
- print("-----------------------------")
557
- print(f"Computing Resources Report - {self.simulation_software} Simulations")
558
- for key, value in resources.items():
559
- print(f"{key} = {value:.2f}")
560
- print("-----------------------------")
561
-
562
- def _get_runs_to_simulate(self, run_list=None, run_range=None):
563
- """
564
- Process run_list and run_range and return the validated list of runs.
565
-
566
- Attributes
567
- ----------
568
- run_list: list
569
- list of runs (integers)
570
- run_range:list
571
- min and max of range of runs to be simulated (two list entries)
572
-
573
- Returns
574
- -------
575
- list
576
- list of unique run numbers (integers)
577
-
578
- """
579
- if run_list is None and run_range is None:
580
- return [] if self.runs is None else self.runs
581
- return self._prepare_run_list_and_range(run_list, run_range)
407
+ _corsika_config = self._get_first_corsika_config()
408
+ self.logger.info(
409
+ f"Production run complete for primary {_corsika_config.primary_particle} showers "
410
+ f"from {_corsika_config.azimuth_angle} azimuth and "
411
+ f"{_corsika_config.zenith_angle} zenith "
412
+ f"at {self.site} site, using {self.model_version} model."
413
+ )
414
+ self.logger.info(
415
+ f"Computing for {self.simulation_software} Simulations: "
416
+ f"{self._make_resources_report(input_file_list)}"
417
+ )
582
418
 
583
419
  def save_file_lists(self):
584
420
  """Save files lists for output and log files."""
585
- for file_type in ["simtel_output", "log", "corsika_log", "hist"]:
421
+ for file_type in ["simtel_output", "log", "corsika_log", "histogram"]:
586
422
  file_name = self.io_handler.get_output_directory().joinpath(f"{file_type}_files.txt")
587
423
  file_list = self.get_file_list(file_type=file_type)
588
424
  if all(element is not None for element in file_list) and len(file_list) > 0:
@@ -634,7 +470,7 @@ class Simulator:
634
470
  output_files = self.get_file_list(file_type="simtel_output")
635
471
  log_files = self.get_file_list(file_type="log")
636
472
  corsika_log_files = self.get_file_list(file_type="corsika_log")
637
- histogram_files = self.get_file_list(file_type="hist")
473
+ histogram_files = self.get_file_list(file_type="histogram")
638
474
  reduced_event_files = (
639
475
  self.get_file_list(file_type="event_data")
640
476
  if self.args_dict.get("save_reduced_event_lists")
@@ -656,7 +492,7 @@ class Simulator:
656
492
  # Group files by model version
657
493
  for model in self.array_models:
658
494
  model_version = model.model_version
659
- model_files = gen.ensure_iterable(model.pack_model_files())
495
+ model_files = general.ensure_iterable(model.pack_model_files())
660
496
 
661
497
  # Filter files for this model version
662
498
  model_logs = [f for f in log_files if model_version in f]
@@ -668,7 +504,7 @@ class Simulator:
668
504
  tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
669
505
  # Add all relevant model, log, histogram, and CORSIKA log files to the tarball
670
506
  files_to_tar = model_logs + model_hists + model_corsika_logs + model_files
671
- gen.pack_tar_file(tar_file_path, files_to_tar)
507
+ general.pack_tar_file(tar_file_path, files_to_tar)
672
508
 
673
509
  for file_to_move in output_files + reduced_event_files:
674
510
  source_file = Path(file_to_move)
@@ -762,8 +598,8 @@ class Simulator:
762
598
  """
763
599
  return run_mode in [
764
600
  "pedestals",
765
- "dark_pedestals",
766
- "nsb_only_pedestals",
601
+ "pedestals_dark",
602
+ "pedestals_nsb_only",
767
603
  "direct_injection",
768
604
  ]
769
605
 
@@ -785,3 +621,182 @@ class Simulator:
785
621
  if run_mode == "direct_injection":
786
622
  return ["flat_fielding"]
787
623
  return []
624
+
625
+ def _get_first_corsika_config(self):
626
+ """
627
+ Return first instance from list of CORSIKA configurations.
628
+
629
+ Most values stored in the CORSIKA configurations are identical,
630
+ with the exception of the simulation model version dependent parameters.
631
+
632
+ Returns
633
+ -------
634
+ CorsikaConfig
635
+ First CORSIKA configuration instance.
636
+ """
637
+ try:
638
+ return (
639
+ self.corsika_configurations[0]
640
+ if isinstance(self.corsika_configurations, list)
641
+ else self.corsika_configurations
642
+ )
643
+ except (IndexError, TypeError) as exc:
644
+ raise ValueError("CORSIKA configuration not found for verification.") from exc
645
+
646
+ def verify_simulations(self):
647
+ """
648
+ Verify simulations.
649
+
650
+ This includes checking the number of simulated events.
651
+
652
+ """
653
+ self.logger.info("Verifying simulations.")
654
+
655
+ _corsika_config = self._get_first_corsika_config()
656
+ expected_shower_events = _corsika_config.shower_events
657
+ expected_mc_events = _corsika_config.mc_events
658
+
659
+ self.logger.info(
660
+ f"Expected number of shower events: {expected_shower_events}, "
661
+ f"expected number of MC events: {expected_mc_events}"
662
+ )
663
+ if self.simulation_software in ["corsika_sim_telarray", "sim_telarray"]:
664
+ self._verify_simulated_events_in_sim_telarray(
665
+ expected_shower_events, expected_mc_events
666
+ )
667
+ if self.simulation_software == "corsika":
668
+ self._verify_simulated_events_corsika(expected_mc_events)
669
+ if self.args_dict.get("save_reduced_event_lists"):
670
+ self._verify_simulated_events_in_reduced_event_lists(expected_mc_events)
671
+
672
+ def _verify_simulated_events_corsika(self, expected_mc_events, tolerance=1.0e-3):
673
+ """
674
+ Verify the number of simulated events in CORSIKA output files.
675
+
676
+ Allow for a small mismatch in the number of requested events.
677
+
678
+ Parameters
679
+ ----------
680
+ expected_mc_events: int
681
+ Expected number of simulated MC events.
682
+
683
+ Raises
684
+ ------
685
+ ValueError
686
+ If the number of simulated events does not match the expected number.
687
+ """
688
+
689
+ def consistent(a, b, tol):
690
+ return abs(a - b) / max(a, b) <= tol
691
+
692
+ event_errors = []
693
+ for file in self.get_file_list(file_type="corsika_output"):
694
+ shower_events, _ = eventio_handler.get_simulated_events(file)
695
+
696
+ if shower_events != expected_mc_events:
697
+ if consistent(shower_events, expected_mc_events, tol=tolerance):
698
+ self.logger.warning(
699
+ f"Small mismatch in number of events in: {file}: "
700
+ f"shower events: {shower_events} (expected: {expected_mc_events})"
701
+ )
702
+ continue
703
+ event_errors.append(
704
+ f"Number of simulated MC events ({shower_events}) does not match "
705
+ f"the expected number ({expected_mc_events}) in CORSIKA {file}."
706
+ )
707
+ else:
708
+ self.logger.info(
709
+ f"Consistent number of events in: {file}: shower events: {shower_events}"
710
+ )
711
+
712
+ if event_errors:
713
+ self.logger.error("Inconsistent event counts found in CORSIKA output:")
714
+ for error in event_errors:
715
+ self.logger.error(f" - {error}")
716
+ error_message = "Inconsistent event counts found in CORSIKA output:\n" + "\n".join(
717
+ f" - {error}" for error in event_errors
718
+ )
719
+ raise ValueError(error_message)
720
+
721
+ def _verify_simulated_events_in_sim_telarray(self, expected_shower_events, expected_mc_events):
722
+ """
723
+ Verify the number of simulated events.
724
+
725
+ Parameters
726
+ ----------
727
+ expected_shower_events: int
728
+ Expected number of simulated shower events.
729
+ expected_mc_events: int
730
+ Expected number of simulated MC events.
731
+
732
+ Raises
733
+ ------
734
+ ValueError
735
+ If the number of simulated events does not match the expected number.
736
+ """
737
+ event_errors = []
738
+ for file in self.get_file_list(file_type="simtel_output"):
739
+ shower_events, mc_events = eventio_handler.get_simulated_events(file)
740
+
741
+ if (shower_events, mc_events) != (expected_shower_events, expected_mc_events):
742
+ event_errors.append(
743
+ f"Event mismatch: shower/MC events in {file}: {shower_events}/{mc_events}"
744
+ f" (expected: {expected_shower_events}/{expected_mc_events})"
745
+ )
746
+ else:
747
+ self.logger.info(
748
+ f"Consistent number of events in: {file}: "
749
+ f"shower events: {shower_events}, "
750
+ f"MC events: {mc_events}"
751
+ )
752
+
753
+ if event_errors:
754
+ self.logger.error("Inconsistent event counts found:")
755
+ for error in event_errors:
756
+ self.logger.error(f" - {error}")
757
+ error_message = "Inconsistent event counts found:\n" + "\n".join(
758
+ f" - {error}" for error in event_errors
759
+ )
760
+ raise ValueError(error_message)
761
+
762
+ def _verify_simulated_events_in_reduced_event_lists(self, expected_mc_events):
763
+ """
764
+ Verify the number of simulated events in reduced event lists.
765
+
766
+ Parameters
767
+ ----------
768
+ expected_mc_events: int
769
+ Expected number of simulated MC events.
770
+
771
+ Raises
772
+ ------
773
+ ValueError
774
+ If the number of simulated events does not match the expected number.
775
+ """
776
+ event_errors = []
777
+ for file in self.get_file_list(file_type="event_data"):
778
+ tables = table_handler.read_tables(file, ["SHOWERS"])
779
+ try:
780
+ mc_events = len(tables["SHOWERS"])
781
+ except KeyError as exc:
782
+ raise ValueError(f"SHOWERS table not found in reduced event list {file}.") from exc
783
+
784
+ if mc_events != expected_mc_events:
785
+ event_errors.append(
786
+ f"Number of simulated MC events ({mc_events}) does not match "
787
+ f"the expected number ({expected_mc_events}) in reduced event list {file}."
788
+ )
789
+ else:
790
+ self.logger.info(
791
+ f"Consistent number of events in reduced event list: {file}: MC events:"
792
+ f" {mc_events}"
793
+ )
794
+
795
+ if event_errors:
796
+ self.logger.error("Inconsistent event counts found in reduced event lists:")
797
+ for error in event_errors:
798
+ self.logger.error(f" - {error}")
799
+ error_message = "Inconsistent event counts found in reduced event lists:\n" + "\n".join(
800
+ f" - {error}" for error in event_errors
801
+ )
802
+ raise ValueError(error_message)