gammasimtools 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/METADATA +1 -1
  2. {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/RECORD +58 -55
  3. {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/entry_points.txt +1 -0
  4. simtools/_version.py +2 -2
  5. simtools/application_control.py +50 -0
  6. simtools/applications/derive_psf_parameters.py +5 -0
  7. simtools/applications/derive_pulse_shape_parameters.py +195 -0
  8. simtools/applications/plot_array_layout.py +63 -1
  9. simtools/applications/simulate_flasher.py +3 -2
  10. simtools/applications/simulate_pedestals.py +1 -1
  11. simtools/applications/simulate_prod.py +8 -23
  12. simtools/applications/simulate_prod_htcondor_generator.py +7 -0
  13. simtools/applications/submit_array_layouts.py +5 -3
  14. simtools/applications/validate_file_using_schema.py +49 -123
  15. simtools/configuration/commandline_parser.py +8 -6
  16. simtools/corsika/corsika_config.py +197 -87
  17. simtools/data_model/model_data_writer.py +14 -2
  18. simtools/data_model/schema.py +112 -5
  19. simtools/data_model/validate_data.py +82 -48
  20. simtools/db/db_model_upload.py +2 -1
  21. simtools/db/mongo_db.py +133 -42
  22. simtools/dependencies.py +5 -9
  23. simtools/io/eventio_handler.py +128 -0
  24. simtools/job_execution/htcondor_script_generator.py +0 -2
  25. simtools/layout/array_layout_utils.py +1 -1
  26. simtools/model/array_model.py +36 -5
  27. simtools/model/model_parameter.py +0 -1
  28. simtools/model/model_repository.py +18 -5
  29. simtools/ray_tracing/psf_analysis.py +11 -8
  30. simtools/ray_tracing/psf_parameter_optimisation.py +822 -679
  31. simtools/reporting/docs_read_parameters.py +69 -9
  32. simtools/runners/corsika_runner.py +12 -3
  33. simtools/runners/corsika_simtel_runner.py +6 -0
  34. simtools/runners/runner_services.py +17 -7
  35. simtools/runners/simtel_runner.py +12 -54
  36. simtools/schemas/model_parameters/flasher_pulse_exp_decay.schema.yml +2 -0
  37. simtools/schemas/model_parameters/flasher_pulse_shape.schema.yml +50 -0
  38. simtools/schemas/model_parameters/flasher_pulse_width.schema.yml +2 -0
  39. simtools/schemas/simulation_models_info.schema.yml +2 -0
  40. simtools/simtel/pulse_shapes.py +268 -0
  41. simtools/simtel/simtel_config_writer.py +82 -1
  42. simtools/simtel/simtel_io_event_writer.py +2 -2
  43. simtools/simtel/simulator_array.py +58 -12
  44. simtools/simtel/simulator_light_emission.py +45 -8
  45. simtools/simulator.py +361 -347
  46. simtools/testing/assertions.py +62 -6
  47. simtools/testing/configuration.py +1 -1
  48. simtools/testing/log_inspector.py +4 -1
  49. simtools/testing/sim_telarray_metadata.py +1 -1
  50. simtools/testing/validate_output.py +44 -9
  51. simtools/utils/names.py +2 -4
  52. simtools/version.py +37 -0
  53. simtools/visualization/legend_handlers.py +14 -4
  54. simtools/visualization/plot_array_layout.py +229 -33
  55. simtools/visualization/plot_mirrors.py +837 -0
  56. simtools/simtel/simtel_io_file_info.py +0 -62
  57. {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/WHEEL +0 -0
  58. {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/licenses/LICENSE +0 -0
  59. {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/top_level.txt +0 -0
simtools/simulator.py CHANGED
@@ -10,9 +10,8 @@ from pathlib import Path
10
10
  import numpy as np
11
11
  from astropy import units as u
12
12
 
13
- import simtools.utils.general as gen
14
13
  from simtools.corsika.corsika_config import CorsikaConfig
15
- from simtools.io import io_handler, table_handler
14
+ from simtools.io import eventio_handler, io_handler, table_handler
16
15
  from simtools.job_execution.job_manager import JobManager
17
16
  from simtools.model.array_model import ArrayModel
18
17
  from simtools.runners.corsika_runner import CorsikaRunner
@@ -20,18 +19,16 @@ from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
20
19
  from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
21
20
  from simtools.simtel.simulator_array import SimulatorArray
22
21
  from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
22
+ from simtools.utils import general, names
23
23
  from simtools.version import semver_to_int
24
24
 
25
25
 
26
- class InvalidRunsToSimulateError(Exception):
27
- """Exception for invalid runs to simulate."""
28
-
29
-
30
26
  class Simulator:
31
27
  """
32
28
  Simulator is managing the simulation of showers and of the array of telescopes.
33
29
 
34
30
  It interfaces with simulation software packages (e.g., CORSIKA or sim_telarray).
31
+ A single run is simulated per instance, possibly for multiple model versions.
35
32
 
36
33
  The configuration is set as a dict corresponding to the command line configuration groups
37
34
  (especially simulation_software, simulation_model, simulation_parameters).
@@ -62,6 +59,8 @@ class Simulator:
62
59
 
63
60
  self.args_dict = args_dict
64
61
  self.db_config = db_config
62
+ self.site = self.args_dict.get("site", None)
63
+ self.model_version = self.args_dict.get("model_version", None)
65
64
 
66
65
  self.simulation_software = self.args_dict.get("simulation_software", "corsika_sim_telarray")
67
66
  self.logger.debug(f"Init Simulator {self.simulation_software}")
@@ -69,19 +68,13 @@ class Simulator:
69
68
 
70
69
  self.io_handler = io_handler.IOHandler()
71
70
 
72
- self.runs = self._initialize_run_list()
71
+ self.run_number = None
73
72
  self._results = defaultdict(list)
74
- self._test = self.args_dict.get("test", False)
73
+ self._test = None
75
74
  self._extra_commands = extra_commands
76
-
77
- self.sim_telarray_seeds = {
78
- "seed": self.args_dict.get("sim_telarray_instrument_seeds"),
79
- "random_instrument_instances": self.args_dict.get(
80
- "sim_telarray_random_instrument_instances"
81
- ),
82
- "seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
83
- }
84
- self.array_models = self._initialize_array_models()
75
+ self.sim_telarray_seeds = None
76
+ self._initialize_from_tool_configuration()
77
+ self.array_models, self.corsika_configurations = self._initialize_array_models()
85
78
  self._simulation_runner = self._initialize_simulation_runner()
86
79
 
87
80
  @property
@@ -109,43 +102,85 @@ class Simulator:
109
102
  raise ValueError(f"Invalid simulation software: {simulation_software}")
110
103
  self._simulation_software = simulation_software.lower()
111
104
 
105
+ def _initialize_from_tool_configuration(self):
106
+ """Initialize simulator from tool configuration."""
107
+ self._test = self.args_dict.get("test", False)
108
+ self.sim_telarray_seeds = {
109
+ "seed": self.args_dict.get("sim_telarray_instrument_seeds"),
110
+ "random_instrument_instances": self.args_dict.get(
111
+ "sim_telarray_random_instrument_instances"
112
+ ),
113
+ "seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
114
+ }
115
+
116
+ if self.args_dict.get("corsika_file"):
117
+ self.run_number = eventio_handler.get_corsika_run_number(self.args_dict["corsika_file"])
118
+ else:
119
+ self.run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
120
+ "run_number", 1
121
+ )
122
+
112
123
  def _initialize_array_models(self):
113
124
  """
114
- Initialize array simulation models.
125
+ Initialize array simulation models and CORSIKA config (one per model version).
115
126
 
116
127
  Returns
117
128
  -------
118
129
  list
119
130
  List of ArrayModel objects.
120
131
  """
121
- versions = gen.ensure_iterable(self.args_dict.get("model_version", []))
122
-
123
- return [
124
- ArrayModel(
125
- label=self.label,
126
- site=self.args_dict.get("site"),
127
- layout_name=self.args_dict.get("array_layout_name"),
128
- db_config=self.db_config,
129
- model_version=version,
130
- sim_telarray_seeds={
131
- "seed": self._get_seed_for_random_instrument_instances(
132
- self.sim_telarray_seeds["seed"], version
133
- ),
134
- "random_instrument_instances": self.sim_telarray_seeds[
135
- "random_instrument_instances"
136
- ],
137
- "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
138
- },
139
- simtel_path=self.args_dict.get("simtel_path", None),
140
- calibration_device_types=self._get_calibration_device_types(
141
- self.args_dict.get("run_mode")
142
- ),
143
- overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters", None),
132
+ versions = general.ensure_iterable(self.model_version)
133
+
134
+ array_model = []
135
+ corsika_configurations = []
136
+
137
+ for version in versions:
138
+ array_model.append(
139
+ ArrayModel(
140
+ label=self.label,
141
+ site=self.site,
142
+ layout_name=self.args_dict.get("array_layout_name"),
143
+ db_config=self.db_config,
144
+ model_version=version,
145
+ calibration_device_types=self._get_calibration_device_types(self.run_mode),
146
+ overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters"),
147
+ simtel_path=self.args_dict.get("simtel_path"),
148
+ )
144
149
  )
145
- for version in versions
146
- ]
150
+ corsika_configurations.append(
151
+ CorsikaConfig(
152
+ array_model=array_model[-1],
153
+ label=self.label,
154
+ args_dict=self.args_dict,
155
+ db_config=self.db_config,
156
+ dummy_simulations=self._is_calibration_run(self.run_mode),
157
+ )
158
+ )
159
+ array_model[-1].sim_telarray_seeds = {
160
+ "seed": self._get_seed_for_random_instrument_instances(
161
+ self.sim_telarray_seeds["seed"],
162
+ version,
163
+ corsika_configurations[-1].zenith_angle,
164
+ corsika_configurations[-1].azimuth_angle,
165
+ ),
166
+ "random_instrument_instances": self.sim_telarray_seeds[
167
+ "random_instrument_instances"
168
+ ],
169
+ "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
170
+ }
171
+
172
+ # 'corsika_sim_telarray' allows for multiple model versions (multipipe option)
173
+ corsika_configurations = (
174
+ corsika_configurations
175
+ if self.simulation_software == "corsika_sim_telarray"
176
+ else corsika_configurations[0]
177
+ )
178
+
179
+ return array_model, corsika_configurations
147
180
 
148
- def _get_seed_for_random_instrument_instances(self, seed, model_version):
181
+ def _get_seed_for_random_instrument_instances(
182
+ self, seed, model_version, zenith_angle, azimuth_angle
183
+ ):
149
184
  """
150
185
  Generate seed for random instances of the instrument.
151
186
 
@@ -155,6 +190,10 @@ class Simulator:
155
190
  Seed string given through configuration.
156
191
  model_version: str
157
192
  Model version.
193
+ zenith_angle: float
194
+ Zenith angle of the observation (in degrees).
195
+ azimuth_angle: float
196
+ Azimuth angle of the observation (in degrees).
158
197
 
159
198
  Returns
160
199
  -------
@@ -164,107 +203,16 @@ class Simulator:
164
203
  if seed:
165
204
  return int(seed.split(",")[0].strip())
166
205
 
167
- seed = semver_to_int(model_version) * 10000000
168
- seed = seed + 1000000 if self.args_dict.get("site") != "North" else seed + 2000000
169
- seed = seed + (int)(self.args_dict.get("zenith_angle", 0.0 * u.deg).value) * 1000
170
- return seed + (int)(self.args_dict.get("azimuth_angle", 0.0 * u.deg).value)
171
-
172
- def _initialize_run_list(self):
173
- """
174
- Initialize run list using the configuration values.
175
-
176
- Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
177
- to create a list of run numbers.
178
-
179
- Returns
180
- -------
181
- list
182
- List of run numbers.
183
- """
184
- offset_run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
185
- "run_number", 1
186
- )
187
- if self.args_dict.get("number_of_runs", 1) <= 1:
188
- return self._prepare_run_list_and_range(
189
- run_list=offset_run_number,
190
- run_range=None,
191
- )
192
- return self._prepare_run_list_and_range(
193
- run_list=None,
194
- run_range=[
195
- offset_run_number,
196
- offset_run_number + self.args_dict["number_of_runs"],
197
- ],
198
- )
199
-
200
- def _prepare_run_list_and_range(self, run_list, run_range):
201
- """
202
- Prepare list of run numbers from a list or from a range.
206
+ def key_index(key):
207
+ try:
208
+ return list(names.site_names()).index(key) + 1
209
+ except ValueError:
210
+ return 1
203
211
 
204
- If both arguments are given, they will be merged into a single list.
205
-
206
- Attributes
207
- ----------
208
- run_list: list
209
- list of runs (integers)
210
- run_range:list
211
- min and max of range of runs to be simulated (two list entries)
212
-
213
- Returns
214
- -------
215
- list
216
- list of unique run numbers (integers)
217
- """
218
- if run_list is None and run_range is None:
219
- self.logger.debug("Nothing to prepare - run_list and run_range not given.")
220
- return None
221
-
222
- validated_runs = []
223
- if run_list is not None:
224
- validated_runs = gen.ensure_iterable(run_list)
225
- if not all(isinstance(r, int) for r in validated_runs):
226
- raise InvalidRunsToSimulateError(f"Run list must contain only integers: {run_list}")
227
-
228
- if run_range is not None:
229
- if not all(isinstance(r, int) for r in run_range) or len(run_range) != 2:
230
- raise InvalidRunsToSimulateError(
231
- f"Run_range must contain two integers only: {run_range}"
232
- )
233
- run_range = np.arange(run_range[0], run_range[1])
234
- validated_runs.extend(list(run_range))
235
-
236
- validated_runs_unique = sorted(set(validated_runs))
237
- self.logger.info(f"Runlist: {validated_runs_unique}")
238
- return validated_runs_unique
239
-
240
- def _corsika_configuration(self):
241
- """
242
- Define CORSIKA configurations based on the simulation model.
243
-
244
- For 'corsika_sim_telarray', this is a list since multiple configurations
245
- might be defined to run in a single job using multipipe.
246
-
247
- Returns
248
- -------
249
- CorsikaConfig or list of CorsikaConfig
250
- CORSIKA configuration(s) based on the simulation model.
251
- """
252
- corsika_configurations = []
253
- for array_model in self.array_models:
254
- corsika_configurations.append(
255
- CorsikaConfig(
256
- array_model=array_model,
257
- label=self.label,
258
- args_dict=self.args_dict,
259
- db_config=self.db_config,
260
- dummy_simulations=self._is_calibration_run(self.run_mode),
261
- )
262
- )
263
- return (
264
- corsika_configurations
265
- if self.simulation_software == "corsika_sim_telarray"
266
- else corsika_configurations[0]
267
- )
212
+ seed = semver_to_int(model_version) * 10000000
213
+ seed = seed + key_index(self.site) * 1000000
214
+ seed = seed + (int)(zenith_angle) * 1000
215
+ return seed + (int)(azimuth_angle)
268
216
 
269
217
  def _initialize_simulation_runner(self):
270
218
  """
@@ -275,8 +223,6 @@ class Simulator:
275
223
  CorsikaRunner or SimulatorArray or CorsikaSimtelRunner
276
224
  Simulation runner object.
277
225
  """
278
- corsika_configurations = self._corsika_configuration()
279
-
280
226
  runner_class = {
281
227
  "corsika": CorsikaRunner,
282
228
  "sim_telarray": SimulatorArray,
@@ -285,13 +231,16 @@ class Simulator:
285
231
 
286
232
  runner_args = {
287
233
  "label": self.label,
288
- "corsika_config": corsika_configurations,
234
+ "corsika_config": self.corsika_configurations,
289
235
  "simtel_path": self.args_dict.get("simtel_path"),
290
236
  "use_multipipe": runner_class is CorsikaSimtelRunner,
291
237
  }
292
238
 
293
239
  if runner_class is not SimulatorArray:
294
240
  runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
241
+ runner_args["curved_atmosphere_min_zenith_angle"] = self.args_dict.get(
242
+ "curved_atmosphere_min_zenith_angle", 65 * u.deg
243
+ )
295
244
  if runner_class is not CorsikaRunner:
296
245
  runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
297
246
  if runner_class is CorsikaSimtelRunner:
@@ -302,179 +251,90 @@ class Simulator:
302
251
 
303
252
  return runner_class(**runner_args)
304
253
 
305
- def _fill_results_without_run(self, input_file_list):
306
- """
307
- Fill results dict without calling submit (e.g., for testing).
308
-
309
- Parameters
310
- ----------
311
- input_file_list: str or list of str
312
- Single file or list of files of shower simulations.
254
+ def simulate(self):
313
255
  """
314
- for file in gen.ensure_iterable(input_file_list):
315
- run = self._guess_run_from_file(file)
316
- self._fill_results(file, run)
317
- if run not in self.runs:
318
- self.runs.append(run)
256
+ Prepare and submit a run script as a job.
319
257
 
320
- def simulate(self, input_file_list=None):
258
+ Writes submission scripts using the simulation runners and submits the
259
+ run script to the job manager. Collects generated files.
321
260
  """
322
- Submit a run script as a job.
323
-
324
- Parameters
325
- ----------
326
- input_file_list: str or list of str
327
- Single file or list of files of shower simulations.
328
- """
329
- runs_and_files_to_submit = self._get_runs_and_files_to_submit(
330
- input_file_list=input_file_list
331
- )
332
- self.logger.info(
333
- f"Starting submission for {len(runs_and_files_to_submit)} "
334
- f"run{'s' if len(runs_and_files_to_submit) > 1 else ''}"
261
+ run_script = self._simulation_runner.prepare_run_script(
262
+ run_number=self.run_number,
263
+ input_file=self._get_corsika_file(),
264
+ extra_commands=self._extra_commands,
335
265
  )
336
266
 
337
- for run_number, input_file in runs_and_files_to_submit.items():
338
- run_script = self._simulation_runner.prepare_run_script(
339
- run_number=run_number, input_file=input_file, extra_commands=self._extra_commands
340
- )
341
-
342
- job_manager = JobManager(test=self._test)
343
- job_manager.submit(
344
- run_script=run_script,
345
- run_out_file=self._simulation_runner.get_file_name(
346
- file_type="sub_log", run_number=run_number
347
- ),
348
- log_file=self._simulation_runner.get_file_name(
349
- file_type=("log"), run_number=run_number
350
- ),
351
- )
352
-
353
- self._fill_results(input_file, run_number)
354
-
355
- def _get_runs_and_files_to_submit(self, input_file_list=None):
356
- """
357
- Return a dictionary with run numbers and simulation files.
358
-
359
- The latter are expected to be given for the sim_telarray simulator.
360
-
361
- Parameters
362
- ----------
363
- input_file_list: str or list of str
364
- Single file or list of files of shower simulations.
365
-
366
- Returns
367
- -------
368
- runs_and_files: dict
369
- dictionary with run number as key and (if available) simulation
370
- file name as value
371
-
372
- Raises
373
- ------
374
- ValueError
375
- If no runs are to be submitted.
376
- """
377
- _runs_and_files = {}
378
- self.logger.debug(f"Getting runs and files to submit ({input_file_list})")
267
+ job_manager = JobManager(test=self._test)
268
+ job_manager.submit(
269
+ run_script=run_script,
270
+ run_out_file=self._simulation_runner.get_file_name(
271
+ file_type="sub_log", run_number=self.run_number
272
+ ),
273
+ log_file=self._simulation_runner.get_file_name(
274
+ file_type=("log"), run_number=self.run_number
275
+ ),
276
+ )
379
277
 
380
- if self.simulation_software == "sim_telarray" and self.run_mode is None:
381
- input_file_list = gen.ensure_iterable(input_file_list)
382
- _runs_and_files = {self._guess_run_from_file(file): file for file in input_file_list}
383
- else:
384
- _runs_and_files = dict.fromkeys(self._get_runs_to_simulate())
385
- if len(_runs_and_files) == 0:
386
- raise ValueError("No runs to submit.")
387
- return _runs_and_files
278
+ self._fill_list_of_generated_files()
388
279
 
389
- def _guess_run_from_file(self, file):
280
+ def _get_corsika_file(self):
390
281
  """
391
- Extract the run number from the given file name.
392
-
393
- Input file names can follow any pattern with the
394
- string 'run' followed by the run number.
395
-
396
- Parameters
397
- ----------
398
- file: Path
399
- Simulation file name
282
+ Get the CORSIKA input file if applicable (for sim_telarray simulations).
400
283
 
401
284
  Returns
402
285
  -------
403
- int
404
- The extracted run number. If extraction fails, returns 1 and logs a warning.
405
- """
406
- file_name = str(Path(file).name)
407
-
408
- try:
409
- run_str = re.search(r"run\d*", file_name).group()
410
- return int(run_str[3:])
411
- except (ValueError, AttributeError):
412
- self.logger.warning(f"Run number could not be guessed from {file_name} using run = 1")
413
- return 1
414
-
415
- def _fill_results(self, file, run_number):
416
- """
417
- Fill the results dict with input, output, hist, and log files.
418
-
419
- Parameters
420
- ----------
421
- file: str
422
- input file name
423
- run_number: int
424
- run number
425
-
426
- """
427
- keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
286
+ Path, None
287
+ Path to the CORSIKA input file.
288
+ """
289
+ if self.simulation_software == "sim_telarray":
290
+ return self.args_dict.get("corsika_file", None)
291
+ return None
292
+
293
+ def _fill_list_of_generated_files(self):
294
+ """Fill a dictionary with lists of generated files."""
295
+ keys = [
296
+ "simtel_output",
297
+ "sub_out",
298
+ "log",
299
+ "input",
300
+ "histogram",
301
+ "corsika_log",
302
+ "corsika_output",
303
+ "event_data",
304
+ ]
428
305
  results = {key: [] for key in keys}
429
306
 
430
307
  def get_file_name(name, **kwargs):
431
308
  return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
432
309
 
433
- if "sim_telarray" in self.simulation_software:
434
- results["input"].append(str(file))
435
-
436
- results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
310
+ results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=self.run_number))
437
311
 
438
312
  for i in range(len(self.array_models)):
439
313
  results["simtel_output"].append(
440
- get_file_name("simtel_output", run_number=run_number, model_version_index=i)
314
+ get_file_name("simtel_output", run_number=self.run_number, model_version_index=i)
441
315
  )
442
316
 
443
317
  if "sim_telarray" in self.simulation_software:
444
- results["log"].append(
445
- get_file_name(
446
- "log",
447
- simulation_software="sim_telarray",
448
- run_number=run_number,
449
- model_version_index=i,
450
- )
451
- )
452
- results["hist"].append(
453
- get_file_name(
454
- "histogram",
455
- simulation_software="sim_telarray",
456
- run_number=run_number,
457
- model_version_index=i,
458
- )
459
- )
460
- results["event_data"].append(
461
- get_file_name(
462
- "event_data",
463
- simulation_software="sim_telarray",
464
- run_number=run_number,
465
- model_version_index=i,
318
+ for file_type in ("log", "histogram", "event_data"):
319
+ results[file_type].append(
320
+ get_file_name(
321
+ file_type,
322
+ simulation_software="sim_telarray",
323
+ run_number=self.run_number,
324
+ model_version_index=i,
325
+ )
466
326
  )
467
- )
468
327
 
469
328
  if "corsika" in self.simulation_software:
470
- results["corsika_log"].append(
471
- get_file_name(
472
- "corsika_log",
473
- simulation_software="corsika",
474
- run_number=run_number,
475
- model_version_index=i,
329
+ for file_type in ("corsika_output", "corsika_log"):
330
+ results[file_type].append(
331
+ get_file_name(
332
+ file_type,
333
+ simulation_software="corsika",
334
+ run_number=self.run_number,
335
+ model_version_index=i,
336
+ )
476
337
  )
477
- )
478
338
 
479
339
  for key in keys:
480
340
  self._results[key].extend(results[key])
@@ -483,7 +343,6 @@ class Simulator:
483
343
  """
484
344
  Get list of files generated by simulations.
485
345
 
486
- Options are "input", "simtel_output", "hist", "log", "corsika_log".
487
346
  Not all file types are available for all simulation types.
488
347
  Returns an empty list for an unknown file type.
489
348
 
@@ -498,7 +357,6 @@ class Simulator:
498
357
  List with the full path of all output files.
499
358
 
500
359
  """
501
- self.logger.info(f"Getting list of {file_type} files")
502
360
  return self._results[file_type]
503
361
 
504
362
  def _make_resources_report(self, input_file_list):
@@ -512,38 +370,31 @@ class Simulator:
512
370
 
513
371
  Returns
514
372
  -------
515
- dict
516
- Dictionary with reports on computing resources
373
+ str
374
+ string reporting on computing resources
517
375
 
518
376
  """
519
- if len(self._results["sub_out"]) == 0:
520
- if input_file_list is None:
521
- return {"Wall time/run [sec]": np.nan}
522
- self._fill_results_without_run(input_file_list)
377
+ if len(self._results["sub_out"]) == 0 and input_file_list is None:
378
+ return "Mean wall time/run [sec]: np.nan"
523
379
 
524
380
  runtime = []
525
381
 
526
382
  _resources = {}
527
- for run in self.runs:
528
- _resources = self._simulation_runner.get_resources(run_number=run)
529
- if _resources.get("runtime"):
530
- runtime.append(_resources["runtime"])
383
+ _resources = self._simulation_runner.get_resources(run_number=self.run_number)
384
+ if _resources.get("runtime"):
385
+ runtime.append(_resources["runtime"])
531
386
 
532
387
  mean_runtime = np.mean(runtime)
533
388
 
534
- resource_summary = {}
535
- resource_summary["Wall time/run [sec]"] = mean_runtime
389
+ resource_summary = f"Mean wall time/run [sec]: {mean_runtime}"
536
390
  if "n_events" in _resources and _resources["n_events"] > 0:
537
- resource_summary["#events/run"] = _resources["n_events"]
538
- resource_summary["Wall time/1000 events [sec]"] = (
539
- mean_runtime * 1000 / _resources["n_events"]
540
- )
391
+ resource_summary += f", #events/run: {_resources['n_events']}"
541
392
 
542
393
  return resource_summary
543
394
 
544
- def resources(self, input_file_list=None):
395
+ def report(self, input_file_list=None):
545
396
  """
546
- Print a simple report on computing resources used.
397
+ Report on simulations and computing resources used.
547
398
 
548
399
  Includes run time per run only at this point.
549
400
 
@@ -553,37 +404,21 @@ class Simulator:
553
404
  Single file or list of files of shower simulations.
554
405
 
555
406
  """
556
- resources = self._make_resources_report(input_file_list)
557
- print("-----------------------------")
558
- print(f"Computing Resources Report - {self.simulation_software} Simulations")
559
- for key, value in resources.items():
560
- print(f"{key} = {value:.2f}")
561
- print("-----------------------------")
562
-
563
- def _get_runs_to_simulate(self, run_list=None, run_range=None):
564
- """
565
- Process run_list and run_range and return the validated list of runs.
566
-
567
- Attributes
568
- ----------
569
- run_list: list
570
- list of runs (integers)
571
- run_range:list
572
- min and max of range of runs to be simulated (two list entries)
573
-
574
- Returns
575
- -------
576
- list
577
- list of unique run numbers (integers)
578
-
579
- """
580
- if run_list is None and run_range is None:
581
- return [] if self.runs is None else self.runs
582
- return self._prepare_run_list_and_range(run_list, run_range)
407
+ _corsika_config = self._get_first_corsika_config()
408
+ self.logger.info(
409
+ f"Production run complete for primary {_corsika_config.primary_particle} showers "
410
+ f"from {_corsika_config.azimuth_angle} azimuth and "
411
+ f"{_corsika_config.zenith_angle} zenith "
412
+ f"at {self.site} site, using {self.model_version} model."
413
+ )
414
+ self.logger.info(
415
+ f"Computing for {self.simulation_software} Simulations: "
416
+ f"{self._make_resources_report(input_file_list)}"
417
+ )
583
418
 
584
419
  def save_file_lists(self):
585
420
  """Save files lists for output and log files."""
586
- for file_type in ["simtel_output", "log", "corsika_log", "hist"]:
421
+ for file_type in ["simtel_output", "log", "corsika_log", "histogram"]:
587
422
  file_name = self.io_handler.get_output_directory().joinpath(f"{file_type}_files.txt")
588
423
  file_list = self.get_file_list(file_type=file_type)
589
424
  if all(element is not None for element in file_list) and len(file_list) > 0:
@@ -635,7 +470,7 @@ class Simulator:
635
470
  output_files = self.get_file_list(file_type="simtel_output")
636
471
  log_files = self.get_file_list(file_type="log")
637
472
  corsika_log_files = self.get_file_list(file_type="corsika_log")
638
- histogram_files = self.get_file_list(file_type="hist")
473
+ histogram_files = self.get_file_list(file_type="histogram")
639
474
  reduced_event_files = (
640
475
  self.get_file_list(file_type="event_data")
641
476
  if self.args_dict.get("save_reduced_event_lists")
@@ -657,7 +492,7 @@ class Simulator:
657
492
  # Group files by model version
658
493
  for model in self.array_models:
659
494
  model_version = model.model_version
660
- model_files = gen.ensure_iterable(model.pack_model_files())
495
+ model_files = general.ensure_iterable(model.pack_model_files())
661
496
 
662
497
  # Filter files for this model version
663
498
  model_logs = [f for f in log_files if model_version in f]
@@ -669,7 +504,7 @@ class Simulator:
669
504
  tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
670
505
  # Add all relevant model, log, histogram, and CORSIKA log files to the tarball
671
506
  files_to_tar = model_logs + model_hists + model_corsika_logs + model_files
672
- gen.pack_tar_file(tar_file_path, files_to_tar)
507
+ general.pack_tar_file(tar_file_path, files_to_tar)
673
508
 
674
509
  for file_to_move in output_files + reduced_event_files:
675
510
  source_file = Path(file_to_move)
@@ -763,8 +598,8 @@ class Simulator:
763
598
  """
764
599
  return run_mode in [
765
600
  "pedestals",
766
- "dark_pedestals",
767
- "nsb_only_pedestals",
601
+ "pedestals_dark",
602
+ "pedestals_nsb_only",
768
603
  "direct_injection",
769
604
  ]
770
605
 
@@ -786,3 +621,182 @@ class Simulator:
786
621
  if run_mode == "direct_injection":
787
622
  return ["flat_fielding"]
788
623
  return []
624
+
625
+ def _get_first_corsika_config(self):
626
+ """
627
+ Return first instance from list of CORSIKA configurations.
628
+
629
+ Most values stored in the CORSIKA configurations are identical,
630
+ with the exception of the simulation model version dependent parameters.
631
+
632
+ Returns
633
+ -------
634
+ CorsikaConfig
635
+ First CORSIKA configuration instance.
636
+ """
637
+ try:
638
+ return (
639
+ self.corsika_configurations[0]
640
+ if isinstance(self.corsika_configurations, list)
641
+ else self.corsika_configurations
642
+ )
643
+ except (IndexError, TypeError) as exc:
644
+ raise ValueError("CORSIKA configuration not found for verification.") from exc
645
+
646
+ def verify_simulations(self):
647
+ """
648
+ Verify simulations.
649
+
650
+ This includes checking the number of simulated events.
651
+
652
+ """
653
+ self.logger.info("Verifying simulations.")
654
+
655
+ _corsika_config = self._get_first_corsika_config()
656
+ expected_shower_events = _corsika_config.shower_events
657
+ expected_mc_events = _corsika_config.mc_events
658
+
659
+ self.logger.info(
660
+ f"Expected number of shower events: {expected_shower_events}, "
661
+ f"expected number of MC events: {expected_mc_events}"
662
+ )
663
+ if self.simulation_software in ["corsika_sim_telarray", "sim_telarray"]:
664
+ self._verify_simulated_events_in_sim_telarray(
665
+ expected_shower_events, expected_mc_events
666
+ )
667
+ if self.simulation_software == "corsika":
668
+ self._verify_simulated_events_corsika(expected_mc_events)
669
+ if self.args_dict.get("save_reduced_event_lists"):
670
+ self._verify_simulated_events_in_reduced_event_lists(expected_mc_events)
671
+
672
+ def _verify_simulated_events_corsika(self, expected_mc_events, tolerance=1.0e-3):
673
+ """
674
+ Verify the number of simulated events in CORSIKA output files.
675
+
676
+ Allow for a small mismatch in the number of requested events.
677
+
678
+ Parameters
679
+ ----------
680
+ expected_mc_events: int
681
+ Expected number of simulated MC events.
682
+
683
+ Raises
684
+ ------
685
+ ValueError
686
+ If the number of simulated events does not match the expected number.
687
+ """
688
+
689
+ def consistent(a, b, tol):
690
+ return abs(a - b) / max(a, b) <= tol
691
+
692
+ event_errors = []
693
+ for file in self.get_file_list(file_type="corsika_output"):
694
+ shower_events, _ = eventio_handler.get_simulated_events(file)
695
+
696
+ if shower_events != expected_mc_events:
697
+ if consistent(shower_events, expected_mc_events, tol=tolerance):
698
+ self.logger.warning(
699
+ f"Small mismatch in number of events in: {file}: "
700
+ f"shower events: {shower_events} (expected: {expected_mc_events})"
701
+ )
702
+ continue
703
+ event_errors.append(
704
+ f"Number of simulated MC events ({shower_events}) does not match "
705
+ f"the expected number ({expected_mc_events}) in CORSIKA {file}."
706
+ )
707
+ else:
708
+ self.logger.info(
709
+ f"Consistent number of events in: {file}: shower events: {shower_events}"
710
+ )
711
+
712
+ if event_errors:
713
+ self.logger.error("Inconsistent event counts found in CORSIKA output:")
714
+ for error in event_errors:
715
+ self.logger.error(f" - {error}")
716
+ error_message = "Inconsistent event counts found in CORSIKA output:\n" + "\n".join(
717
+ f" - {error}" for error in event_errors
718
+ )
719
+ raise ValueError(error_message)
720
+
721
+ def _verify_simulated_events_in_sim_telarray(self, expected_shower_events, expected_mc_events):
722
+ """
723
+ Verify the number of simulated events.
724
+
725
+ Parameters
726
+ ----------
727
+ expected_shower_events: int
728
+ Expected number of simulated shower events.
729
+ expected_mc_events: int
730
+ Expected number of simulated MC events.
731
+
732
+ Raises
733
+ ------
734
+ ValueError
735
+ If the number of simulated events does not match the expected number.
736
+ """
737
+ event_errors = []
738
+ for file in self.get_file_list(file_type="simtel_output"):
739
+ shower_events, mc_events = eventio_handler.get_simulated_events(file)
740
+
741
+ if (shower_events, mc_events) != (expected_shower_events, expected_mc_events):
742
+ event_errors.append(
743
+ f"Event mismatch: shower/MC events in {file}: {shower_events}/{mc_events}"
744
+ f" (expected: {expected_shower_events}/{expected_mc_events})"
745
+ )
746
+ else:
747
+ self.logger.info(
748
+ f"Consistent number of events in: {file}: "
749
+ f"shower events: {shower_events}, "
750
+ f"MC events: {mc_events}"
751
+ )
752
+
753
+ if event_errors:
754
+ self.logger.error("Inconsistent event counts found:")
755
+ for error in event_errors:
756
+ self.logger.error(f" - {error}")
757
+ error_message = "Inconsistent event counts found:\n" + "\n".join(
758
+ f" - {error}" for error in event_errors
759
+ )
760
+ raise ValueError(error_message)
761
+
762
+ def _verify_simulated_events_in_reduced_event_lists(self, expected_mc_events):
763
+ """
764
+ Verify the number of simulated events in reduced event lists.
765
+
766
+ Parameters
767
+ ----------
768
+ expected_mc_events: int
769
+ Expected number of simulated MC events.
770
+
771
+ Raises
772
+ ------
773
+ ValueError
774
+ If the number of simulated events does not match the expected number.
775
+ """
776
+ event_errors = []
777
+ for file in self.get_file_list(file_type="event_data"):
778
+ tables = table_handler.read_tables(file, ["SHOWERS"])
779
+ try:
780
+ mc_events = len(tables["SHOWERS"])
781
+ except KeyError as exc:
782
+ raise ValueError(f"SHOWERS table not found in reduced event list {file}.") from exc
783
+
784
+ if mc_events != expected_mc_events:
785
+ event_errors.append(
786
+ f"Number of simulated MC events ({mc_events}) does not match "
787
+ f"the expected number ({expected_mc_events}) in reduced event list {file}."
788
+ )
789
+ else:
790
+ self.logger.info(
791
+ f"Consistent number of events in reduced event list: {file}: MC events:"
792
+ f" {mc_events}"
793
+ )
794
+
795
+ if event_errors:
796
+ self.logger.error("Inconsistent event counts found in reduced event lists:")
797
+ for error in event_errors:
798
+ self.logger.error(f" - {error}")
799
+ error_message = "Inconsistent event counts found in reduced event lists:\n" + "\n".join(
800
+ f" - {error}" for error in event_errors
801
+ )
802
+ raise ValueError(error_message)