gammasimtools 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/METADATA +1 -1
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/RECORD +58 -55
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/entry_points.txt +1 -0
- simtools/_version.py +2 -2
- simtools/application_control.py +50 -0
- simtools/applications/derive_psf_parameters.py +5 -0
- simtools/applications/derive_pulse_shape_parameters.py +195 -0
- simtools/applications/plot_array_layout.py +63 -1
- simtools/applications/simulate_flasher.py +3 -2
- simtools/applications/simulate_pedestals.py +1 -1
- simtools/applications/simulate_prod.py +8 -23
- simtools/applications/simulate_prod_htcondor_generator.py +7 -0
- simtools/applications/submit_array_layouts.py +5 -3
- simtools/applications/validate_file_using_schema.py +49 -123
- simtools/configuration/commandline_parser.py +8 -6
- simtools/corsika/corsika_config.py +197 -87
- simtools/data_model/model_data_writer.py +14 -2
- simtools/data_model/schema.py +112 -5
- simtools/data_model/validate_data.py +82 -48
- simtools/db/db_model_upload.py +2 -1
- simtools/db/mongo_db.py +133 -42
- simtools/dependencies.py +5 -9
- simtools/io/eventio_handler.py +128 -0
- simtools/job_execution/htcondor_script_generator.py +0 -2
- simtools/layout/array_layout_utils.py +1 -1
- simtools/model/array_model.py +36 -5
- simtools/model/model_parameter.py +0 -1
- simtools/model/model_repository.py +18 -5
- simtools/ray_tracing/psf_analysis.py +11 -8
- simtools/ray_tracing/psf_parameter_optimisation.py +822 -679
- simtools/reporting/docs_read_parameters.py +69 -9
- simtools/runners/corsika_runner.py +12 -3
- simtools/runners/corsika_simtel_runner.py +6 -0
- simtools/runners/runner_services.py +17 -7
- simtools/runners/simtel_runner.py +12 -54
- simtools/schemas/model_parameters/flasher_pulse_exp_decay.schema.yml +2 -0
- simtools/schemas/model_parameters/flasher_pulse_shape.schema.yml +50 -0
- simtools/schemas/model_parameters/flasher_pulse_width.schema.yml +2 -0
- simtools/schemas/simulation_models_info.schema.yml +2 -0
- simtools/simtel/pulse_shapes.py +268 -0
- simtools/simtel/simtel_config_writer.py +82 -1
- simtools/simtel/simtel_io_event_writer.py +2 -2
- simtools/simtel/simulator_array.py +58 -12
- simtools/simtel/simulator_light_emission.py +45 -8
- simtools/simulator.py +361 -347
- simtools/testing/assertions.py +62 -6
- simtools/testing/configuration.py +1 -1
- simtools/testing/log_inspector.py +4 -1
- simtools/testing/sim_telarray_metadata.py +1 -1
- simtools/testing/validate_output.py +44 -9
- simtools/utils/names.py +2 -4
- simtools/version.py +37 -0
- simtools/visualization/legend_handlers.py +14 -4
- simtools/visualization/plot_array_layout.py +229 -33
- simtools/visualization/plot_mirrors.py +837 -0
- simtools/simtel/simtel_io_file_info.py +0 -62
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/WHEEL +0 -0
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/licenses/LICENSE +0 -0
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.25.0.dist-info}/top_level.txt +0 -0
simtools/simulator.py
CHANGED
|
@@ -10,9 +10,8 @@ from pathlib import Path
|
|
|
10
10
|
import numpy as np
|
|
11
11
|
from astropy import units as u
|
|
12
12
|
|
|
13
|
-
import simtools.utils.general as gen
|
|
14
13
|
from simtools.corsika.corsika_config import CorsikaConfig
|
|
15
|
-
from simtools.io import io_handler, table_handler
|
|
14
|
+
from simtools.io import eventio_handler, io_handler, table_handler
|
|
16
15
|
from simtools.job_execution.job_manager import JobManager
|
|
17
16
|
from simtools.model.array_model import ArrayModel
|
|
18
17
|
from simtools.runners.corsika_runner import CorsikaRunner
|
|
@@ -20,18 +19,16 @@ from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
|
|
|
20
19
|
from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
|
|
21
20
|
from simtools.simtel.simulator_array import SimulatorArray
|
|
22
21
|
from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
|
|
22
|
+
from simtools.utils import general, names
|
|
23
23
|
from simtools.version import semver_to_int
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
class InvalidRunsToSimulateError(Exception):
|
|
27
|
-
"""Exception for invalid runs to simulate."""
|
|
28
|
-
|
|
29
|
-
|
|
30
26
|
class Simulator:
|
|
31
27
|
"""
|
|
32
28
|
Simulator is managing the simulation of showers and of the array of telescopes.
|
|
33
29
|
|
|
34
30
|
It interfaces with simulation software packages (e.g., CORSIKA or sim_telarray).
|
|
31
|
+
A single run is simulated per instance, possibly for multiple model versions.
|
|
35
32
|
|
|
36
33
|
The configuration is set as a dict corresponding to the command line configuration groups
|
|
37
34
|
(especially simulation_software, simulation_model, simulation_parameters).
|
|
@@ -62,6 +59,8 @@ class Simulator:
|
|
|
62
59
|
|
|
63
60
|
self.args_dict = args_dict
|
|
64
61
|
self.db_config = db_config
|
|
62
|
+
self.site = self.args_dict.get("site", None)
|
|
63
|
+
self.model_version = self.args_dict.get("model_version", None)
|
|
65
64
|
|
|
66
65
|
self.simulation_software = self.args_dict.get("simulation_software", "corsika_sim_telarray")
|
|
67
66
|
self.logger.debug(f"Init Simulator {self.simulation_software}")
|
|
@@ -69,19 +68,13 @@ class Simulator:
|
|
|
69
68
|
|
|
70
69
|
self.io_handler = io_handler.IOHandler()
|
|
71
70
|
|
|
72
|
-
self.
|
|
71
|
+
self.run_number = None
|
|
73
72
|
self._results = defaultdict(list)
|
|
74
|
-
self._test =
|
|
73
|
+
self._test = None
|
|
75
74
|
self._extra_commands = extra_commands
|
|
76
|
-
|
|
77
|
-
self.
|
|
78
|
-
|
|
79
|
-
"random_instrument_instances": self.args_dict.get(
|
|
80
|
-
"sim_telarray_random_instrument_instances"
|
|
81
|
-
),
|
|
82
|
-
"seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
|
|
83
|
-
}
|
|
84
|
-
self.array_models = self._initialize_array_models()
|
|
75
|
+
self.sim_telarray_seeds = None
|
|
76
|
+
self._initialize_from_tool_configuration()
|
|
77
|
+
self.array_models, self.corsika_configurations = self._initialize_array_models()
|
|
85
78
|
self._simulation_runner = self._initialize_simulation_runner()
|
|
86
79
|
|
|
87
80
|
@property
|
|
@@ -109,43 +102,85 @@ class Simulator:
|
|
|
109
102
|
raise ValueError(f"Invalid simulation software: {simulation_software}")
|
|
110
103
|
self._simulation_software = simulation_software.lower()
|
|
111
104
|
|
|
105
|
+
def _initialize_from_tool_configuration(self):
|
|
106
|
+
"""Initialize simulator from tool configuration."""
|
|
107
|
+
self._test = self.args_dict.get("test", False)
|
|
108
|
+
self.sim_telarray_seeds = {
|
|
109
|
+
"seed": self.args_dict.get("sim_telarray_instrument_seeds"),
|
|
110
|
+
"random_instrument_instances": self.args_dict.get(
|
|
111
|
+
"sim_telarray_random_instrument_instances"
|
|
112
|
+
),
|
|
113
|
+
"seed_file_name": "sim_telarray_instrument_seeds.txt", # name only; no directory
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if self.args_dict.get("corsika_file"):
|
|
117
|
+
self.run_number = eventio_handler.get_corsika_run_number(self.args_dict["corsika_file"])
|
|
118
|
+
else:
|
|
119
|
+
self.run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
|
|
120
|
+
"run_number", 1
|
|
121
|
+
)
|
|
122
|
+
|
|
112
123
|
def _initialize_array_models(self):
|
|
113
124
|
"""
|
|
114
|
-
Initialize array simulation models.
|
|
125
|
+
Initialize array simulation models and CORSIKA config (one per model version).
|
|
115
126
|
|
|
116
127
|
Returns
|
|
117
128
|
-------
|
|
118
129
|
list
|
|
119
130
|
List of ArrayModel objects.
|
|
120
131
|
"""
|
|
121
|
-
versions =
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
},
|
|
139
|
-
simtel_path=self.args_dict.get("simtel_path", None),
|
|
140
|
-
calibration_device_types=self._get_calibration_device_types(
|
|
141
|
-
self.args_dict.get("run_mode")
|
|
142
|
-
),
|
|
143
|
-
overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters", None),
|
|
132
|
+
versions = general.ensure_iterable(self.model_version)
|
|
133
|
+
|
|
134
|
+
array_model = []
|
|
135
|
+
corsika_configurations = []
|
|
136
|
+
|
|
137
|
+
for version in versions:
|
|
138
|
+
array_model.append(
|
|
139
|
+
ArrayModel(
|
|
140
|
+
label=self.label,
|
|
141
|
+
site=self.site,
|
|
142
|
+
layout_name=self.args_dict.get("array_layout_name"),
|
|
143
|
+
db_config=self.db_config,
|
|
144
|
+
model_version=version,
|
|
145
|
+
calibration_device_types=self._get_calibration_device_types(self.run_mode),
|
|
146
|
+
overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters"),
|
|
147
|
+
simtel_path=self.args_dict.get("simtel_path"),
|
|
148
|
+
)
|
|
144
149
|
)
|
|
145
|
-
|
|
146
|
-
|
|
150
|
+
corsika_configurations.append(
|
|
151
|
+
CorsikaConfig(
|
|
152
|
+
array_model=array_model[-1],
|
|
153
|
+
label=self.label,
|
|
154
|
+
args_dict=self.args_dict,
|
|
155
|
+
db_config=self.db_config,
|
|
156
|
+
dummy_simulations=self._is_calibration_run(self.run_mode),
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
array_model[-1].sim_telarray_seeds = {
|
|
160
|
+
"seed": self._get_seed_for_random_instrument_instances(
|
|
161
|
+
self.sim_telarray_seeds["seed"],
|
|
162
|
+
version,
|
|
163
|
+
corsika_configurations[-1].zenith_angle,
|
|
164
|
+
corsika_configurations[-1].azimuth_angle,
|
|
165
|
+
),
|
|
166
|
+
"random_instrument_instances": self.sim_telarray_seeds[
|
|
167
|
+
"random_instrument_instances"
|
|
168
|
+
],
|
|
169
|
+
"seed_file_name": self.sim_telarray_seeds["seed_file_name"],
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
# 'corsika_sim_telarray' allows for multiple model versions (multipipe option)
|
|
173
|
+
corsika_configurations = (
|
|
174
|
+
corsika_configurations
|
|
175
|
+
if self.simulation_software == "corsika_sim_telarray"
|
|
176
|
+
else corsika_configurations[0]
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
return array_model, corsika_configurations
|
|
147
180
|
|
|
148
|
-
def _get_seed_for_random_instrument_instances(
|
|
181
|
+
def _get_seed_for_random_instrument_instances(
|
|
182
|
+
self, seed, model_version, zenith_angle, azimuth_angle
|
|
183
|
+
):
|
|
149
184
|
"""
|
|
150
185
|
Generate seed for random instances of the instrument.
|
|
151
186
|
|
|
@@ -155,6 +190,10 @@ class Simulator:
|
|
|
155
190
|
Seed string given through configuration.
|
|
156
191
|
model_version: str
|
|
157
192
|
Model version.
|
|
193
|
+
zenith_angle: float
|
|
194
|
+
Zenith angle of the observation (in degrees).
|
|
195
|
+
azimuth_angle: float
|
|
196
|
+
Azimuth angle of the observation (in degrees).
|
|
158
197
|
|
|
159
198
|
Returns
|
|
160
199
|
-------
|
|
@@ -164,107 +203,16 @@ class Simulator:
|
|
|
164
203
|
if seed:
|
|
165
204
|
return int(seed.split(",")[0].strip())
|
|
166
205
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
def _initialize_run_list(self):
|
|
173
|
-
"""
|
|
174
|
-
Initialize run list using the configuration values.
|
|
175
|
-
|
|
176
|
-
Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
|
|
177
|
-
to create a list of run numbers.
|
|
178
|
-
|
|
179
|
-
Returns
|
|
180
|
-
-------
|
|
181
|
-
list
|
|
182
|
-
List of run numbers.
|
|
183
|
-
"""
|
|
184
|
-
offset_run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
|
|
185
|
-
"run_number", 1
|
|
186
|
-
)
|
|
187
|
-
if self.args_dict.get("number_of_runs", 1) <= 1:
|
|
188
|
-
return self._prepare_run_list_and_range(
|
|
189
|
-
run_list=offset_run_number,
|
|
190
|
-
run_range=None,
|
|
191
|
-
)
|
|
192
|
-
return self._prepare_run_list_and_range(
|
|
193
|
-
run_list=None,
|
|
194
|
-
run_range=[
|
|
195
|
-
offset_run_number,
|
|
196
|
-
offset_run_number + self.args_dict["number_of_runs"],
|
|
197
|
-
],
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
def _prepare_run_list_and_range(self, run_list, run_range):
|
|
201
|
-
"""
|
|
202
|
-
Prepare list of run numbers from a list or from a range.
|
|
206
|
+
def key_index(key):
|
|
207
|
+
try:
|
|
208
|
+
return list(names.site_names()).index(key) + 1
|
|
209
|
+
except ValueError:
|
|
210
|
+
return 1
|
|
203
211
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
run_list: list
|
|
209
|
-
list of runs (integers)
|
|
210
|
-
run_range:list
|
|
211
|
-
min and max of range of runs to be simulated (two list entries)
|
|
212
|
-
|
|
213
|
-
Returns
|
|
214
|
-
-------
|
|
215
|
-
list
|
|
216
|
-
list of unique run numbers (integers)
|
|
217
|
-
"""
|
|
218
|
-
if run_list is None and run_range is None:
|
|
219
|
-
self.logger.debug("Nothing to prepare - run_list and run_range not given.")
|
|
220
|
-
return None
|
|
221
|
-
|
|
222
|
-
validated_runs = []
|
|
223
|
-
if run_list is not None:
|
|
224
|
-
validated_runs = gen.ensure_iterable(run_list)
|
|
225
|
-
if not all(isinstance(r, int) for r in validated_runs):
|
|
226
|
-
raise InvalidRunsToSimulateError(f"Run list must contain only integers: {run_list}")
|
|
227
|
-
|
|
228
|
-
if run_range is not None:
|
|
229
|
-
if not all(isinstance(r, int) for r in run_range) or len(run_range) != 2:
|
|
230
|
-
raise InvalidRunsToSimulateError(
|
|
231
|
-
f"Run_range must contain two integers only: {run_range}"
|
|
232
|
-
)
|
|
233
|
-
run_range = np.arange(run_range[0], run_range[1])
|
|
234
|
-
validated_runs.extend(list(run_range))
|
|
235
|
-
|
|
236
|
-
validated_runs_unique = sorted(set(validated_runs))
|
|
237
|
-
self.logger.info(f"Runlist: {validated_runs_unique}")
|
|
238
|
-
return validated_runs_unique
|
|
239
|
-
|
|
240
|
-
def _corsika_configuration(self):
|
|
241
|
-
"""
|
|
242
|
-
Define CORSIKA configurations based on the simulation model.
|
|
243
|
-
|
|
244
|
-
For 'corsika_sim_telarray', this is a list since multiple configurations
|
|
245
|
-
might be defined to run in a single job using multipipe.
|
|
246
|
-
|
|
247
|
-
Returns
|
|
248
|
-
-------
|
|
249
|
-
CorsikaConfig or list of CorsikaConfig
|
|
250
|
-
CORSIKA configuration(s) based on the simulation model.
|
|
251
|
-
"""
|
|
252
|
-
corsika_configurations = []
|
|
253
|
-
for array_model in self.array_models:
|
|
254
|
-
corsika_configurations.append(
|
|
255
|
-
CorsikaConfig(
|
|
256
|
-
array_model=array_model,
|
|
257
|
-
label=self.label,
|
|
258
|
-
args_dict=self.args_dict,
|
|
259
|
-
db_config=self.db_config,
|
|
260
|
-
dummy_simulations=self._is_calibration_run(self.run_mode),
|
|
261
|
-
)
|
|
262
|
-
)
|
|
263
|
-
return (
|
|
264
|
-
corsika_configurations
|
|
265
|
-
if self.simulation_software == "corsika_sim_telarray"
|
|
266
|
-
else corsika_configurations[0]
|
|
267
|
-
)
|
|
212
|
+
seed = semver_to_int(model_version) * 10000000
|
|
213
|
+
seed = seed + key_index(self.site) * 1000000
|
|
214
|
+
seed = seed + (int)(zenith_angle) * 1000
|
|
215
|
+
return seed + (int)(azimuth_angle)
|
|
268
216
|
|
|
269
217
|
def _initialize_simulation_runner(self):
|
|
270
218
|
"""
|
|
@@ -275,8 +223,6 @@ class Simulator:
|
|
|
275
223
|
CorsikaRunner or SimulatorArray or CorsikaSimtelRunner
|
|
276
224
|
Simulation runner object.
|
|
277
225
|
"""
|
|
278
|
-
corsika_configurations = self._corsika_configuration()
|
|
279
|
-
|
|
280
226
|
runner_class = {
|
|
281
227
|
"corsika": CorsikaRunner,
|
|
282
228
|
"sim_telarray": SimulatorArray,
|
|
@@ -285,13 +231,16 @@ class Simulator:
|
|
|
285
231
|
|
|
286
232
|
runner_args = {
|
|
287
233
|
"label": self.label,
|
|
288
|
-
"corsika_config": corsika_configurations,
|
|
234
|
+
"corsika_config": self.corsika_configurations,
|
|
289
235
|
"simtel_path": self.args_dict.get("simtel_path"),
|
|
290
236
|
"use_multipipe": runner_class is CorsikaSimtelRunner,
|
|
291
237
|
}
|
|
292
238
|
|
|
293
239
|
if runner_class is not SimulatorArray:
|
|
294
240
|
runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
|
|
241
|
+
runner_args["curved_atmosphere_min_zenith_angle"] = self.args_dict.get(
|
|
242
|
+
"curved_atmosphere_min_zenith_angle", 65 * u.deg
|
|
243
|
+
)
|
|
295
244
|
if runner_class is not CorsikaRunner:
|
|
296
245
|
runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
|
|
297
246
|
if runner_class is CorsikaSimtelRunner:
|
|
@@ -302,179 +251,90 @@ class Simulator:
|
|
|
302
251
|
|
|
303
252
|
return runner_class(**runner_args)
|
|
304
253
|
|
|
305
|
-
def
|
|
306
|
-
"""
|
|
307
|
-
Fill results dict without calling submit (e.g., for testing).
|
|
308
|
-
|
|
309
|
-
Parameters
|
|
310
|
-
----------
|
|
311
|
-
input_file_list: str or list of str
|
|
312
|
-
Single file or list of files of shower simulations.
|
|
254
|
+
def simulate(self):
|
|
313
255
|
"""
|
|
314
|
-
|
|
315
|
-
run = self._guess_run_from_file(file)
|
|
316
|
-
self._fill_results(file, run)
|
|
317
|
-
if run not in self.runs:
|
|
318
|
-
self.runs.append(run)
|
|
256
|
+
Prepare and submit a run script as a job.
|
|
319
257
|
|
|
320
|
-
|
|
258
|
+
Writes submission scripts using the simulation runners and submits the
|
|
259
|
+
run script to the job manager. Collects generated files.
|
|
321
260
|
"""
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
input_file_list: str or list of str
|
|
327
|
-
Single file or list of files of shower simulations.
|
|
328
|
-
"""
|
|
329
|
-
runs_and_files_to_submit = self._get_runs_and_files_to_submit(
|
|
330
|
-
input_file_list=input_file_list
|
|
331
|
-
)
|
|
332
|
-
self.logger.info(
|
|
333
|
-
f"Starting submission for {len(runs_and_files_to_submit)} "
|
|
334
|
-
f"run{'s' if len(runs_and_files_to_submit) > 1 else ''}"
|
|
261
|
+
run_script = self._simulation_runner.prepare_run_script(
|
|
262
|
+
run_number=self.run_number,
|
|
263
|
+
input_file=self._get_corsika_file(),
|
|
264
|
+
extra_commands=self._extra_commands,
|
|
335
265
|
)
|
|
336
266
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
),
|
|
348
|
-
log_file=self._simulation_runner.get_file_name(
|
|
349
|
-
file_type=("log"), run_number=run_number
|
|
350
|
-
),
|
|
351
|
-
)
|
|
352
|
-
|
|
353
|
-
self._fill_results(input_file, run_number)
|
|
354
|
-
|
|
355
|
-
def _get_runs_and_files_to_submit(self, input_file_list=None):
|
|
356
|
-
"""
|
|
357
|
-
Return a dictionary with run numbers and simulation files.
|
|
358
|
-
|
|
359
|
-
The latter are expected to be given for the sim_telarray simulator.
|
|
360
|
-
|
|
361
|
-
Parameters
|
|
362
|
-
----------
|
|
363
|
-
input_file_list: str or list of str
|
|
364
|
-
Single file or list of files of shower simulations.
|
|
365
|
-
|
|
366
|
-
Returns
|
|
367
|
-
-------
|
|
368
|
-
runs_and_files: dict
|
|
369
|
-
dictionary with run number as key and (if available) simulation
|
|
370
|
-
file name as value
|
|
371
|
-
|
|
372
|
-
Raises
|
|
373
|
-
------
|
|
374
|
-
ValueError
|
|
375
|
-
If no runs are to be submitted.
|
|
376
|
-
"""
|
|
377
|
-
_runs_and_files = {}
|
|
378
|
-
self.logger.debug(f"Getting runs and files to submit ({input_file_list})")
|
|
267
|
+
job_manager = JobManager(test=self._test)
|
|
268
|
+
job_manager.submit(
|
|
269
|
+
run_script=run_script,
|
|
270
|
+
run_out_file=self._simulation_runner.get_file_name(
|
|
271
|
+
file_type="sub_log", run_number=self.run_number
|
|
272
|
+
),
|
|
273
|
+
log_file=self._simulation_runner.get_file_name(
|
|
274
|
+
file_type=("log"), run_number=self.run_number
|
|
275
|
+
),
|
|
276
|
+
)
|
|
379
277
|
|
|
380
|
-
|
|
381
|
-
input_file_list = gen.ensure_iterable(input_file_list)
|
|
382
|
-
_runs_and_files = {self._guess_run_from_file(file): file for file in input_file_list}
|
|
383
|
-
else:
|
|
384
|
-
_runs_and_files = dict.fromkeys(self._get_runs_to_simulate())
|
|
385
|
-
if len(_runs_and_files) == 0:
|
|
386
|
-
raise ValueError("No runs to submit.")
|
|
387
|
-
return _runs_and_files
|
|
278
|
+
self._fill_list_of_generated_files()
|
|
388
279
|
|
|
389
|
-
def
|
|
280
|
+
def _get_corsika_file(self):
|
|
390
281
|
"""
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
Input file names can follow any pattern with the
|
|
394
|
-
string 'run' followed by the run number.
|
|
395
|
-
|
|
396
|
-
Parameters
|
|
397
|
-
----------
|
|
398
|
-
file: Path
|
|
399
|
-
Simulation file name
|
|
282
|
+
Get the CORSIKA input file if applicable (for sim_telarray simulations).
|
|
400
283
|
|
|
401
284
|
Returns
|
|
402
285
|
-------
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
"""
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
input file name
|
|
423
|
-
run_number: int
|
|
424
|
-
run number
|
|
425
|
-
|
|
426
|
-
"""
|
|
427
|
-
keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
|
|
286
|
+
Path, None
|
|
287
|
+
Path to the CORSIKA input file.
|
|
288
|
+
"""
|
|
289
|
+
if self.simulation_software == "sim_telarray":
|
|
290
|
+
return self.args_dict.get("corsika_file", None)
|
|
291
|
+
return None
|
|
292
|
+
|
|
293
|
+
def _fill_list_of_generated_files(self):
|
|
294
|
+
"""Fill a dictionary with lists of generated files."""
|
|
295
|
+
keys = [
|
|
296
|
+
"simtel_output",
|
|
297
|
+
"sub_out",
|
|
298
|
+
"log",
|
|
299
|
+
"input",
|
|
300
|
+
"histogram",
|
|
301
|
+
"corsika_log",
|
|
302
|
+
"corsika_output",
|
|
303
|
+
"event_data",
|
|
304
|
+
]
|
|
428
305
|
results = {key: [] for key in keys}
|
|
429
306
|
|
|
430
307
|
def get_file_name(name, **kwargs):
|
|
431
308
|
return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
|
|
432
309
|
|
|
433
|
-
|
|
434
|
-
results["input"].append(str(file))
|
|
435
|
-
|
|
436
|
-
results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
|
|
310
|
+
results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=self.run_number))
|
|
437
311
|
|
|
438
312
|
for i in range(len(self.array_models)):
|
|
439
313
|
results["simtel_output"].append(
|
|
440
|
-
get_file_name("simtel_output", run_number=run_number, model_version_index=i)
|
|
314
|
+
get_file_name("simtel_output", run_number=self.run_number, model_version_index=i)
|
|
441
315
|
)
|
|
442
316
|
|
|
443
317
|
if "sim_telarray" in self.simulation_software:
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
results["hist"].append(
|
|
453
|
-
get_file_name(
|
|
454
|
-
"histogram",
|
|
455
|
-
simulation_software="sim_telarray",
|
|
456
|
-
run_number=run_number,
|
|
457
|
-
model_version_index=i,
|
|
458
|
-
)
|
|
459
|
-
)
|
|
460
|
-
results["event_data"].append(
|
|
461
|
-
get_file_name(
|
|
462
|
-
"event_data",
|
|
463
|
-
simulation_software="sim_telarray",
|
|
464
|
-
run_number=run_number,
|
|
465
|
-
model_version_index=i,
|
|
318
|
+
for file_type in ("log", "histogram", "event_data"):
|
|
319
|
+
results[file_type].append(
|
|
320
|
+
get_file_name(
|
|
321
|
+
file_type,
|
|
322
|
+
simulation_software="sim_telarray",
|
|
323
|
+
run_number=self.run_number,
|
|
324
|
+
model_version_index=i,
|
|
325
|
+
)
|
|
466
326
|
)
|
|
467
|
-
)
|
|
468
327
|
|
|
469
328
|
if "corsika" in self.simulation_software:
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
329
|
+
for file_type in ("corsika_output", "corsika_log"):
|
|
330
|
+
results[file_type].append(
|
|
331
|
+
get_file_name(
|
|
332
|
+
file_type,
|
|
333
|
+
simulation_software="corsika",
|
|
334
|
+
run_number=self.run_number,
|
|
335
|
+
model_version_index=i,
|
|
336
|
+
)
|
|
476
337
|
)
|
|
477
|
-
)
|
|
478
338
|
|
|
479
339
|
for key in keys:
|
|
480
340
|
self._results[key].extend(results[key])
|
|
@@ -483,7 +343,6 @@ class Simulator:
|
|
|
483
343
|
"""
|
|
484
344
|
Get list of files generated by simulations.
|
|
485
345
|
|
|
486
|
-
Options are "input", "simtel_output", "hist", "log", "corsika_log".
|
|
487
346
|
Not all file types are available for all simulation types.
|
|
488
347
|
Returns an empty list for an unknown file type.
|
|
489
348
|
|
|
@@ -498,7 +357,6 @@ class Simulator:
|
|
|
498
357
|
List with the full path of all output files.
|
|
499
358
|
|
|
500
359
|
"""
|
|
501
|
-
self.logger.info(f"Getting list of {file_type} files")
|
|
502
360
|
return self._results[file_type]
|
|
503
361
|
|
|
504
362
|
def _make_resources_report(self, input_file_list):
|
|
@@ -512,38 +370,31 @@ class Simulator:
|
|
|
512
370
|
|
|
513
371
|
Returns
|
|
514
372
|
-------
|
|
515
|
-
|
|
516
|
-
|
|
373
|
+
str
|
|
374
|
+
string reporting on computing resources
|
|
517
375
|
|
|
518
376
|
"""
|
|
519
|
-
if len(self._results["sub_out"]) == 0:
|
|
520
|
-
|
|
521
|
-
return {"Wall time/run [sec]": np.nan}
|
|
522
|
-
self._fill_results_without_run(input_file_list)
|
|
377
|
+
if len(self._results["sub_out"]) == 0 and input_file_list is None:
|
|
378
|
+
return "Mean wall time/run [sec]: np.nan"
|
|
523
379
|
|
|
524
380
|
runtime = []
|
|
525
381
|
|
|
526
382
|
_resources = {}
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
runtime.append(_resources["runtime"])
|
|
383
|
+
_resources = self._simulation_runner.get_resources(run_number=self.run_number)
|
|
384
|
+
if _resources.get("runtime"):
|
|
385
|
+
runtime.append(_resources["runtime"])
|
|
531
386
|
|
|
532
387
|
mean_runtime = np.mean(runtime)
|
|
533
388
|
|
|
534
|
-
resource_summary = {}
|
|
535
|
-
resource_summary["Wall time/run [sec]"] = mean_runtime
|
|
389
|
+
resource_summary = f"Mean wall time/run [sec]: {mean_runtime}"
|
|
536
390
|
if "n_events" in _resources and _resources["n_events"] > 0:
|
|
537
|
-
resource_summary
|
|
538
|
-
resource_summary["Wall time/1000 events [sec]"] = (
|
|
539
|
-
mean_runtime * 1000 / _resources["n_events"]
|
|
540
|
-
)
|
|
391
|
+
resource_summary += f", #events/run: {_resources['n_events']}"
|
|
541
392
|
|
|
542
393
|
return resource_summary
|
|
543
394
|
|
|
544
|
-
def
|
|
395
|
+
def report(self, input_file_list=None):
|
|
545
396
|
"""
|
|
546
|
-
|
|
397
|
+
Report on simulations and computing resources used.
|
|
547
398
|
|
|
548
399
|
Includes run time per run only at this point.
|
|
549
400
|
|
|
@@ -553,37 +404,21 @@ class Simulator:
|
|
|
553
404
|
Single file or list of files of shower simulations.
|
|
554
405
|
|
|
555
406
|
"""
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
Attributes
|
|
568
|
-
----------
|
|
569
|
-
run_list: list
|
|
570
|
-
list of runs (integers)
|
|
571
|
-
run_range:list
|
|
572
|
-
min and max of range of runs to be simulated (two list entries)
|
|
573
|
-
|
|
574
|
-
Returns
|
|
575
|
-
-------
|
|
576
|
-
list
|
|
577
|
-
list of unique run numbers (integers)
|
|
578
|
-
|
|
579
|
-
"""
|
|
580
|
-
if run_list is None and run_range is None:
|
|
581
|
-
return [] if self.runs is None else self.runs
|
|
582
|
-
return self._prepare_run_list_and_range(run_list, run_range)
|
|
407
|
+
_corsika_config = self._get_first_corsika_config()
|
|
408
|
+
self.logger.info(
|
|
409
|
+
f"Production run complete for primary {_corsika_config.primary_particle} showers "
|
|
410
|
+
f"from {_corsika_config.azimuth_angle} azimuth and "
|
|
411
|
+
f"{_corsika_config.zenith_angle} zenith "
|
|
412
|
+
f"at {self.site} site, using {self.model_version} model."
|
|
413
|
+
)
|
|
414
|
+
self.logger.info(
|
|
415
|
+
f"Computing for {self.simulation_software} Simulations: "
|
|
416
|
+
f"{self._make_resources_report(input_file_list)}"
|
|
417
|
+
)
|
|
583
418
|
|
|
584
419
|
def save_file_lists(self):
|
|
585
420
|
"""Save files lists for output and log files."""
|
|
586
|
-
for file_type in ["simtel_output", "log", "corsika_log", "
|
|
421
|
+
for file_type in ["simtel_output", "log", "corsika_log", "histogram"]:
|
|
587
422
|
file_name = self.io_handler.get_output_directory().joinpath(f"{file_type}_files.txt")
|
|
588
423
|
file_list = self.get_file_list(file_type=file_type)
|
|
589
424
|
if all(element is not None for element in file_list) and len(file_list) > 0:
|
|
@@ -635,7 +470,7 @@ class Simulator:
|
|
|
635
470
|
output_files = self.get_file_list(file_type="simtel_output")
|
|
636
471
|
log_files = self.get_file_list(file_type="log")
|
|
637
472
|
corsika_log_files = self.get_file_list(file_type="corsika_log")
|
|
638
|
-
histogram_files = self.get_file_list(file_type="
|
|
473
|
+
histogram_files = self.get_file_list(file_type="histogram")
|
|
639
474
|
reduced_event_files = (
|
|
640
475
|
self.get_file_list(file_type="event_data")
|
|
641
476
|
if self.args_dict.get("save_reduced_event_lists")
|
|
@@ -657,7 +492,7 @@ class Simulator:
|
|
|
657
492
|
# Group files by model version
|
|
658
493
|
for model in self.array_models:
|
|
659
494
|
model_version = model.model_version
|
|
660
|
-
model_files =
|
|
495
|
+
model_files = general.ensure_iterable(model.pack_model_files())
|
|
661
496
|
|
|
662
497
|
# Filter files for this model version
|
|
663
498
|
model_logs = [f for f in log_files if model_version in f]
|
|
@@ -669,7 +504,7 @@ class Simulator:
|
|
|
669
504
|
tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
|
|
670
505
|
# Add all relevant model, log, histogram, and CORSIKA log files to the tarball
|
|
671
506
|
files_to_tar = model_logs + model_hists + model_corsika_logs + model_files
|
|
672
|
-
|
|
507
|
+
general.pack_tar_file(tar_file_path, files_to_tar)
|
|
673
508
|
|
|
674
509
|
for file_to_move in output_files + reduced_event_files:
|
|
675
510
|
source_file = Path(file_to_move)
|
|
@@ -763,8 +598,8 @@ class Simulator:
|
|
|
763
598
|
"""
|
|
764
599
|
return run_mode in [
|
|
765
600
|
"pedestals",
|
|
766
|
-
"
|
|
767
|
-
"
|
|
601
|
+
"pedestals_dark",
|
|
602
|
+
"pedestals_nsb_only",
|
|
768
603
|
"direct_injection",
|
|
769
604
|
]
|
|
770
605
|
|
|
@@ -786,3 +621,182 @@ class Simulator:
|
|
|
786
621
|
if run_mode == "direct_injection":
|
|
787
622
|
return ["flat_fielding"]
|
|
788
623
|
return []
|
|
624
|
+
|
|
625
|
+
def _get_first_corsika_config(self):
|
|
626
|
+
"""
|
|
627
|
+
Return first instance from list of CORSIKA configurations.
|
|
628
|
+
|
|
629
|
+
Most values stored in the CORSIKA configurations are identical,
|
|
630
|
+
with the exception of the simulation model version dependent parameters.
|
|
631
|
+
|
|
632
|
+
Returns
|
|
633
|
+
-------
|
|
634
|
+
CorsikaConfig
|
|
635
|
+
First CORSIKA configuration instance.
|
|
636
|
+
"""
|
|
637
|
+
try:
|
|
638
|
+
return (
|
|
639
|
+
self.corsika_configurations[0]
|
|
640
|
+
if isinstance(self.corsika_configurations, list)
|
|
641
|
+
else self.corsika_configurations
|
|
642
|
+
)
|
|
643
|
+
except (IndexError, TypeError) as exc:
|
|
644
|
+
raise ValueError("CORSIKA configuration not found for verification.") from exc
|
|
645
|
+
|
|
646
|
+
def verify_simulations(self):
|
|
647
|
+
"""
|
|
648
|
+
Verify simulations.
|
|
649
|
+
|
|
650
|
+
This includes checking the number of simulated events.
|
|
651
|
+
|
|
652
|
+
"""
|
|
653
|
+
self.logger.info("Verifying simulations.")
|
|
654
|
+
|
|
655
|
+
_corsika_config = self._get_first_corsika_config()
|
|
656
|
+
expected_shower_events = _corsika_config.shower_events
|
|
657
|
+
expected_mc_events = _corsika_config.mc_events
|
|
658
|
+
|
|
659
|
+
self.logger.info(
|
|
660
|
+
f"Expected number of shower events: {expected_shower_events}, "
|
|
661
|
+
f"expected number of MC events: {expected_mc_events}"
|
|
662
|
+
)
|
|
663
|
+
if self.simulation_software in ["corsika_sim_telarray", "sim_telarray"]:
|
|
664
|
+
self._verify_simulated_events_in_sim_telarray(
|
|
665
|
+
expected_shower_events, expected_mc_events
|
|
666
|
+
)
|
|
667
|
+
if self.simulation_software == "corsika":
|
|
668
|
+
self._verify_simulated_events_corsika(expected_mc_events)
|
|
669
|
+
if self.args_dict.get("save_reduced_event_lists"):
|
|
670
|
+
self._verify_simulated_events_in_reduced_event_lists(expected_mc_events)
|
|
671
|
+
|
|
672
|
+
def _verify_simulated_events_corsika(self, expected_mc_events, tolerance=1.0e-3):
|
|
673
|
+
"""
|
|
674
|
+
Verify the number of simulated events in CORSIKA output files.
|
|
675
|
+
|
|
676
|
+
Allow for a small mismatch in the number of requested events.
|
|
677
|
+
|
|
678
|
+
Parameters
|
|
679
|
+
----------
|
|
680
|
+
expected_mc_events: int
|
|
681
|
+
Expected number of simulated MC events.
|
|
682
|
+
|
|
683
|
+
Raises
|
|
684
|
+
------
|
|
685
|
+
ValueError
|
|
686
|
+
If the number of simulated events does not match the expected number.
|
|
687
|
+
"""
|
|
688
|
+
|
|
689
|
+
def consistent(a, b, tol):
|
|
690
|
+
return abs(a - b) / max(a, b) <= tol
|
|
691
|
+
|
|
692
|
+
event_errors = []
|
|
693
|
+
for file in self.get_file_list(file_type="corsika_output"):
|
|
694
|
+
shower_events, _ = eventio_handler.get_simulated_events(file)
|
|
695
|
+
|
|
696
|
+
if shower_events != expected_mc_events:
|
|
697
|
+
if consistent(shower_events, expected_mc_events, tol=tolerance):
|
|
698
|
+
self.logger.warning(
|
|
699
|
+
f"Small mismatch in number of events in: {file}: "
|
|
700
|
+
f"shower events: {shower_events} (expected: {expected_mc_events})"
|
|
701
|
+
)
|
|
702
|
+
continue
|
|
703
|
+
event_errors.append(
|
|
704
|
+
f"Number of simulated MC events ({shower_events}) does not match "
|
|
705
|
+
f"the expected number ({expected_mc_events}) in CORSIKA {file}."
|
|
706
|
+
)
|
|
707
|
+
else:
|
|
708
|
+
self.logger.info(
|
|
709
|
+
f"Consistent number of events in: {file}: shower events: {shower_events}"
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
if event_errors:
|
|
713
|
+
self.logger.error("Inconsistent event counts found in CORSIKA output:")
|
|
714
|
+
for error in event_errors:
|
|
715
|
+
self.logger.error(f" - {error}")
|
|
716
|
+
error_message = "Inconsistent event counts found in CORSIKA output:\n" + "\n".join(
|
|
717
|
+
f" - {error}" for error in event_errors
|
|
718
|
+
)
|
|
719
|
+
raise ValueError(error_message)
|
|
720
|
+
|
|
721
|
+
def _verify_simulated_events_in_sim_telarray(self, expected_shower_events, expected_mc_events):
|
|
722
|
+
"""
|
|
723
|
+
Verify the number of simulated events.
|
|
724
|
+
|
|
725
|
+
Parameters
|
|
726
|
+
----------
|
|
727
|
+
expected_shower_events: int
|
|
728
|
+
Expected number of simulated shower events.
|
|
729
|
+
expected_mc_events: int
|
|
730
|
+
Expected number of simulated MC events.
|
|
731
|
+
|
|
732
|
+
Raises
|
|
733
|
+
------
|
|
734
|
+
ValueError
|
|
735
|
+
If the number of simulated events does not match the expected number.
|
|
736
|
+
"""
|
|
737
|
+
event_errors = []
|
|
738
|
+
for file in self.get_file_list(file_type="simtel_output"):
|
|
739
|
+
shower_events, mc_events = eventio_handler.get_simulated_events(file)
|
|
740
|
+
|
|
741
|
+
if (shower_events, mc_events) != (expected_shower_events, expected_mc_events):
|
|
742
|
+
event_errors.append(
|
|
743
|
+
f"Event mismatch: shower/MC events in {file}: {shower_events}/{mc_events}"
|
|
744
|
+
f" (expected: {expected_shower_events}/{expected_mc_events})"
|
|
745
|
+
)
|
|
746
|
+
else:
|
|
747
|
+
self.logger.info(
|
|
748
|
+
f"Consistent number of events in: {file}: "
|
|
749
|
+
f"shower events: {shower_events}, "
|
|
750
|
+
f"MC events: {mc_events}"
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
if event_errors:
|
|
754
|
+
self.logger.error("Inconsistent event counts found:")
|
|
755
|
+
for error in event_errors:
|
|
756
|
+
self.logger.error(f" - {error}")
|
|
757
|
+
error_message = "Inconsistent event counts found:\n" + "\n".join(
|
|
758
|
+
f" - {error}" for error in event_errors
|
|
759
|
+
)
|
|
760
|
+
raise ValueError(error_message)
|
|
761
|
+
|
|
762
|
+
def _verify_simulated_events_in_reduced_event_lists(self, expected_mc_events):
|
|
763
|
+
"""
|
|
764
|
+
Verify the number of simulated events in reduced event lists.
|
|
765
|
+
|
|
766
|
+
Parameters
|
|
767
|
+
----------
|
|
768
|
+
expected_mc_events: int
|
|
769
|
+
Expected number of simulated MC events.
|
|
770
|
+
|
|
771
|
+
Raises
|
|
772
|
+
------
|
|
773
|
+
ValueError
|
|
774
|
+
If the number of simulated events does not match the expected number.
|
|
775
|
+
"""
|
|
776
|
+
event_errors = []
|
|
777
|
+
for file in self.get_file_list(file_type="event_data"):
|
|
778
|
+
tables = table_handler.read_tables(file, ["SHOWERS"])
|
|
779
|
+
try:
|
|
780
|
+
mc_events = len(tables["SHOWERS"])
|
|
781
|
+
except KeyError as exc:
|
|
782
|
+
raise ValueError(f"SHOWERS table not found in reduced event list {file}.") from exc
|
|
783
|
+
|
|
784
|
+
if mc_events != expected_mc_events:
|
|
785
|
+
event_errors.append(
|
|
786
|
+
f"Number of simulated MC events ({mc_events}) does not match "
|
|
787
|
+
f"the expected number ({expected_mc_events}) in reduced event list {file}."
|
|
788
|
+
)
|
|
789
|
+
else:
|
|
790
|
+
self.logger.info(
|
|
791
|
+
f"Consistent number of events in reduced event list: {file}: MC events:"
|
|
792
|
+
f" {mc_events}"
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
if event_errors:
|
|
796
|
+
self.logger.error("Inconsistent event counts found in reduced event lists:")
|
|
797
|
+
for error in event_errors:
|
|
798
|
+
self.logger.error(f" - {error}")
|
|
799
|
+
error_message = "Inconsistent event counts found in reduced event lists:\n" + "\n".join(
|
|
800
|
+
f" - {error}" for error in event_errors
|
|
801
|
+
)
|
|
802
|
+
raise ValueError(error_message)
|