gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {gammasimtools-0.5.1.dist-info → gammasimtools-0.6.1.dist-info}/METADATA +80 -28
  2. gammasimtools-0.6.1.dist-info/RECORD +91 -0
  3. {gammasimtools-0.5.1.dist-info → gammasimtools-0.6.1.dist-info}/WHEEL +1 -1
  4. {gammasimtools-0.5.1.dist-info → gammasimtools-0.6.1.dist-info}/entry_points.txt +4 -2
  5. simtools/_version.py +14 -2
  6. simtools/applications/add_file_to_db.py +2 -1
  7. simtools/applications/compare_cumulative_psf.py +10 -15
  8. simtools/applications/db_development_tools/add_new_parameter_to_db.py +12 -6
  9. simtools/applications/derive_mirror_rnda.py +95 -71
  10. simtools/applications/generate_corsika_histograms.py +216 -131
  11. simtools/applications/generate_default_metadata.py +110 -0
  12. simtools/applications/generate_simtel_array_histograms.py +192 -0
  13. simtools/applications/get_file_from_db.py +1 -1
  14. simtools/applications/get_parameter.py +3 -3
  15. simtools/applications/make_regular_arrays.py +89 -93
  16. simtools/applications/{plot_layout_array.py → plot_array_layout.py} +15 -14
  17. simtools/applications/print_array_elements.py +81 -34
  18. simtools/applications/produce_array_config.py +2 -2
  19. simtools/applications/production.py +39 -5
  20. simtools/applications/sim_showers_for_trigger_rates.py +26 -30
  21. simtools/applications/simulate_prod.py +49 -107
  22. simtools/applications/submit_data_from_external.py +8 -10
  23. simtools/applications/tune_psf.py +16 -18
  24. simtools/applications/validate_camera_efficiency.py +63 -9
  25. simtools/applications/validate_camera_fov.py +9 -13
  26. simtools/applications/validate_file_using_schema.py +127 -0
  27. simtools/applications/validate_optics.py +13 -15
  28. simtools/camera_efficiency.py +73 -80
  29. simtools/configuration/commandline_parser.py +52 -22
  30. simtools/configuration/configurator.py +98 -33
  31. simtools/constants.py +9 -0
  32. simtools/corsika/corsika_config.py +28 -22
  33. simtools/corsika/corsika_default_config.py +282 -0
  34. simtools/corsika/corsika_histograms.py +328 -282
  35. simtools/corsika/corsika_histograms_visualize.py +162 -163
  36. simtools/corsika/corsika_runner.py +8 -4
  37. simtools/corsika_simtel/corsika_simtel_runner.py +18 -23
  38. simtools/data_model/data_reader.py +129 -0
  39. simtools/data_model/metadata_collector.py +346 -118
  40. simtools/data_model/metadata_model.py +123 -218
  41. simtools/data_model/model_data_writer.py +79 -22
  42. simtools/data_model/validate_data.py +96 -46
  43. simtools/db_handler.py +67 -42
  44. simtools/io_operations/__init__.py +0 -0
  45. simtools/io_operations/hdf5_handler.py +112 -0
  46. simtools/{io_handler.py → io_operations/io_handler.py} +51 -22
  47. simtools/job_execution/job_manager.py +1 -1
  48. simtools/layout/{layout_array.py → array_layout.py} +168 -199
  49. simtools/layout/geo_coordinates.py +196 -0
  50. simtools/layout/telescope_position.py +12 -12
  51. simtools/model/array_model.py +16 -14
  52. simtools/model/camera.py +5 -8
  53. simtools/model/mirrors.py +136 -73
  54. simtools/model/model_utils.py +1 -69
  55. simtools/model/telescope_model.py +32 -25
  56. simtools/psf_analysis.py +26 -19
  57. simtools/ray_tracing.py +54 -26
  58. simtools/schemas/data.metaschema.yml +400 -0
  59. simtools/schemas/metadata.metaschema.yml +566 -0
  60. simtools/simtel/simtel_config_writer.py +14 -5
  61. simtools/simtel/simtel_histograms.py +266 -83
  62. simtools/simtel/simtel_runner.py +8 -7
  63. simtools/simtel/simtel_runner_array.py +7 -8
  64. simtools/simtel/simtel_runner_camera_efficiency.py +48 -2
  65. simtools/simtel/simtel_runner_ray_tracing.py +61 -25
  66. simtools/simulator.py +43 -50
  67. simtools/utils/general.py +232 -286
  68. simtools/utils/geometry.py +163 -0
  69. simtools/utils/names.py +294 -142
  70. simtools/visualization/legend_handlers.py +115 -9
  71. simtools/visualization/visualize.py +13 -13
  72. gammasimtools-0.5.1.dist-info/RECORD +0 -83
  73. simtools/applications/plot_simtel_histograms.py +0 -120
  74. simtools/applications/validate_schema_files.py +0 -135
  75. simtools/corsika/corsika_output_visualize.py +0 -345
  76. simtools/data_model/validate_schema.py +0 -285
  77. {gammasimtools-0.5.1.dist-info → gammasimtools-0.6.1.dist-info}/LICENSE +0 -0
  78. {gammasimtools-0.5.1.dist-info → gammasimtools-0.6.1.dist-info}/top_level.txt +0 -0
@@ -3,7 +3,7 @@ import logging
3
3
  import astropy.units as u
4
4
 
5
5
  import simtools.utils.general as gen
6
- from simtools import io_handler
6
+ from simtools.io_operations import io_handler
7
7
  from simtools.simtel.simtel_runner import SimtelRunner
8
8
  from simtools.utils import names
9
9
 
@@ -31,10 +31,13 @@ class SimtelRunnerRayTracing(SimtelRunner):
31
31
  len: 1
32
32
  unit: km
33
33
  default: 10 km
34
+ single_mirror_mode:
35
+ len: 1
36
+ default: False
34
37
  use_random_focal_length:
35
38
  len: 1
36
39
  default: False
37
- mirror_number:
40
+ mirror_numbers:
38
41
  len: 1
39
42
  default: 1
40
43
 
@@ -50,8 +53,6 @@ class SimtelRunnerRayTracing(SimtelRunner):
50
53
  Dict containing the configurable parameters.
51
54
  config_file: str or Path
52
55
  Path of the yaml file containing the configurable parameters.
53
- single_mirror_mode: bool
54
- True for single mirror simulations.
55
56
  force_simulate: bool
56
57
  Remove existing files and force re-running of the ray-tracing simulation.
57
58
  """
@@ -63,8 +64,8 @@ class SimtelRunnerRayTracing(SimtelRunner):
63
64
  simtel_source_path=None,
64
65
  config_data=None,
65
66
  config_file=None,
66
- single_mirror_mode=False,
67
67
  force_simulate=False,
68
+ test=False,
68
69
  ):
69
70
  """
70
71
  Initialize SimtelRunner.
@@ -80,20 +81,16 @@ class SimtelRunnerRayTracing(SimtelRunner):
80
81
  self.io_handler = io_handler.IOHandler()
81
82
  self._base_directory = self.io_handler.get_output_directory(self.label, "ray-tracing")
82
83
 
83
- self._single_mirror_mode = single_mirror_mode
84
+ # Loading config_data
85
+ self.config = gen.validate_config_data(
86
+ gen.collect_data_from_file_or_dict(config_file, config_data),
87
+ self.ray_tracing_default_configuration(True),
88
+ )
84
89
 
85
90
  # RayTracing - default parameters
86
91
  self._rep_number = 0
87
- self.RUNS_PER_SET = 1 if self._single_mirror_mode else 20
88
- self.PHOTONS_PER_RUN = 100000
89
-
90
- # Loading config_data
91
- _config_data_in = gen.collect_data_from_yaml_or_dict(config_file, config_data)
92
- _parameter_file = self.io_handler.get_input_data_file(
93
- "parameters", "simtel-runner-ray-tracing_parameters.yml"
94
- )
95
- _parameters = gen.collect_data_from_yaml_or_dict(_parameter_file, None)
96
- self.config = gen.validate_config_data(_config_data_in, _parameters)
92
+ self.runs_per_set = 1 if self.config.single_mirror_mode else 20
93
+ self.photons_per_run = 100000 if not test else 10000
97
94
 
98
95
  self._load_required_files(force_simulate)
99
96
 
@@ -117,7 +114,7 @@ class SimtelRunnerRayTracing(SimtelRunner):
117
114
  self.config.source_distance,
118
115
  self.config.zenith_angle,
119
116
  self.config.off_axis_angle,
120
- self.config.mirror_number if self._single_mirror_mode else None,
117
+ self.config.mirror_numbers if self.config.single_mirror_mode else None,
121
118
  self.label,
122
119
  base_name,
123
120
  )
@@ -137,8 +134,8 @@ class SimtelRunnerRayTracing(SimtelRunner):
137
134
  file.write(f"# zenith_angle [deg] = {self.config.zenith_angle}\n")
138
135
  file.write(f"# off_axis_angle [deg] = {self.config.off_axis_angle}\n")
139
136
  file.write(f"# source_distance [km] = {self.config.source_distance}\n")
140
- if self._single_mirror_mode:
141
- file.write(f"# mirror_number = {self.config.mirror_number}\n\n")
137
+ if self.config.single_mirror_mode:
138
+ file.write(f"# mirror_number = {self.config.mirror_numbers}\n\n")
142
139
 
143
140
  # Filling in star file with a single light source.
144
141
  # Parameters defining light source:
@@ -158,7 +155,7 @@ class SimtelRunnerRayTracing(SimtelRunner):
158
155
  def _make_run_command(self, **kwargs): # pylint: disable=unused-argument
159
156
  """Return the command to run simtel_array."""
160
157
 
161
- if self._single_mirror_mode:
158
+ if self.config.single_mirror_mode:
162
159
  _mirror_focal_length = float(
163
160
  self.telescope_model.get_parameter_value("mirror_focal_length")
164
161
  )
@@ -168,6 +165,7 @@ class SimtelRunnerRayTracing(SimtelRunner):
168
165
  command += f" -c {self.telescope_model.get_config_file()}"
169
166
  command += " -I../cfg/CTA"
170
167
  command += f" -I{self.telescope_model.get_config_directory()}"
168
+ command += super()._config_option("random_state", "none")
171
169
  command += super()._config_option("IMAGING_LIST", str(self._photons_file))
172
170
  command += super()._config_option("stars", str(self._stars_file))
173
171
  command += super()._config_option(
@@ -176,7 +174,7 @@ class SimtelRunnerRayTracing(SimtelRunner):
176
174
  command += super()._config_option(
177
175
  "telescope_theta", self.config.zenith_angle + self.config.off_axis_angle
178
176
  )
179
- command += super()._config_option("star_photons", str(self.PHOTONS_PER_RUN))
177
+ command += super()._config_option("star_photons", str(self.photons_per_run))
180
178
  command += super()._config_option("telescope_phi", "0")
181
179
  command += super()._config_option("camera_transmission", "1.0")
182
180
  command += super()._config_option("nightsky_background", "all:0.")
@@ -187,7 +185,7 @@ class SimtelRunnerRayTracing(SimtelRunner):
187
185
  command += super()._config_option("maximum_telescopes", "1")
188
186
  command += super()._config_option("show", "all")
189
187
  command += super()._config_option("camera_filter", "none")
190
- if self._single_mirror_mode:
188
+ if self.config.single_mirror_mode:
191
189
  command += super()._config_option("focus_offset", "all:0.")
192
190
  command += super()._config_option("camera_config_file", "single_pixel_camera.dat")
193
191
  command += super()._config_option("camera_pixels", "1")
@@ -196,7 +194,7 @@ class SimtelRunnerRayTracing(SimtelRunner):
196
194
  command += super()._config_option(
197
195
  "mirror_list",
198
196
  self.telescope_model.get_single_mirror_list_file(
199
- self.config.mirror_number, self.config.use_random_focal_length
197
+ self.config.mirror_numbers, self.config.use_random_focal_length
200
198
  ),
201
199
  )
202
200
  command += super()._config_option(
@@ -205,7 +203,6 @@ class SimtelRunnerRayTracing(SimtelRunner):
205
203
  command += super()._config_option("dish_shape_length", _mirror_focal_length)
206
204
  command += super()._config_option("mirror_focal_length", _mirror_focal_length)
207
205
  command += super()._config_option("parabolic_dish", "0")
208
- # command += super()._config_option('random_focal_length', '0.')
209
206
  command += super()._config_option("mirror_align_random_distance", "0.")
210
207
  command += super()._config_option("mirror_align_random_vertical", "0.,28.,0.,0.")
211
208
  command += " " + str(self._corsika_file)
@@ -232,10 +229,49 @@ class SimtelRunnerRayTracing(SimtelRunner):
232
229
  def _is_photon_list_file_ok(self):
233
230
  """Check if the photon list is valid,"""
234
231
  n_lines = 0
235
- with open(self._photons_file, "r", encoding="utf-8") as ff:
232
+ with open(self._photons_file, "rb") as ff:
236
233
  for _ in ff:
237
234
  n_lines += 1
238
235
  if n_lines > 100:
239
236
  break
240
237
 
241
238
  return n_lines > 100
239
+
240
+ @staticmethod
241
+ def ray_tracing_default_configuration(config_runner=False):
242
+ """
243
+ Get default ray tracing configuration.
244
+
245
+ Returns
246
+ -------
247
+ dict
248
+ Default configuration for ray tracing.
249
+
250
+ """
251
+
252
+ return {
253
+ "zenith_angle": {
254
+ "len": 1,
255
+ "unit": u.Unit("deg"),
256
+ "default": 20.0 * u.deg,
257
+ "names": ["zenith", "theta"],
258
+ },
259
+ "off_axis_angle": {
260
+ "len": 1 if config_runner else None,
261
+ "unit": u.Unit("deg"),
262
+ "default": 0.0 * u.deg,
263
+ "names": ["offaxis", "offset"],
264
+ },
265
+ "source_distance": {
266
+ "len": 1,
267
+ "unit": u.Unit("km"),
268
+ "default": 10.0 * u.km,
269
+ "names": ["sourcedist", "srcdist"],
270
+ },
271
+ "single_mirror_mode": {"len": 1, "default": False},
272
+ "use_random_focal_length": {"len": 1, "default": False},
273
+ "mirror_numbers": {
274
+ "len": 1 if config_runner else None,
275
+ "default": 1 if config_runner else "all",
276
+ },
277
+ }
simtools/simulator.py CHANGED
@@ -8,12 +8,11 @@ import astropy.units as u
8
8
  import numpy as np
9
9
 
10
10
  import simtools.utils.general as gen
11
- from simtools import io_handler
12
11
  from simtools.corsika.corsika_runner import CorsikaRunner
13
12
  from simtools.corsika_simtel.corsika_simtel_runner import CorsikaSimtelRunner
13
+ from simtools.io_operations import io_handler
14
14
  from simtools.job_execution.job_manager import JobManager
15
15
  from simtools.model.array_model import ArrayModel
16
- from simtools.simtel.simtel_histograms import SimtelHistograms
17
16
  from simtools.simtel.simtel_runner_array import SimtelRunnerArray
18
17
  from simtools.utils import names
19
18
 
@@ -79,7 +78,7 @@ class Simulator:
79
78
  simulator: choices: [simtel, corsika]
80
79
  implemented are sim_telarray and CORSIKA
81
80
  simulator_source_path: str or Path
82
- Location of exectutables for simulation software \
81
+ Location of executables for simulation software \
83
82
  (e.g. path with CORSIKA or sim_telarray)
84
83
  label: str
85
84
  Instance label.
@@ -178,19 +177,22 @@ class Simulator:
178
177
  Path to yaml file containing configurable data.
179
178
 
180
179
  """
181
- config_data = gen.collect_data_from_yaml_or_dict(config_file, config_data)
180
+ simulator_config_data = gen.collect_data_from_file_or_dict(config_file, config_data)
182
181
  if self.simulator == "corsika":
183
- self._load_corsika_config_and_model(config_data)
182
+ self._load_corsika_config_and_model(simulator_config_data)
184
183
  if self.simulator == "simtel":
185
- self._load_sim_tel_config_and_model(config_data)
184
+ self._load_sim_tel_config_and_model(simulator_config_data)
186
185
  if self.simulator == "corsika_simtel":
187
- self._load_corsika_config_and_model(config_data)
188
- self._load_sim_tel_config_and_model(config_data)
186
+ config_showers, config_arrays = self._separate_corsika_and_simtel_config_data(
187
+ simulator_config_data
188
+ )
189
+ self._load_corsika_config_and_model(config_showers)
190
+ self._load_sim_tel_config_and_model(config_arrays)
189
191
 
190
192
  def _load_corsika_config_and_model(self, config_data):
191
193
  """
192
194
  Validate configuration data for CORSIKA shower simulation and
193
- remove entries need needed for CorsikaRunner.
195
+ remove entries needed for CorsikaRunner.
194
196
 
195
197
  Parameters
196
198
  ----------
@@ -203,7 +205,7 @@ class Simulator:
203
205
 
204
206
  try:
205
207
  self.site = names.validate_site_name(self._corsika_config_data.pop("site"))
206
- self.layout_name = names.validate_layout_array_name(
208
+ self.layout_name = names.validate_array_layout_name(
207
209
  self._corsika_config_data.pop("layout_name")
208
210
  )
209
211
  except KeyError:
@@ -219,12 +221,6 @@ class Simulator:
219
221
  "corsika_parameters_file", None
220
222
  )
221
223
 
222
- # Remove sim_telarray parameters from the CORSIKA config dictionary
223
- # TODO - Replace this with a more elegant solution!
224
- tel_keys = [k for k in self._corsika_config_data.keys() if k[1:4] in ["ST-", "CT-"]]
225
- for key in ["model_version", "default"] + tel_keys:
226
- self._corsika_config_data.pop(key, None)
227
-
228
224
  def _load_sim_tel_config_and_model(self, config_data):
229
225
  """
230
226
  Load array model and configuration parameters for array simulations
@@ -240,7 +236,7 @@ class Simulator:
240
236
  _parameter_file = self.io_handler.get_input_data_file(
241
237
  "parameters", "array-simulator_parameters.yml"
242
238
  )
243
- _parameters = gen.collect_data_from_yaml_or_dict(_parameter_file, None)
239
+ _parameters = gen.collect_data_from_file_or_dict(_parameter_file, None)
244
240
  self.config = gen.validate_config_data(_rest_config, _parameters, ignore_unidentified=True)
245
241
 
246
242
  self.array_model = ArrayModel(
@@ -249,6 +245,30 @@ class Simulator:
249
245
  mongo_db_config=self._mongo_db_config,
250
246
  )
251
247
 
248
+ def _separate_corsika_and_simtel_config_data(self, config_data):
249
+ """
250
+ Separate the CORSIKA and sim_telarray simulation configuration to two dictionaries.
251
+
252
+ Parameters
253
+ ----------
254
+ config_data: dict
255
+ Dictionary with both the CORSIKA and sim_telarray simulation configuration data.
256
+
257
+ Returns
258
+ -------
259
+ dict
260
+ Configuration of shower simulations.
261
+ dict
262
+ Configuration of array simulations.
263
+
264
+ """
265
+
266
+ common = copy(config_data.pop("common", {}))
267
+ config_showers = copy(config_data.pop("showers", {})) | common
268
+ config_arrays = copy(config_data.pop("array", {})) | common
269
+
270
+ return config_showers, config_arrays
271
+
252
272
  def _validate_run_list_and_range(self, run_list, run_range):
253
273
  """
254
274
  Prepares list of run numbers from a list or from a range.
@@ -311,7 +331,7 @@ class Simulator:
311
331
 
312
332
  try:
313
333
  _array_model_data["site"] = names.validate_site_name(_rest_data.pop("site"))
314
- _array_model_data["layout_name"] = names.validate_layout_array_name(
334
+ _array_model_data["layout_name"] = names.validate_array_layout_name(
315
335
  _rest_data.pop("layout_name")
316
336
  )
317
337
  _array_model_data["model_version"] = _rest_data.pop("model_version")
@@ -343,7 +363,6 @@ class Simulator:
343
363
  "corsika_parameters_file": self._corsika_parameters_file,
344
364
  "corsika_config_data": self._corsika_config_data,
345
365
  }
346
- # TODO: This is not very elegant, find a nicer solution?
347
366
  if self.simulator in ["simtel", "corsika_simtel"]:
348
367
  simtel_args = {
349
368
  "array_model": self.array_model,
@@ -397,7 +416,8 @@ class Simulator:
397
416
  for file in input_file_list:
398
417
  run = self._guess_run_from_file(file)
399
418
  self._fill_results(file, run)
400
- self.runs.append(run)
419
+ if run not in self.runs:
420
+ self.runs.append(run)
401
421
 
402
422
  def simulate(self, input_file_list=None):
403
423
  """
@@ -572,34 +592,6 @@ class Simulator:
572
592
  self._results["hist"].append(None)
573
593
  self._results["log"].append(None)
574
594
 
575
- def print_histograms(self, input_file_list=None):
576
- """
577
- Print histograms and save a pdf file.
578
-
579
- Parameters
580
- ----------
581
- input_file_list: str or list of str
582
- Single file or list of files of shower simulations.
583
-
584
- Returns
585
- -------
586
- path
587
- Path of the pdf file.
588
- """
589
-
590
- fig_name = None
591
-
592
- if self.simulator in ["simtel", "corsika_simtel"]:
593
- if len(self._results["hist"]) == 0 and input_file_list is not None:
594
- self._fill_results_without_run(input_file_list)
595
-
596
- fig_name = self._output_directory.joinpath("histograms.pdf")
597
- hist_file_list = self.get_list_of_histogram_files()
598
- simtel_histograms = SimtelHistograms(hist_file_list)
599
- simtel_histograms.plot_and_save_figures(fig_name)
600
-
601
- return fig_name
602
-
603
595
  def get_list_of_output_files(self, run_list=None, run_range=None):
604
596
  """
605
597
  Get list of output files.
@@ -627,7 +619,6 @@ class Simulator:
627
619
  file_type="output", **self._simulation_runner.get_info_for_file_name(run)
628
620
  )
629
621
  self._results["output"].append(str(output_file_name))
630
-
631
622
  return self._results["output"]
632
623
 
633
624
  def get_list_of_histogram_files(self):
@@ -709,7 +700,9 @@ class Simulator:
709
700
 
710
701
  """
711
702
 
712
- if len(self._results["sub_out"]) == 0 and input_file_list is not None:
703
+ if len(self._results["sub_out"]) == 0:
704
+ if input_file_list is None:
705
+ return {"Walltime/run [sec]": np.nan}
713
706
  self._fill_results_without_run(input_file_list)
714
707
 
715
708
  runtime = []