gammasimtools 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. {gammasimtools-0.25.0.dist-info → gammasimtools-0.27.0.dist-info}/METADATA +6 -1
  2. {gammasimtools-0.25.0.dist-info → gammasimtools-0.27.0.dist-info}/RECORD +135 -130
  3. {gammasimtools-0.25.0.dist-info → gammasimtools-0.27.0.dist-info}/WHEEL +1 -1
  4. {gammasimtools-0.25.0.dist-info → gammasimtools-0.27.0.dist-info}/entry_points.txt +3 -2
  5. {gammasimtools-0.25.0.dist-info → gammasimtools-0.27.0.dist-info}/licenses/LICENSE +1 -1
  6. simtools/_version.py +2 -2
  7. simtools/application_control.py +35 -7
  8. simtools/applications/convert_geo_coordinates_of_array_elements.py +3 -3
  9. simtools/applications/db_add_file_to_db.py +1 -1
  10. simtools/applications/db_add_simulation_model_from_repository_to_db.py +1 -1
  11. simtools/applications/db_add_value_from_json_to_db.py +1 -1
  12. simtools/applications/db_generate_compound_indexes.py +1 -1
  13. simtools/applications/db_get_array_layouts_from_db.py +3 -7
  14. simtools/applications/db_get_file_from_db.py +1 -1
  15. simtools/applications/db_get_parameter_from_db.py +1 -1
  16. simtools/applications/db_inspect_databases.py +1 -1
  17. simtools/applications/db_upload_model_repository.py +1 -1
  18. simtools/applications/derive_ctao_array_layouts.py +1 -2
  19. simtools/applications/{calculate_incident_angles.py → derive_incident_angle.py} +16 -18
  20. simtools/applications/derive_mirror_rnda.py +112 -180
  21. simtools/applications/derive_psf_parameters.py +0 -1
  22. simtools/applications/derive_pulse_shape_parameters.py +0 -1
  23. simtools/applications/derive_trigger_rates.py +1 -1
  24. simtools/applications/docs_produce_array_element_report.py +2 -8
  25. simtools/applications/docs_produce_calibration_reports.py +1 -3
  26. simtools/applications/docs_produce_model_parameter_reports.py +0 -2
  27. simtools/applications/docs_produce_simulation_configuration_report.py +1 -3
  28. simtools/applications/generate_array_config.py +0 -1
  29. simtools/applications/generate_corsika_histograms.py +79 -229
  30. simtools/applications/generate_regular_arrays.py +76 -69
  31. simtools/applications/generate_simtel_event_data.py +2 -2
  32. simtools/applications/maintain_simulation_model_add_production.py +2 -2
  33. simtools/applications/maintain_simulation_model_write_array_element_positions.py +87 -0
  34. simtools/applications/plot_array_layout.py +5 -111
  35. simtools/applications/plot_simulated_event_distributions.py +57 -0
  36. simtools/applications/plot_tabular_data.py +0 -1
  37. simtools/applications/plot_tabular_data_for_model_parameter.py +1 -6
  38. simtools/applications/production_derive_corsika_limits.py +1 -1
  39. simtools/applications/production_generate_grid.py +0 -1
  40. simtools/applications/run_application.py +1 -1
  41. simtools/applications/simulate_flasher.py +3 -15
  42. simtools/applications/simulate_illuminator.py +2 -11
  43. simtools/applications/simulate_pedestals.py +1 -5
  44. simtools/applications/simulate_prod.py +8 -11
  45. simtools/applications/simulate_prod_htcondor_generator.py +1 -1
  46. simtools/applications/submit_array_layouts.py +2 -4
  47. simtools/applications/submit_data_from_external.py +2 -1
  48. simtools/applications/submit_model_parameter_from_external.py +1 -3
  49. simtools/applications/validate_camera_efficiency.py +28 -28
  50. simtools/applications/validate_camera_fov.py +0 -1
  51. simtools/applications/validate_cumulative_psf.py +1 -5
  52. simtools/applications/validate_optics.py +2 -14
  53. simtools/atmosphere.py +83 -0
  54. simtools/camera/camera_efficiency.py +171 -53
  55. simtools/camera/single_photon_electron_spectrum.py +8 -7
  56. simtools/configuration/commandline_parser.py +82 -11
  57. simtools/configuration/configurator.py +6 -11
  58. simtools/constants.py +5 -0
  59. simtools/corsika/corsika_config.py +100 -202
  60. simtools/corsika/corsika_histograms.py +561 -1708
  61. simtools/corsika/primary_particle.py +1 -1
  62. simtools/data_model/metadata_collector.py +5 -2
  63. simtools/data_model/metadata_model.py +0 -4
  64. simtools/data_model/model_data_writer.py +59 -64
  65. simtools/data_model/schema.py +2 -0
  66. simtools/data_model/validate_data.py +1 -3
  67. simtools/db/db_handler.py +23 -10
  68. simtools/db/mongo_db.py +2 -2
  69. simtools/dependencies.py +81 -38
  70. simtools/io/ascii_handler.py +55 -5
  71. simtools/io/io_handler.py +23 -12
  72. simtools/io/table_handler.py +1 -1
  73. simtools/job_execution/job_manager.py +154 -79
  74. simtools/job_execution/process_pool.py +137 -0
  75. simtools/layout/array_layout.py +4 -13
  76. simtools/layout/array_layout_utils.py +348 -57
  77. simtools/model/array_model.py +23 -63
  78. simtools/model/calibration_model.py +4 -8
  79. simtools/model/legacy_model_parameter.py +134 -0
  80. simtools/model/model_parameter.py +147 -86
  81. simtools/model/model_utils.py +40 -6
  82. simtools/model/site_model.py +4 -8
  83. simtools/model/telescope_model.py +10 -16
  84. simtools/production_configuration/derive_corsika_limits.py +6 -11
  85. simtools/production_configuration/interpolation_handler.py +16 -16
  86. simtools/ray_tracing/incident_angles.py +92 -17
  87. simtools/ray_tracing/mirror_panel_psf.py +338 -222
  88. simtools/ray_tracing/psf_analysis.py +62 -48
  89. simtools/ray_tracing/psf_parameter_optimisation.py +3 -3
  90. simtools/ray_tracing/ray_tracing.py +43 -25
  91. simtools/reporting/docs_auto_report_generator.py +8 -13
  92. simtools/reporting/docs_read_parameters.py +2 -8
  93. simtools/runners/corsika_runner.py +52 -195
  94. simtools/runners/corsika_simtel_runner.py +77 -108
  95. simtools/runners/runner_services.py +214 -213
  96. simtools/runners/simtel_runner.py +27 -160
  97. simtools/runners/simtools_runner.py +11 -73
  98. simtools/schemas/application_workflow.metaschema.yml +8 -0
  99. simtools/settings.py +173 -0
  100. simtools/{io/eventio_handler.py → sim_events/file_info.py} +3 -3
  101. simtools/{simtel/simtel_io_event_histograms.py → sim_events/histograms.py} +25 -15
  102. simtools/{simtel/simtel_io_event_reader.py → sim_events/reader.py} +20 -17
  103. simtools/{simtel/simtel_io_event_writer.py → sim_events/writer.py} +84 -25
  104. simtools/simtel/pulse_shapes.py +7 -2
  105. simtools/simtel/simtel_config_writer.py +79 -91
  106. simtools/simtel/simtel_seeds.py +184 -0
  107. simtools/simtel/simtel_table_reader.py +6 -4
  108. simtools/simtel/simulator_array.py +114 -109
  109. simtools/simtel/simulator_camera_efficiency.py +68 -46
  110. simtools/simtel/simulator_light_emission.py +164 -132
  111. simtools/simtel/simulator_ray_tracing.py +80 -71
  112. simtools/simulator.py +137 -355
  113. simtools/telescope_trigger_rates.py +3 -4
  114. simtools/testing/assertions.py +84 -33
  115. simtools/testing/configuration.py +1 -2
  116. simtools/testing/helpers.py +2 -3
  117. simtools/testing/log_inspector.py +1 -0
  118. simtools/testing/sim_telarray_metadata.py +14 -12
  119. simtools/testing/validate_output.py +121 -42
  120. simtools/utils/general.py +43 -17
  121. simtools/utils/geometry.py +0 -77
  122. simtools/utils/names.py +5 -5
  123. simtools/utils/random.py +36 -0
  124. simtools/visualization/legend_handlers.py +7 -6
  125. simtools/visualization/plot_array_layout.py +91 -16
  126. simtools/visualization/plot_corsika_histograms.py +145 -605
  127. simtools/visualization/plot_incident_angles.py +48 -1
  128. simtools/visualization/plot_mirrors.py +1 -4
  129. simtools/visualization/plot_pixels.py +2 -4
  130. simtools/visualization/plot_psf.py +160 -19
  131. simtools/visualization/plot_simtel_event_histograms.py +4 -4
  132. simtools/visualization/plot_simtel_events.py +6 -11
  133. simtools/visualization/plot_tables.py +8 -19
  134. simtools/visualization/visualize.py +22 -2
  135. simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +0 -160
  136. simtools/applications/print_version.py +0 -53
  137. simtools/io/hdf5_handler.py +0 -139
  138. {gammasimtools-0.25.0.dist-info → gammasimtools-0.27.0.dist-info}/top_level.txt +0 -0
@@ -5,10 +5,10 @@ from collections import namedtuple
5
5
 
6
6
  import astropy.units as u
7
7
 
8
+ from simtools import settings
8
9
  from simtools.io import io_handler
9
10
  from simtools.runners.simtel_runner import SimtelRunner
10
11
  from simtools.utils import names
11
- from simtools.utils.general import clear_default_sim_telarray_cfg_directories
12
12
 
13
13
  # pylint: disable=no-member
14
14
  # The line above is needed because there are members which are created
@@ -27,8 +27,6 @@ class SimulatorRayTracing(SimtelRunner):
27
27
  site model
28
28
  label: str
29
29
  label used for output file naming.
30
- simtel_path: str or Path
31
- Location of sim_telarray installation.
32
30
  config_data: namedtuple
33
31
  namedtuple containing the configurable parameters as values (expected units in
34
32
  brackets): zenith_angle (deg), off_axis_angle (deg), source_distance (km),
@@ -43,7 +41,6 @@ class SimulatorRayTracing(SimtelRunner):
43
41
  telescope_model,
44
42
  site_model,
45
43
  label=None,
46
- simtel_path=None,
47
44
  config_data=None,
48
45
  force_simulate=False,
49
46
  test=False,
@@ -52,7 +49,7 @@ class SimulatorRayTracing(SimtelRunner):
52
49
  self._logger = logging.getLogger(__name__)
53
50
  self._logger.debug("Init SimulatorRayTracing")
54
51
 
55
- super().__init__(label=label, simtel_path=simtel_path)
52
+ super().__init__(label=label)
56
53
 
57
54
  self.telescope_model = telescope_model
58
55
  self.site_model = site_model
@@ -69,6 +66,8 @@ class SimulatorRayTracing(SimtelRunner):
69
66
  self._rep_number = 0
70
67
  self.runs_per_set = 1 if self.config.single_mirror_mode else 20
71
68
  self.photons_per_run = 100000 if not test else 5000
69
+ self._single_pixel_camera_file = None
70
+ self._funnel_file = None
72
71
 
73
72
  self._load_required_files(force_simulate)
74
73
 
@@ -83,10 +82,6 @@ class SimulatorRayTracing(SimtelRunner):
83
82
  force_simulate: bool
84
83
  Remove existing files and force re-running of the ray-tracing simulation.
85
84
  """
86
- # This file is not actually needed and does not exist in simtools.
87
- # It is required as CORSIKA input file to sim_telarray
88
- self._corsika_file = self._simtel_path.joinpath("run9991.corsika.gz")
89
-
90
85
  # Loop to define and remove existing files.
91
86
  # Files will be named _base_file = self.__dict__['_' + base + 'File']
92
87
  for base_name in ["stars", "photons", "log"]:
@@ -112,12 +107,13 @@ class SimulatorRayTracing(SimtelRunner):
112
107
  self.__dict__["_" + base_name + "_file"] = file
113
108
 
114
109
  if not file.exists() or force_simulate:
110
+ config_file_path = self.telescope_model.get_config_file_path(label=self.label)
115
111
  # Adding header to photon list file.
116
112
  with self._photons_file.open("w", encoding="utf-8") as file:
117
113
  file.write(f"#{50 * '='}\n")
118
114
  file.write("# List of photons for RayTracing simulations\n")
119
115
  file.write(f"#{50 * '='}\n")
120
- file.write(f"# config_file = {self.telescope_model.config_file_path}\n")
116
+ file.write(f"# config_file = {config_file_path}\n")
121
117
  file.write(f"# zenith_angle [deg] = {self.config.zenith_angle}\n")
122
118
  file.write(f"# off_axis_angle [deg] = {self.config.off_axis_angle}\n")
123
119
  file.write(f"# source_distance [km] = {self.config.source_distance}\n")
@@ -136,15 +132,20 @@ class SimulatorRayTracing(SimtelRunner):
136
132
 
137
133
  if self.config.single_mirror_mode:
138
134
  self._logger.debug("For single mirror mode, need to prepare the single pixel camera.")
139
- self._write_out_single_pixel_camera_file()
135
+ self._write_out_single_pixel_camera_files()
140
136
 
141
- def _make_run_command(self, run_number=None, input_file=None): # pylint: disable=unused-argument
137
+ def make_run_command(self, run_number=None, input_file=None): # pylint: disable=unused-argument
142
138
  """
143
139
  Generate sim_telarray run command. Export sim_telarray configuration file(s).
144
140
 
145
141
  The run_number and input_file parameters are not relevant for the ray tracing simulation.
146
142
  """
147
- self.telescope_model.write_sim_telarray_config_file(additional_models=self.site_model)
143
+ self.telescope_model.write_sim_telarray_config_file(
144
+ additional_models=self.site_model,
145
+ label=self.label,
146
+ )
147
+
148
+ config_file_path = self.telescope_model.get_config_file_path(label=self.label)
148
149
 
149
150
  if self.config.single_mirror_mode:
150
151
  # Note: no mirror length defined for dual-mirror telescopes
@@ -152,54 +153,56 @@ class SimulatorRayTracing(SimtelRunner):
152
153
  self.telescope_model.get_parameter_value("mirror_focal_length")
153
154
  )
154
155
 
155
- # RayTracing
156
- command = str(self._simtel_path.joinpath("sim_telarray/bin/sim_telarray"))
157
- command += f" -c {self.telescope_model.config_file_path}"
158
- command += f" -I{self.telescope_model.config_file_directory}"
159
- command += super().get_config_option("random_state", "none")
160
- command += super().get_config_option("IMAGING_LIST", str(self._photons_file))
161
- command += super().get_config_option("stars", str(self._stars_file))
162
- command += super().get_config_option(
163
- "altitude", self.site_model.get_parameter_value("corsika_observation_level")
164
- )
165
- command += super().get_config_option(
166
- "telescope_theta",
167
- self.config.zenith_angle + self.config.off_axis_angle,
168
- )
169
- command += super().get_config_option("star_photons", str(self.photons_per_run))
170
- command += super().get_config_option("telescope_phi", "0")
171
- command += super().get_config_option("camera_transmission", "1.0")
172
- command += super().get_config_option("nightsky_background", "all:0.")
173
- command += super().get_config_option("trigger_current_limit", "1e10")
174
- command += super().get_config_option("telescope_random_angle", "0")
175
- command += super().get_config_option("telescope_random_error", "0")
176
- command += super().get_config_option("convergent_depth", "0")
177
- command += super().get_config_option("maximum_telescopes", "1")
178
- command += super().get_config_option("show", "all")
179
- command += super().get_config_option("camera_filter", "none")
156
+ options = {
157
+ "random_state": "none",
158
+ "IMAGING_LIST": str(self._photons_file),
159
+ "stars": str(self._stars_file),
160
+ "altitude": self.site_model.get_parameter_value("corsika_observation_level"),
161
+ "telescope_theta": self.config.zenith_angle + self.config.off_axis_angle,
162
+ "star_photons": str(self.photons_per_run),
163
+ "telescope_phi": "0",
164
+ "camera_transmission": "1.0",
165
+ "nightsky_background": "all:0.",
166
+ "trigger_current_limit": "1e10",
167
+ "telescope_random_angle": "0",
168
+ "telescope_random_error": "0",
169
+ "convergent_depth": "0",
170
+ "maximum_telescopes": "1",
171
+ "show": "all",
172
+ "camera_filter": "none",
173
+ }
174
+
180
175
  if self.config.single_mirror_mode:
181
- command += super().get_config_option("focus_offset", "all:0.")
182
- command += super().get_config_option("camera_config_file", "single_pixel_camera.dat")
183
- command += super().get_config_option("camera_pixels", "1")
184
- command += super().get_config_option("trigger_pixels", "1")
185
- command += super().get_config_option("camera_body_diameter", "0")
186
- command += super().get_config_option(
187
- "mirror_list",
188
- self.telescope_model.get_single_mirror_list_file(
189
- self.config.mirror_numbers, self.config.use_random_focal_length
190
- ),
191
- )
192
- command += super().get_config_option(
193
- "focal_length", self.config.source_distance * u.km.to(u.cm)
176
+ options.update(
177
+ {
178
+ "focus_offset": "all:0.",
179
+ "camera_config_file": str(self._single_pixel_camera_file),
180
+ "camera_pixels": "1",
181
+ "trigger_pixels": "1",
182
+ "camera_body_diameter": "0",
183
+ "mirror_list": self.telescope_model.get_single_mirror_list_file(
184
+ self.config.mirror_numbers, self.config.use_random_focal_length
185
+ ),
186
+ "focal_length": self.config.source_distance * u.km.to(u.cm),
187
+ "dish_shape_length": _mirror_focal_length,
188
+ "mirror_focal_length": _mirror_focal_length,
189
+ "parabolic_dish": "0",
190
+ "mirror_align_random_distance": "0.",
191
+ "mirror_align_random_vertical": "0.,28.,0.,0.",
192
+ }
194
193
  )
195
- command += super().get_config_option("dish_shape_length", _mirror_focal_length)
196
- command += super().get_config_option("mirror_focal_length", _mirror_focal_length)
197
- command += super().get_config_option("parabolic_dish", "0")
198
- command += super().get_config_option("mirror_align_random_distance", "0.")
199
- command += super().get_config_option("mirror_align_random_vertical", "0.,28.,0.,0.")
200
- command += " " + str(self._corsika_file)
201
194
 
202
- return clear_default_sim_telarray_cfg_directories(command), self._log_file, self._log_file
195
+ cmd = [
196
+ str(settings.config.sim_telarray_exe),
197
+ "-c",
198
+ str(config_file_path),
199
+ f"-I{self.telescope_model.config_file_directory}",
200
+ ]
201
+ for key, value in options.items():
202
+ cmd.extend(["-C", f"{key}={value}"])
203
+ cmd.append(str(settings.config.corsika_dummy_file))
204
+
205
+ return cmd, self._log_file, self._log_file
203
206
 
204
207
  def _check_run_result(self, run_number=None): # pylint: disable=unused-argument
205
208
  """
@@ -223,20 +226,19 @@ class SimulatorRayTracing(SimtelRunner):
223
226
  raise RuntimeError("Photon list is empty.")
224
227
  return True
225
228
 
226
- def _write_out_single_pixel_camera_file(self):
227
- """Write out the single pixel camera file."""
228
- with self.telescope_model.config_file_directory.joinpath("single_pixel_camera.dat").open(
229
- "w"
230
- ) as file:
231
- file.write("# Single pixel camera\n")
232
- file.write('PixType 1 0 0 300 1 300 0.00 "funnel_perfect.dat"\n')
233
- file.write("Pixel 0 1 0. 0. 0 0 0 0x00 1\n")
234
- file.write("Trigger 1 of 0\n")
229
+ def _write_out_single_pixel_camera_files(self):
230
+ """Write out per-label single pixel camera + funnel files.
231
+
232
+ These files are referenced by sim_telarray and must not be shared across
233
+ parallel worker processes (otherwise they can be truncated mid-read).
234
+ """
235
+ self._single_pixel_camera_file = (
236
+ self._base_directory / f"single_pixel_camera_{self.label}.dat"
237
+ )
238
+ self._funnel_file = self._base_directory / f"funnel_perfect_{self.label}.dat"
235
239
 
236
- # need to also write out the funnel_perfect.dat file
237
- with self.telescope_model.config_file_directory.joinpath("funnel_perfect.dat").open(
238
- "w"
239
- ) as file:
240
+ # Funnel file (referenced by absolute path from the camera file).
241
+ with self._funnel_file.open("w", encoding="utf-8") as file:
240
242
  file.write(
241
243
  "# Perfect light collection where the angular efficiency of funnels is needed\n"
242
244
  )
@@ -245,6 +247,13 @@ class SimulatorRayTracing(SimtelRunner):
245
247
  file.write("60 1.0\n")
246
248
  file.write("90 1.0\n")
247
249
 
250
+ # Camera config.
251
+ with self._single_pixel_camera_file.open("w", encoding="utf-8") as file:
252
+ file.write("# Single pixel camera\n")
253
+ file.write(f'PixType 1 0 0 300 1 300 0.00 "{self._funnel_file}"\n')
254
+ file.write("Pixel 0 1 0. 0. 0 0 0 0x00 1\n")
255
+ file.write("Trigger 1 of 0\n")
256
+
248
257
  def _config_to_namedtuple(self, data_dict):
249
258
  """Convert dict to namedtuple for configuration."""
250
259
  config_data = namedtuple(