gammasimtools 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/METADATA +2 -1
  2. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/RECORD +134 -130
  3. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/entry_points.txt +3 -1
  4. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/licenses/LICENSE +1 -1
  5. simtools/_version.py +2 -2
  6. simtools/application_control.py +78 -0
  7. simtools/applications/calculate_incident_angles.py +0 -2
  8. simtools/applications/convert_geo_coordinates_of_array_elements.py +1 -2
  9. simtools/applications/db_add_file_to_db.py +1 -1
  10. simtools/applications/db_add_simulation_model_from_repository_to_db.py +1 -1
  11. simtools/applications/db_add_value_from_json_to_db.py +1 -1
  12. simtools/applications/db_generate_compound_indexes.py +1 -1
  13. simtools/applications/db_get_array_layouts_from_db.py +2 -6
  14. simtools/applications/db_get_file_from_db.py +1 -1
  15. simtools/applications/db_get_parameter_from_db.py +1 -1
  16. simtools/applications/db_inspect_databases.py +1 -1
  17. simtools/applications/db_upload_model_repository.py +1 -1
  18. simtools/applications/derive_ctao_array_layouts.py +1 -2
  19. simtools/applications/derive_mirror_rnda.py +1 -3
  20. simtools/applications/derive_psf_parameters.py +5 -1
  21. simtools/applications/derive_pulse_shape_parameters.py +194 -0
  22. simtools/applications/derive_trigger_rates.py +1 -1
  23. simtools/applications/docs_produce_array_element_report.py +2 -8
  24. simtools/applications/docs_produce_calibration_reports.py +1 -3
  25. simtools/applications/docs_produce_model_parameter_reports.py +0 -2
  26. simtools/applications/docs_produce_simulation_configuration_report.py +1 -3
  27. simtools/applications/generate_array_config.py +0 -1
  28. simtools/applications/generate_corsika_histograms.py +48 -235
  29. simtools/applications/generate_regular_arrays.py +5 -35
  30. simtools/applications/generate_simtel_event_data.py +2 -2
  31. simtools/applications/maintain_simulation_model_add_production.py +2 -2
  32. simtools/applications/maintain_simulation_model_write_array_element_positions.py +87 -0
  33. simtools/applications/plot_array_layout.py +64 -108
  34. simtools/applications/plot_simulated_event_distributions.py +57 -0
  35. simtools/applications/plot_tabular_data.py +0 -1
  36. simtools/applications/plot_tabular_data_for_model_parameter.py +1 -6
  37. simtools/applications/production_derive_corsika_limits.py +1 -1
  38. simtools/applications/production_generate_grid.py +0 -1
  39. simtools/applications/run_application.py +1 -1
  40. simtools/applications/simulate_flasher.py +3 -4
  41. simtools/applications/simulate_illuminator.py +0 -1
  42. simtools/applications/simulate_pedestals.py +2 -6
  43. simtools/applications/simulate_prod.py +9 -28
  44. simtools/applications/simulate_prod_htcondor_generator.py +8 -1
  45. simtools/applications/submit_array_layouts.py +7 -7
  46. simtools/applications/submit_model_parameter_from_external.py +1 -3
  47. simtools/applications/validate_camera_efficiency.py +0 -1
  48. simtools/applications/validate_camera_fov.py +0 -1
  49. simtools/applications/validate_cumulative_psf.py +0 -2
  50. simtools/applications/validate_file_using_schema.py +49 -123
  51. simtools/applications/validate_optics.py +0 -13
  52. simtools/camera/camera_efficiency.py +1 -6
  53. simtools/camera/single_photon_electron_spectrum.py +2 -1
  54. simtools/configuration/commandline_parser.py +43 -8
  55. simtools/configuration/configurator.py +6 -11
  56. simtools/corsika/corsika_config.py +204 -99
  57. simtools/corsika/corsika_histograms.py +411 -1735
  58. simtools/corsika/primary_particle.py +1 -1
  59. simtools/data_model/metadata_collector.py +5 -2
  60. simtools/data_model/metadata_model.py +0 -4
  61. simtools/data_model/model_data_writer.py +27 -17
  62. simtools/data_model/schema.py +112 -5
  63. simtools/data_model/validate_data.py +80 -48
  64. simtools/db/db_handler.py +19 -8
  65. simtools/db/db_model_upload.py +2 -1
  66. simtools/db/mongo_db.py +133 -42
  67. simtools/dependencies.py +83 -44
  68. simtools/io/ascii_handler.py +4 -2
  69. simtools/io/table_handler.py +1 -1
  70. simtools/job_execution/htcondor_script_generator.py +0 -2
  71. simtools/layout/array_layout.py +4 -12
  72. simtools/layout/array_layout_utils.py +227 -58
  73. simtools/model/array_model.py +37 -18
  74. simtools/model/calibration_model.py +0 -4
  75. simtools/model/legacy_model_parameter.py +134 -0
  76. simtools/model/model_parameter.py +24 -14
  77. simtools/model/model_repository.py +18 -5
  78. simtools/model/model_utils.py +1 -6
  79. simtools/model/site_model.py +0 -4
  80. simtools/model/telescope_model.py +6 -11
  81. simtools/production_configuration/derive_corsika_limits.py +6 -11
  82. simtools/production_configuration/interpolation_handler.py +16 -16
  83. simtools/ray_tracing/incident_angles.py +5 -11
  84. simtools/ray_tracing/mirror_panel_psf.py +3 -7
  85. simtools/ray_tracing/psf_analysis.py +29 -27
  86. simtools/ray_tracing/psf_parameter_optimisation.py +822 -680
  87. simtools/ray_tracing/ray_tracing.py +6 -15
  88. simtools/reporting/docs_auto_report_generator.py +8 -13
  89. simtools/reporting/docs_read_parameters.py +70 -16
  90. simtools/runners/corsika_runner.py +15 -10
  91. simtools/runners/corsika_simtel_runner.py +9 -8
  92. simtools/runners/runner_services.py +17 -7
  93. simtools/runners/simtel_runner.py +11 -58
  94. simtools/runners/simtools_runner.py +2 -4
  95. simtools/schemas/model_parameters/flasher_pulse_exp_decay.schema.yml +2 -0
  96. simtools/schemas/model_parameters/flasher_pulse_shape.schema.yml +50 -0
  97. simtools/schemas/model_parameters/flasher_pulse_width.schema.yml +2 -0
  98. simtools/schemas/simulation_models_info.schema.yml +2 -0
  99. simtools/settings.py +154 -0
  100. simtools/sim_events/file_info.py +128 -0
  101. simtools/{simtel/simtel_io_event_histograms.py → sim_events/histograms.py} +25 -15
  102. simtools/{simtel/simtel_io_event_reader.py → sim_events/reader.py} +20 -17
  103. simtools/{simtel/simtel_io_event_writer.py → sim_events/writer.py} +84 -25
  104. simtools/simtel/pulse_shapes.py +273 -0
  105. simtools/simtel/simtel_config_writer.py +146 -22
  106. simtools/simtel/simtel_table_reader.py +6 -4
  107. simtools/simtel/simulator_array.py +62 -23
  108. simtools/simtel/simulator_camera_efficiency.py +4 -6
  109. simtools/simtel/simulator_light_emission.py +101 -19
  110. simtools/simtel/simulator_ray_tracing.py +4 -10
  111. simtools/simulator.py +360 -353
  112. simtools/telescope_trigger_rates.py +3 -4
  113. simtools/testing/assertions.py +115 -8
  114. simtools/testing/configuration.py +2 -3
  115. simtools/testing/helpers.py +2 -3
  116. simtools/testing/log_inspector.py +5 -1
  117. simtools/testing/sim_telarray_metadata.py +1 -1
  118. simtools/testing/validate_output.py +69 -23
  119. simtools/utils/general.py +37 -0
  120. simtools/utils/geometry.py +0 -77
  121. simtools/utils/names.py +7 -9
  122. simtools/version.py +37 -0
  123. simtools/visualization/legend_handlers.py +21 -10
  124. simtools/visualization/plot_array_layout.py +312 -41
  125. simtools/visualization/plot_corsika_histograms.py +143 -605
  126. simtools/visualization/plot_mirrors.py +834 -0
  127. simtools/visualization/plot_pixels.py +2 -4
  128. simtools/visualization/plot_psf.py +0 -1
  129. simtools/visualization/plot_simtel_event_histograms.py +4 -4
  130. simtools/visualization/plot_simtel_events.py +6 -11
  131. simtools/visualization/plot_tables.py +8 -19
  132. simtools/visualization/visualize.py +22 -2
  133. simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +0 -160
  134. simtools/applications/print_version.py +0 -53
  135. simtools/io/hdf5_handler.py +0 -139
  136. simtools/simtel/simtel_io_file_info.py +0 -62
  137. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/WHEEL +0 -0
  138. {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/top_level.txt +0 -0
@@ -8,13 +8,13 @@ from ctao_cr_spectra.definitions import IRFDOC_PROTON_SPECTRUM
8
8
 
9
9
  from simtools.io import ascii_handler, io_handler
10
10
  from simtools.layout.array_layout_utils import get_array_elements_from_db_for_layouts
11
- from simtools.simtel.simtel_io_event_histograms import SimtelIOEventHistograms
11
+ from simtools.sim_events.histograms import EventDataHistograms
12
12
  from simtools.visualization import plot_simtel_event_histograms
13
13
 
14
14
  _logger = logging.getLogger(__name__)
15
15
 
16
16
 
17
- def telescope_trigger_rates(args_dict, db_config):
17
+ def telescope_trigger_rates(args_dict):
18
18
  """
19
19
  Calculate trigger rates for single telescopes or arrays of telescopes.
20
20
 
@@ -27,7 +27,6 @@ def telescope_trigger_rates(args_dict, db_config):
27
27
  args_dict["array_layout_name"],
28
28
  args_dict.get("site"),
29
29
  args_dict.get("model_version"),
30
- db_config,
31
30
  )
32
31
  else:
33
32
  telescope_configs = ascii_handler.collect_data_from_file(args_dict["telescope_ids"])[
@@ -38,7 +37,7 @@ def telescope_trigger_rates(args_dict, db_config):
38
37
  _logger.info(
39
38
  f"Processing file: {args_dict['event_data_file']} with telescope config: {array_name}"
40
39
  )
41
- histograms = SimtelIOEventHistograms(
40
+ histograms = EventDataHistograms(
42
41
  args_dict["event_data_file"], array_name=array_name, telescope_list=telescope_ids
43
42
  )
44
43
  histograms.fill()
@@ -1,7 +1,9 @@
1
1
  """Functions asserting certain conditions are met (used e.g., in integration tests)."""
2
2
 
3
+ import gzip
3
4
  import json
4
5
  import logging
6
+ import tarfile
5
7
  from collections import defaultdict
6
8
  from pathlib import Path
7
9
 
@@ -66,8 +68,7 @@ def assert_n_showers_and_energy_range(file):
66
68
  simulation_config = {}
67
69
  with SimTelFile(file, skip_non_triggered=False) as f:
68
70
  simulation_config = f.mc_run_headers[0]
69
- for event in f:
70
- simulated_energies.append(event["mc_shower"]["energy"])
71
+ simulated_energies.extend(event["mc_shower"]["energy"] for event in f)
71
72
 
72
73
  # The relative tolerance is set to 1% because ~0.5% shower simulations do not
73
74
  # succeed, without resulting in an error. This tolerance therefore is not an issue.
@@ -174,12 +175,6 @@ def check_output_from_sim_telarray(file, file_test):
174
175
  _logger.debug(f"No expected output or metadata provided, skipping checks {file_test}")
175
176
  return True
176
177
 
177
- if file.suffix != ".zst":
178
- raise ValueError(
179
- f"Expected output file {file} is not a zstd compressed file "
180
- f"(i.e., a sim_telarray file)."
181
- )
182
-
183
178
  assert_output = assert_metadata = True
184
179
 
185
180
  if "expected_output" in file_test:
@@ -193,3 +188,115 @@ def check_output_from_sim_telarray(file, file_test):
193
188
  )
194
189
 
195
190
  return assert_n_showers_and_energy_range(file=file) and assert_output and assert_metadata
191
+
192
+
193
+ def check_simulation_logs(tar_file, file_test):
194
+ """
195
+ Check simulation logs for wanted and forbidden patterns.
196
+
197
+ Parameters
198
+ ----------
199
+ tar_file : str
200
+ Path to the tar file.
201
+ file_test : dict
202
+ Dictionary with the test configuration.
203
+
204
+ Returns
205
+ -------
206
+ bool
207
+ True if the logs are correct.
208
+ """
209
+ wanted, forbidden = _get_expected_patterns(file_test)
210
+ if wanted is None:
211
+ return True
212
+
213
+ if not tarfile.is_tarfile(tar_file):
214
+ raise ValueError(f"{tar_file} is not a tar file")
215
+
216
+ found_wanted = set()
217
+ found_forbidden = set()
218
+ with tarfile.open(tar_file, "r:*") as tar:
219
+ for member in tar.getmembers():
220
+ if not member.name.endswith(".log.gz"):
221
+ continue
222
+ _logger.info(f"Scanning {member.name}")
223
+ text = _read_log(member, tar)
224
+ found_wanted |= _find_patterns(text, wanted)
225
+ found_forbidden |= _find_patterns(text, forbidden)
226
+
227
+ return _validate_patterns(found_wanted, found_forbidden, wanted)
228
+
229
+
230
+ def check_plain_log(log_file, file_test):
231
+ """
232
+ Check plain log file for wanted and forbidden patterns.
233
+
234
+ Parameters
235
+ ----------
236
+ log_file : str
237
+ Path to the log file.
238
+ file_test : dict
239
+ Dictionary with the test configuration.
240
+
241
+ Returns
242
+ -------
243
+ bool
244
+ True if the logs are correct.
245
+ """
246
+ wanted, forbidden = _get_expected_patterns(file_test)
247
+ if wanted is None:
248
+ return True
249
+
250
+ try:
251
+ with open(log_file, encoding="utf-8") as f:
252
+ text = f.read()
253
+ except FileNotFoundError:
254
+ _logger.error(f"Log file {log_file} not found")
255
+ return False
256
+
257
+ found = _find_patterns(text, wanted)
258
+ bad = _find_patterns(text, forbidden)
259
+
260
+ return _validate_patterns(found, bad, wanted)
261
+
262
+
263
+ def _get_expected_patterns(file_test):
264
+ """Get wanted and forbidden patterns from file test configuration."""
265
+ expected_log = file_test.get("expected_log_output")
266
+ if isinstance(expected_log, dict):
267
+ wanted = expected_log.get("pattern", [])
268
+ forbidden = expected_log.get("forbidden_pattern", [])
269
+ else:
270
+ wanted = file_test.get("pattern", [])
271
+ forbidden = file_test.get("forbidden_pattern", [])
272
+ if not (wanted or forbidden):
273
+ _logger.debug(f"No expected log output provided, skipping checks {file_test}")
274
+ return None, None
275
+
276
+ return wanted, forbidden
277
+
278
+
279
+ def _validate_patterns(found, bad, wanted):
280
+ """Validate found patterns against wanted and forbidden ones."""
281
+ if bad:
282
+ _logger.error(f"Forbidden patterns found: {list(bad)}")
283
+ return False
284
+ missing = [p for p in wanted if p not in found]
285
+ if missing:
286
+ _logger.error(f"Missing expected patterns: {missing}")
287
+ return False
288
+
289
+ _logger.debug(f"All expected patterns found: {wanted}")
290
+ return True
291
+
292
+
293
+ def _find_patterns(text, patterns):
294
+ """Find patterns in text (case insensitive)."""
295
+ text_lower = text.lower()
296
+ return {p for p in patterns if p.lower() in text_lower}
297
+
298
+
299
+ def _read_log(member, tar):
300
+ """Read and decode a gzipped log file from a tar archive."""
301
+ with tar.extractfile(member) as gz, gzip.open(gz, "rb") as f:
302
+ return f.read().decode("utf-8", "ignore")
@@ -76,8 +76,7 @@ def _read_configs_from_files(config_files):
76
76
  _dict = gen.remove_substring_recursively_from_dict(
77
77
  ascii_handler.collect_data_from_file(file_name=config_file), substring="\n"
78
78
  )
79
- for application in _dict.get("applications", []):
80
- configs.append(application)
79
+ configs.extend(_dict.get("applications", []))
81
80
  return configs
82
81
 
83
82
 
@@ -204,7 +203,7 @@ def _prepare_test_options(config, output_path, model_version=None):
204
203
  if model_version and "model_version" in config:
205
204
  config.update({"model_version": model_version})
206
205
 
207
- for key in ["output_path", "data_directory", "pack_for_grid_register"]:
206
+ for key in ["output_path", "pack_for_grid_register"]:
208
207
  if key in config:
209
208
  config[key] = str(Path(output_path).joinpath(config[key]))
210
209
 
@@ -1,7 +1,6 @@
1
1
  """Helper functions for integration testing."""
2
2
 
3
- import os
4
- from pathlib import Path
3
+ from simtools import settings
5
4
 
6
5
 
7
6
  def skip_camera_efficiency(config):
@@ -21,7 +20,7 @@ def _new_testeff_version():
21
20
 
22
21
  This test checks if the new version is used.
23
22
  """
24
- testeff_path = Path(os.getenv("SIMTOOLS_SIMTEL_PATH")) / "sim_telarray/testeff.c"
23
+ testeff_path = settings.config.sim_telarray_path / "testeff.c"
25
24
  try:
26
25
  with open(testeff_path, encoding="utf-8") as file:
27
26
  file_content = file.read()
@@ -15,7 +15,11 @@ ERROR_PATTERNS = [
15
15
  re.compile(r"segmentation fault", re.IGNORECASE),
16
16
  ]
17
17
 
18
- IGNORE_PATTERNS = [re.compile(r"Falling back to 'utf-8' with errors='ignore'", re.IGNORECASE)]
18
+ IGNORE_PATTERNS = [
19
+ re.compile(r"Falling back to 'utf-8' with errors='ignore'", re.IGNORECASE),
20
+ re.compile(r"Failed to get user name[^\n]*setting it to UNKNOWN_USER", re.IGNORECASE),
21
+ re.compile(r"adjust_text::Error", re.IGNORECASE),
22
+ ]
19
23
 
20
24
 
21
25
  def inspect(log_text):
@@ -4,9 +4,9 @@ import logging
4
4
 
5
5
  import numpy as np
6
6
 
7
+ from simtools.sim_events.file_info import get_corsika_run_number
7
8
  from simtools.simtel.simtel_config_reader import SimtelConfigReader
8
9
  from simtools.simtel.simtel_config_writer import sim_telarray_random_seeds
9
- from simtools.simtel.simtel_io_file_info import get_corsika_run_number
10
10
  from simtools.simtel.simtel_io_metadata import (
11
11
  get_sim_telarray_telescope_id,
12
12
  read_sim_telarray_metadata,
@@ -14,9 +14,35 @@ from simtools.testing import assertions
14
14
  _logger = logging.getLogger(__name__)
15
15
 
16
16
 
17
- def validate_application_output(
18
- config, from_command_line=None, from_config_file=None, db_config=None
19
- ):
17
+ def _versions_match(from_command_line, from_config_file):
18
+ """Return True if validations should run for the given versions.
19
+
20
+ Behavior:
21
+ - If no version is provided from the command line, run validations.
22
+ - If a filter is provided from the command line, run only when it matches
23
+ the version(s) from the config file.
24
+ """
25
+ if from_command_line is None:
26
+ return True
27
+
28
+ # Normalize to collections for comparison
29
+ cmd_versions = from_command_line if isinstance(from_command_line, list) else [from_command_line]
30
+ cfg_versions = from_config_file if isinstance(from_config_file, list) else [from_config_file]
31
+
32
+ # Consider a match if any overlap exists
33
+ return any(cv in cmd_versions for cv in cfg_versions)
34
+
35
+
36
+ # Keys to ignore when comparing sim_telarray configuration files
37
+ # (e.g., version numbers, system dependent parameters, CORSIKA options)
38
+ cfg_ignore_keys = [
39
+ "config_release",
40
+ "Label",
41
+ "simtools_", # ignore all simtools_ keys - version/build info dependence
42
+ ]
43
+
44
+
45
+ def validate_application_output(config, from_command_line=None, from_config_file=None):
20
46
  """
21
47
  Validate application output against expected output.
22
48
 
@@ -39,9 +65,13 @@ def validate_application_output(
39
65
 
40
66
  for integration_test in config["integration_tests"]:
41
67
  _logger.info(f"Testing application output: {integration_test}")
68
+ _logger.debug(
69
+ f"Model version from command line: {from_command_line}, "
70
+ f"from config file: {from_config_file}"
71
+ )
42
72
 
43
- if from_command_line == from_config_file:
44
- _validate_output_files(config, integration_test, db_config)
73
+ if _versions_match(from_command_line, from_config_file):
74
+ _validate_output_files(config, integration_test)
45
75
 
46
76
  if "file_type" in integration_test:
47
77
  assert assertions.assert_file_type(
@@ -53,7 +83,7 @@ def validate_application_output(
53
83
  _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file)
54
84
 
55
85
 
56
- def _validate_output_files(config, integration_test, db_config):
86
+ def _validate_output_files(config, integration_test):
57
87
  """Validate output files."""
58
88
  if "reference_output_file" in integration_test:
59
89
  _validate_reference_output_file(config, integration_test)
@@ -65,11 +95,7 @@ def _validate_output_files(config, integration_test, db_config):
65
95
  [{"path_descriptor": "output_path", "file": integration_test["output_file"]}],
66
96
  )
67
97
  if "model_parameter_validation" in integration_test:
68
- _validate_model_parameter_json_file(
69
- config,
70
- integration_test["model_parameter_validation"],
71
- db_config,
72
- )
98
+ _validate_model_parameter_json_file(config, integration_test["model_parameter_validation"])
73
99
 
74
100
 
75
101
  def _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file):
@@ -115,12 +141,20 @@ def _validate_output_path_and_file(config, integration_file_tests):
115
141
  try:
116
142
  assert output_file_path.exists()
117
143
  except AssertionError as exc:
118
- raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
144
+ raise AssertionError(
145
+ f"Output file {output_file_path} does not exist. "
146
+ f"Directory contents: {list(output_file_path.parent.iterdir())}"
147
+ ) from exc
119
148
 
120
- assert assertions.check_output_from_sim_telarray(output_file_path, file_test)
149
+ if output_file_path.name.endswith(".simtel.zst"):
150
+ assert assertions.check_output_from_sim_telarray(output_file_path, file_test)
151
+ elif output_file_path.name.endswith(".log_hist.tar.gz"):
152
+ assert assertions.check_simulation_logs(output_file_path, file_test)
153
+ elif output_file_path.suffix == ".log":
154
+ assert assertions.check_plain_log(output_file_path, file_test)
121
155
 
122
156
 
123
- def _validate_model_parameter_json_file(config, model_parameter_validation, db_config):
157
+ def _validate_model_parameter_json_file(config, model_parameter_validation):
124
158
  """
125
159
  Validate model parameter json file and compare it with a reference parameter from the database.
126
160
 
@@ -135,7 +169,7 @@ def _validate_model_parameter_json_file(config, model_parameter_validation, db_c
135
169
 
136
170
  """
137
171
  _logger.info(f"Checking model parameter json file: {model_parameter_validation}")
138
- db = db_handler.DatabaseHandler(db_config=db_config)
172
+ db = db_handler.DatabaseHandler()
139
173
 
140
174
  reference_parameter_name = model_parameter_validation.get("reference_parameter_name")
141
175
 
@@ -350,8 +384,8 @@ def _compare_simtel_cfg_files(reference_file, test_file):
350
384
  Compare two sim_telarray configuration files.
351
385
 
352
386
  Line-by-line string comparison. Requires similar sequence of
353
- parameters in the files. Ignore lines containing 'config_release'
354
- (as it contains the simtools package version).
387
+ parameters in the files. Ignore lines listed in cfg_ignore_keys
388
+ (e.g., simtools package versions or hadronic interaction model strings).
355
389
 
356
390
  Parameters
357
391
  ----------
@@ -370,16 +404,28 @@ def _compare_simtel_cfg_files(reference_file, test_file):
370
404
  reference_cfg = [line.rstrip() for line in f1 if line.strip()]
371
405
  test_cfg = [line.rstrip() for line in f2 if line.strip()]
372
406
 
373
- if len(reference_cfg) != len(test_cfg):
407
+ def filter_ignored(cfg_lines, file_label):
408
+ filtered = []
409
+ for line in cfg_lines:
410
+ ignored_key = next((ignore for ignore in cfg_ignore_keys if ignore in line), None)
411
+ if ignored_key:
412
+ _logger.debug(f"Ignoring line in {file_label} due to key '{ignored_key}': {line}")
413
+ continue
414
+ filtered.append(line)
415
+ return filtered
416
+
417
+ reference_cfg_filtered = filter_ignored(reference_cfg, "reference file")
418
+ test_cfg_filtered = filter_ignored(test_cfg, "test file")
419
+
420
+ if len(reference_cfg_filtered) != len(test_cfg_filtered):
374
421
  _logger.error(
375
- f"Line counts differ: {reference_file} ({len(reference_cfg)} lines), "
376
- f"{test_file} ({len(test_cfg)} lines)."
422
+ f"Line counts differ after filtering: {reference_file} "
423
+ f"({len(reference_cfg_filtered)} lines), "
424
+ f"{test_file} ({len(test_cfg_filtered)} lines)."
377
425
  )
378
426
  return False
379
427
 
380
- for ref_line, test_line in zip(reference_cfg, test_cfg):
381
- if any(ignore in ref_line for ignore in ("config_release", "Label", "simtools_version")):
382
- continue
428
+ for ref_line, test_line in zip(reference_cfg_filtered, test_cfg_filtered):
383
429
  if ref_line != test_line:
384
430
  _logger.error(
385
431
  f"Configuration files {reference_file} and {test_file} do not match: "
simtools/utils/general.py CHANGED
@@ -11,6 +11,7 @@ import urllib.request
11
11
  from pathlib import Path
12
12
  from urllib.parse import urlparse
13
13
 
14
+ import dotenv
14
15
  import numpy as np
15
16
 
16
17
  _logger = logging.getLogger(__name__)
@@ -850,3 +851,39 @@ def get_list_of_files_from_command_line(file_names, suffix_list):
850
851
  def now_date_time_in_isoformat():
851
852
  """Return date and time in isoformat and second accuracy."""
852
853
  return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
854
+
855
+
856
+ def load_environment_variables(env_file=".env", env_list=None):
857
+ """
858
+ Load environment variables (from a .env file or directly from the environment).
859
+
860
+ Allow to read a specific list of variables or all variables from the .env file.
861
+
862
+ Parameters
863
+ ----------
864
+ env_file: str
865
+ Path to the .env file.
866
+ env_list: list, optional
867
+ List of environment variables to be read. If None, all variables are read.
868
+
869
+ Returns
870
+ -------
871
+ dict
872
+ Dictionary mapping environment variable names (lowercase, without the
873
+ ``SIMTOOLS_`` prefix) to their cleaned string values.
874
+ """
875
+ dotenv.load_dotenv(env_file or None)
876
+ keys = (
877
+ list(dotenv.dotenv_values(env_file).keys())
878
+ if env_list is None
879
+ else [f"SIMTOOLS_{s.upper()}" for s in env_list]
880
+ )
881
+
882
+ env_values = {}
883
+ for key in keys:
884
+ env_value = os.environ.get(key)
885
+ if env_value is None:
886
+ continue
887
+ cleaned_value = env_value.split("#")[0].strip().replace('"', "").replace("'", "")
888
+ env_values[key.removeprefix("SIMTOOLS_").lower()] = cleaned_value
889
+ return env_values
@@ -1,88 +1,11 @@
1
1
  """A collection of functions related to geometrical transformations."""
2
2
 
3
- import logging
4
3
  import math
5
4
 
6
5
  import astropy.units as u
7
6
  import numpy as np
8
7
  from astropy.units import UnitsError
9
8
 
10
- _logger = logging.getLogger(__name__)
11
-
12
-
13
- def convert_2d_to_radial_distr(hist_2d, xaxis, yaxis, bins=50, max_dist=1000):
14
- """
15
- Convert a 2d histogram of positions, e.g. photon positions on the ground, to a 1D distribution.
16
-
17
- Parameters
18
- ----------
19
- hist_2d: numpy.ndarray
20
- The histogram counts.
21
- xaxis: numpy.array
22
- The values of the x axis (histogram bin edges) on the ground.
23
- yaxis: numpy.array
24
- The values of the y axis (histogram bin edges) on the ground.
25
- bins: float
26
- Number of bins in distance.
27
- max_dist: float
28
- Maximum distance to consider in the 1D histogram, usually in meters.
29
-
30
- Returns
31
- -------
32
- np.array
33
- The values of the 1D histogram with size = int(max_dist/bin_size).
34
- np.array
35
- The bin edges of the 1D histogram with size = int(max_dist/bin_size) + 1.
36
-
37
- """
38
- # Check if the histogram will make sense
39
- bins_step = 2 * max_dist / bins # in the 2d array, the positive and negative direction count.
40
- for axis in [xaxis, yaxis]:
41
- if (bins_step < np.diff(axis)).any():
42
- msg = (
43
- f"The histogram with number of bins {bins} and maximum distance of {max_dist} "
44
- f"resulted in a bin size smaller than the original array. Please adjust those "
45
- f"parameters to increase the bin size and avoid nan in the histogram values."
46
- )
47
- _logger.warning(msg)
48
- break
49
-
50
- grid_2d_x, grid_2d_y = np.meshgrid(xaxis[:-1], yaxis[:-1]) # [:-1], since xaxis and yaxis are
51
- # the hist bin_edges (n + 1).
52
- # radial_distance_map maps the distance to the center from each element in a square matrix.
53
- radial_distance_map = np.sqrt(grid_2d_x**2 + grid_2d_y**2)
54
- # The sorting and unravel_index give us the two indices for the position of the sorted element
55
- # in the original 2d matrix
56
- sorted_indices = np.unravel_index(
57
- np.argsort(radial_distance_map, axis=None), np.shape(radial_distance_map)
58
- )
59
- x_indices_sorted, y_indices_sorted = sorted_indices[0], sorted_indices[1]
60
-
61
- # We construct a 1D array with the histogram counts sorted according to the distance to the
62
- # center.
63
- hist_sorted = np.array(
64
- [hist_2d[i_x, i_y] for i_x, i_y in zip(x_indices_sorted, y_indices_sorted)]
65
- )
66
- distance_sorted = np.sort(radial_distance_map, axis=None)
67
-
68
- # For larger distances, we have more elements in a slice 'dr' in radius, hence, we need to
69
- # account for it using weights below.
70
-
71
- weights, radial_bin_edges = np.histogram(distance_sorted, bins=bins, range=(0, max_dist))
72
- histogram_1d = np.empty_like(weights, dtype=float)
73
-
74
- for i_radial, _ in enumerate(radial_bin_edges[:-1]):
75
- # Here we sum all the events within a radial interval 'dr' and then divide by the number of
76
- # bins that fit this interval.
77
- indices_to_sum = (distance_sorted >= radial_bin_edges[i_radial]) * (
78
- distance_sorted < radial_bin_edges[i_radial + 1]
79
- )
80
- if weights[i_radial] != 0:
81
- histogram_1d[i_radial] = np.sum(hist_sorted[indices_to_sum]) / weights[i_radial]
82
- else:
83
- histogram_1d[i_radial] = 0
84
- return histogram_1d, radial_bin_edges
85
-
86
9
 
87
10
  @u.quantity_input(rotation_angle_phi=u.rad, rotation_angle_theta=u.rad)
88
11
  def rotate(x, y, rotation_around_z_axis, rotation_around_y_axis=0):
simtools/utils/names.py CHANGED
@@ -189,13 +189,13 @@ def model_parameters(class_key_list=None):
189
189
  dict
190
190
  Model parameters definitions.
191
191
  """
192
- _parameters = {}
193
192
  if class_key_list is None:
194
193
  return _load_model_parameters()
195
- for key, value in _load_model_parameters().items():
196
- if value.get("instrument", {}).get("class", "") in class_key_list:
197
- _parameters[key] = value
198
- return _parameters
194
+ return {
195
+ key: value
196
+ for key, value in _load_model_parameters().items()
197
+ if value.get("instrument", {}).get("class", "") in class_key_list
198
+ }
199
199
 
200
200
 
201
201
  def site_parameters():
@@ -516,6 +516,8 @@ def get_site_from_array_element_name(array_element_name):
516
516
  Site name(s).
517
517
  """
518
518
  try: # e.g. instrument is 'North' as given for the site parameters
519
+ if array_element_name.startswith("OBS"):
520
+ return validate_site_name(array_element_name.split("-")[1])
519
521
  return validate_site_name(array_element_name)
520
522
  except ValueError: # e.g. instrument is 'LSTN' as given for the array element types
521
523
  return array_elements()[get_array_element_type_from_name(array_element_name)]["site"]
@@ -630,7 +632,6 @@ def get_simulation_software_name_from_parameter_name(
630
632
 
631
633
  def simtel_config_file_name(
632
634
  site,
633
- model_version,
634
635
  array_name=None,
635
636
  telescope_model_name=None,
636
637
  label=None,
@@ -645,8 +646,6 @@ def simtel_config_file_name(
645
646
  South or North.
646
647
  telescope_model_name: str
647
648
  LST-1, MST-FlashCam, ...
648
- model_version: str
649
- Version of the model.
650
649
  label: str
651
650
  Instance label.
652
651
  extra_label: str
@@ -661,7 +660,6 @@ def simtel_config_file_name(
661
660
  name += f"-{array_name}" if array_name is not None else ""
662
661
  name += f"-{site}"
663
662
  name += f"-{telescope_model_name}" if telescope_model_name is not None else ""
664
- name += f"-{model_version}"
665
663
  name += f"_{label}" if label is not None else ""
666
664
  name += f"_{extra_label}" if extra_label is not None else ""
667
665
  name += ".cfg"
simtools/version.py CHANGED
@@ -4,6 +4,8 @@
4
4
  # which is adapted from https://github.com/astropy/astropy/blob/master/astropy/version.py
5
5
  # see https://github.com/astropy/astropy/pull/10774 for a discussion on why this needed.
6
6
 
7
+ import re
8
+
7
9
  from packaging.specifiers import SpecifierSet
8
10
  from packaging.version import InvalidVersion, Version
9
11
 
@@ -192,6 +194,41 @@ def compare_versions(version_string_1, version_string_2, level=MAJOR_MINOR_PATCH
192
194
  return (ver1 > ver2) - (ver1 < ver2)
193
195
 
194
196
 
197
+ def is_valid_semantic_version(version_string, strict=True):
198
+ """
199
+ Check if a string is a valid semantic version.
200
+
201
+ Parameters
202
+ ----------
203
+ version_string : str
204
+ The version string to validate (e.g., "6.0.2", "1.0.0-alpha").
205
+ strict : bool, optional
206
+ If True, use PEP 440 validation (packaging.version.Version).
207
+ If False, use SemVer 2.0.0 regex pattern (allows more flexible pre-release identifiers).
208
+
209
+ Returns
210
+ -------
211
+ bool
212
+ True if the version string is valid, False otherwise.
213
+ """
214
+ if not version_string:
215
+ return False
216
+
217
+ if strict:
218
+ try:
219
+ Version(version_string)
220
+ return True
221
+ except InvalidVersion:
222
+ return False
223
+ else:
224
+ semver_regex = (
225
+ r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)" # major.minor.patch
226
+ r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?" # pre-release
227
+ r"(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$" # build metadata
228
+ )
229
+ return bool(re.match(semver_regex, version_string))
230
+
231
+
195
232
  def check_version_constraint(version_string, constraint):
196
233
  """
197
234
  Check if a version satisfies a constraint.