gammasimtools 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/METADATA +2 -1
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/RECORD +134 -130
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/entry_points.txt +3 -1
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/licenses/LICENSE +1 -1
- simtools/_version.py +2 -2
- simtools/application_control.py +78 -0
- simtools/applications/calculate_incident_angles.py +0 -2
- simtools/applications/convert_geo_coordinates_of_array_elements.py +1 -2
- simtools/applications/db_add_file_to_db.py +1 -1
- simtools/applications/db_add_simulation_model_from_repository_to_db.py +1 -1
- simtools/applications/db_add_value_from_json_to_db.py +1 -1
- simtools/applications/db_generate_compound_indexes.py +1 -1
- simtools/applications/db_get_array_layouts_from_db.py +2 -6
- simtools/applications/db_get_file_from_db.py +1 -1
- simtools/applications/db_get_parameter_from_db.py +1 -1
- simtools/applications/db_inspect_databases.py +1 -1
- simtools/applications/db_upload_model_repository.py +1 -1
- simtools/applications/derive_ctao_array_layouts.py +1 -2
- simtools/applications/derive_mirror_rnda.py +1 -3
- simtools/applications/derive_psf_parameters.py +5 -1
- simtools/applications/derive_pulse_shape_parameters.py +194 -0
- simtools/applications/derive_trigger_rates.py +1 -1
- simtools/applications/docs_produce_array_element_report.py +2 -8
- simtools/applications/docs_produce_calibration_reports.py +1 -3
- simtools/applications/docs_produce_model_parameter_reports.py +0 -2
- simtools/applications/docs_produce_simulation_configuration_report.py +1 -3
- simtools/applications/generate_array_config.py +0 -1
- simtools/applications/generate_corsika_histograms.py +48 -235
- simtools/applications/generate_regular_arrays.py +5 -35
- simtools/applications/generate_simtel_event_data.py +2 -2
- simtools/applications/maintain_simulation_model_add_production.py +2 -2
- simtools/applications/maintain_simulation_model_write_array_element_positions.py +87 -0
- simtools/applications/plot_array_layout.py +64 -108
- simtools/applications/plot_simulated_event_distributions.py +57 -0
- simtools/applications/plot_tabular_data.py +0 -1
- simtools/applications/plot_tabular_data_for_model_parameter.py +1 -6
- simtools/applications/production_derive_corsika_limits.py +1 -1
- simtools/applications/production_generate_grid.py +0 -1
- simtools/applications/run_application.py +1 -1
- simtools/applications/simulate_flasher.py +3 -4
- simtools/applications/simulate_illuminator.py +0 -1
- simtools/applications/simulate_pedestals.py +2 -6
- simtools/applications/simulate_prod.py +9 -28
- simtools/applications/simulate_prod_htcondor_generator.py +8 -1
- simtools/applications/submit_array_layouts.py +7 -7
- simtools/applications/submit_model_parameter_from_external.py +1 -3
- simtools/applications/validate_camera_efficiency.py +0 -1
- simtools/applications/validate_camera_fov.py +0 -1
- simtools/applications/validate_cumulative_psf.py +0 -2
- simtools/applications/validate_file_using_schema.py +49 -123
- simtools/applications/validate_optics.py +0 -13
- simtools/camera/camera_efficiency.py +1 -6
- simtools/camera/single_photon_electron_spectrum.py +2 -1
- simtools/configuration/commandline_parser.py +43 -8
- simtools/configuration/configurator.py +6 -11
- simtools/corsika/corsika_config.py +204 -99
- simtools/corsika/corsika_histograms.py +411 -1735
- simtools/corsika/primary_particle.py +1 -1
- simtools/data_model/metadata_collector.py +5 -2
- simtools/data_model/metadata_model.py +0 -4
- simtools/data_model/model_data_writer.py +27 -17
- simtools/data_model/schema.py +112 -5
- simtools/data_model/validate_data.py +80 -48
- simtools/db/db_handler.py +19 -8
- simtools/db/db_model_upload.py +2 -1
- simtools/db/mongo_db.py +133 -42
- simtools/dependencies.py +83 -44
- simtools/io/ascii_handler.py +4 -2
- simtools/io/table_handler.py +1 -1
- simtools/job_execution/htcondor_script_generator.py +0 -2
- simtools/layout/array_layout.py +4 -12
- simtools/layout/array_layout_utils.py +227 -58
- simtools/model/array_model.py +37 -18
- simtools/model/calibration_model.py +0 -4
- simtools/model/legacy_model_parameter.py +134 -0
- simtools/model/model_parameter.py +24 -14
- simtools/model/model_repository.py +18 -5
- simtools/model/model_utils.py +1 -6
- simtools/model/site_model.py +0 -4
- simtools/model/telescope_model.py +6 -11
- simtools/production_configuration/derive_corsika_limits.py +6 -11
- simtools/production_configuration/interpolation_handler.py +16 -16
- simtools/ray_tracing/incident_angles.py +5 -11
- simtools/ray_tracing/mirror_panel_psf.py +3 -7
- simtools/ray_tracing/psf_analysis.py +29 -27
- simtools/ray_tracing/psf_parameter_optimisation.py +822 -680
- simtools/ray_tracing/ray_tracing.py +6 -15
- simtools/reporting/docs_auto_report_generator.py +8 -13
- simtools/reporting/docs_read_parameters.py +70 -16
- simtools/runners/corsika_runner.py +15 -10
- simtools/runners/corsika_simtel_runner.py +9 -8
- simtools/runners/runner_services.py +17 -7
- simtools/runners/simtel_runner.py +11 -58
- simtools/runners/simtools_runner.py +2 -4
- simtools/schemas/model_parameters/flasher_pulse_exp_decay.schema.yml +2 -0
- simtools/schemas/model_parameters/flasher_pulse_shape.schema.yml +50 -0
- simtools/schemas/model_parameters/flasher_pulse_width.schema.yml +2 -0
- simtools/schemas/simulation_models_info.schema.yml +2 -0
- simtools/settings.py +154 -0
- simtools/sim_events/file_info.py +128 -0
- simtools/{simtel/simtel_io_event_histograms.py → sim_events/histograms.py} +25 -15
- simtools/{simtel/simtel_io_event_reader.py → sim_events/reader.py} +20 -17
- simtools/{simtel/simtel_io_event_writer.py → sim_events/writer.py} +84 -25
- simtools/simtel/pulse_shapes.py +273 -0
- simtools/simtel/simtel_config_writer.py +146 -22
- simtools/simtel/simtel_table_reader.py +6 -4
- simtools/simtel/simulator_array.py +62 -23
- simtools/simtel/simulator_camera_efficiency.py +4 -6
- simtools/simtel/simulator_light_emission.py +101 -19
- simtools/simtel/simulator_ray_tracing.py +4 -10
- simtools/simulator.py +360 -353
- simtools/telescope_trigger_rates.py +3 -4
- simtools/testing/assertions.py +115 -8
- simtools/testing/configuration.py +2 -3
- simtools/testing/helpers.py +2 -3
- simtools/testing/log_inspector.py +5 -1
- simtools/testing/sim_telarray_metadata.py +1 -1
- simtools/testing/validate_output.py +69 -23
- simtools/utils/general.py +37 -0
- simtools/utils/geometry.py +0 -77
- simtools/utils/names.py +7 -9
- simtools/version.py +37 -0
- simtools/visualization/legend_handlers.py +21 -10
- simtools/visualization/plot_array_layout.py +312 -41
- simtools/visualization/plot_corsika_histograms.py +143 -605
- simtools/visualization/plot_mirrors.py +834 -0
- simtools/visualization/plot_pixels.py +2 -4
- simtools/visualization/plot_psf.py +0 -1
- simtools/visualization/plot_simtel_event_histograms.py +4 -4
- simtools/visualization/plot_simtel_events.py +6 -11
- simtools/visualization/plot_tables.py +8 -19
- simtools/visualization/visualize.py +22 -2
- simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +0 -160
- simtools/applications/print_version.py +0 -53
- simtools/io/hdf5_handler.py +0 -139
- simtools/simtel/simtel_io_file_info.py +0 -62
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/WHEEL +0 -0
- {gammasimtools-0.24.0.dist-info → gammasimtools-0.26.0.dist-info}/top_level.txt +0 -0
|
@@ -8,13 +8,13 @@ from ctao_cr_spectra.definitions import IRFDOC_PROTON_SPECTRUM
|
|
|
8
8
|
|
|
9
9
|
from simtools.io import ascii_handler, io_handler
|
|
10
10
|
from simtools.layout.array_layout_utils import get_array_elements_from_db_for_layouts
|
|
11
|
-
from simtools.
|
|
11
|
+
from simtools.sim_events.histograms import EventDataHistograms
|
|
12
12
|
from simtools.visualization import plot_simtel_event_histograms
|
|
13
13
|
|
|
14
14
|
_logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def telescope_trigger_rates(args_dict
|
|
17
|
+
def telescope_trigger_rates(args_dict):
|
|
18
18
|
"""
|
|
19
19
|
Calculate trigger rates for single telescopes or arrays of telescopes.
|
|
20
20
|
|
|
@@ -27,7 +27,6 @@ def telescope_trigger_rates(args_dict, db_config):
|
|
|
27
27
|
args_dict["array_layout_name"],
|
|
28
28
|
args_dict.get("site"),
|
|
29
29
|
args_dict.get("model_version"),
|
|
30
|
-
db_config,
|
|
31
30
|
)
|
|
32
31
|
else:
|
|
33
32
|
telescope_configs = ascii_handler.collect_data_from_file(args_dict["telescope_ids"])[
|
|
@@ -38,7 +37,7 @@ def telescope_trigger_rates(args_dict, db_config):
|
|
|
38
37
|
_logger.info(
|
|
39
38
|
f"Processing file: {args_dict['event_data_file']} with telescope config: {array_name}"
|
|
40
39
|
)
|
|
41
|
-
histograms =
|
|
40
|
+
histograms = EventDataHistograms(
|
|
42
41
|
args_dict["event_data_file"], array_name=array_name, telescope_list=telescope_ids
|
|
43
42
|
)
|
|
44
43
|
histograms.fill()
|
simtools/testing/assertions.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""Functions asserting certain conditions are met (used e.g., in integration tests)."""
|
|
2
2
|
|
|
3
|
+
import gzip
|
|
3
4
|
import json
|
|
4
5
|
import logging
|
|
6
|
+
import tarfile
|
|
5
7
|
from collections import defaultdict
|
|
6
8
|
from pathlib import Path
|
|
7
9
|
|
|
@@ -66,8 +68,7 @@ def assert_n_showers_and_energy_range(file):
|
|
|
66
68
|
simulation_config = {}
|
|
67
69
|
with SimTelFile(file, skip_non_triggered=False) as f:
|
|
68
70
|
simulation_config = f.mc_run_headers[0]
|
|
69
|
-
for event in f
|
|
70
|
-
simulated_energies.append(event["mc_shower"]["energy"])
|
|
71
|
+
simulated_energies.extend(event["mc_shower"]["energy"] for event in f)
|
|
71
72
|
|
|
72
73
|
# The relative tolerance is set to 1% because ~0.5% shower simulations do not
|
|
73
74
|
# succeed, without resulting in an error. This tolerance therefore is not an issue.
|
|
@@ -174,12 +175,6 @@ def check_output_from_sim_telarray(file, file_test):
|
|
|
174
175
|
_logger.debug(f"No expected output or metadata provided, skipping checks {file_test}")
|
|
175
176
|
return True
|
|
176
177
|
|
|
177
|
-
if file.suffix != ".zst":
|
|
178
|
-
raise ValueError(
|
|
179
|
-
f"Expected output file {file} is not a zstd compressed file "
|
|
180
|
-
f"(i.e., a sim_telarray file)."
|
|
181
|
-
)
|
|
182
|
-
|
|
183
178
|
assert_output = assert_metadata = True
|
|
184
179
|
|
|
185
180
|
if "expected_output" in file_test:
|
|
@@ -193,3 +188,115 @@ def check_output_from_sim_telarray(file, file_test):
|
|
|
193
188
|
)
|
|
194
189
|
|
|
195
190
|
return assert_n_showers_and_energy_range(file=file) and assert_output and assert_metadata
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def check_simulation_logs(tar_file, file_test):
|
|
194
|
+
"""
|
|
195
|
+
Check simulation logs for wanted and forbidden patterns.
|
|
196
|
+
|
|
197
|
+
Parameters
|
|
198
|
+
----------
|
|
199
|
+
tar_file : str
|
|
200
|
+
Path to the tar file.
|
|
201
|
+
file_test : dict
|
|
202
|
+
Dictionary with the test configuration.
|
|
203
|
+
|
|
204
|
+
Returns
|
|
205
|
+
-------
|
|
206
|
+
bool
|
|
207
|
+
True if the logs are correct.
|
|
208
|
+
"""
|
|
209
|
+
wanted, forbidden = _get_expected_patterns(file_test)
|
|
210
|
+
if wanted is None:
|
|
211
|
+
return True
|
|
212
|
+
|
|
213
|
+
if not tarfile.is_tarfile(tar_file):
|
|
214
|
+
raise ValueError(f"{tar_file} is not a tar file")
|
|
215
|
+
|
|
216
|
+
found_wanted = set()
|
|
217
|
+
found_forbidden = set()
|
|
218
|
+
with tarfile.open(tar_file, "r:*") as tar:
|
|
219
|
+
for member in tar.getmembers():
|
|
220
|
+
if not member.name.endswith(".log.gz"):
|
|
221
|
+
continue
|
|
222
|
+
_logger.info(f"Scanning {member.name}")
|
|
223
|
+
text = _read_log(member, tar)
|
|
224
|
+
found_wanted |= _find_patterns(text, wanted)
|
|
225
|
+
found_forbidden |= _find_patterns(text, forbidden)
|
|
226
|
+
|
|
227
|
+
return _validate_patterns(found_wanted, found_forbidden, wanted)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def check_plain_log(log_file, file_test):
|
|
231
|
+
"""
|
|
232
|
+
Check plain log file for wanted and forbidden patterns.
|
|
233
|
+
|
|
234
|
+
Parameters
|
|
235
|
+
----------
|
|
236
|
+
log_file : str
|
|
237
|
+
Path to the log file.
|
|
238
|
+
file_test : dict
|
|
239
|
+
Dictionary with the test configuration.
|
|
240
|
+
|
|
241
|
+
Returns
|
|
242
|
+
-------
|
|
243
|
+
bool
|
|
244
|
+
True if the logs are correct.
|
|
245
|
+
"""
|
|
246
|
+
wanted, forbidden = _get_expected_patterns(file_test)
|
|
247
|
+
if wanted is None:
|
|
248
|
+
return True
|
|
249
|
+
|
|
250
|
+
try:
|
|
251
|
+
with open(log_file, encoding="utf-8") as f:
|
|
252
|
+
text = f.read()
|
|
253
|
+
except FileNotFoundError:
|
|
254
|
+
_logger.error(f"Log file {log_file} not found")
|
|
255
|
+
return False
|
|
256
|
+
|
|
257
|
+
found = _find_patterns(text, wanted)
|
|
258
|
+
bad = _find_patterns(text, forbidden)
|
|
259
|
+
|
|
260
|
+
return _validate_patterns(found, bad, wanted)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _get_expected_patterns(file_test):
|
|
264
|
+
"""Get wanted and forbidden patterns from file test configuration."""
|
|
265
|
+
expected_log = file_test.get("expected_log_output")
|
|
266
|
+
if isinstance(expected_log, dict):
|
|
267
|
+
wanted = expected_log.get("pattern", [])
|
|
268
|
+
forbidden = expected_log.get("forbidden_pattern", [])
|
|
269
|
+
else:
|
|
270
|
+
wanted = file_test.get("pattern", [])
|
|
271
|
+
forbidden = file_test.get("forbidden_pattern", [])
|
|
272
|
+
if not (wanted or forbidden):
|
|
273
|
+
_logger.debug(f"No expected log output provided, skipping checks {file_test}")
|
|
274
|
+
return None, None
|
|
275
|
+
|
|
276
|
+
return wanted, forbidden
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _validate_patterns(found, bad, wanted):
|
|
280
|
+
"""Validate found patterns against wanted and forbidden ones."""
|
|
281
|
+
if bad:
|
|
282
|
+
_logger.error(f"Forbidden patterns found: {list(bad)}")
|
|
283
|
+
return False
|
|
284
|
+
missing = [p for p in wanted if p not in found]
|
|
285
|
+
if missing:
|
|
286
|
+
_logger.error(f"Missing expected patterns: {missing}")
|
|
287
|
+
return False
|
|
288
|
+
|
|
289
|
+
_logger.debug(f"All expected patterns found: {wanted}")
|
|
290
|
+
return True
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _find_patterns(text, patterns):
|
|
294
|
+
"""Find patterns in text (case insensitive)."""
|
|
295
|
+
text_lower = text.lower()
|
|
296
|
+
return {p for p in patterns if p.lower() in text_lower}
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _read_log(member, tar):
|
|
300
|
+
"""Read and decode a gzipped log file from a tar archive."""
|
|
301
|
+
with tar.extractfile(member) as gz, gzip.open(gz, "rb") as f:
|
|
302
|
+
return f.read().decode("utf-8", "ignore")
|
|
@@ -76,8 +76,7 @@ def _read_configs_from_files(config_files):
|
|
|
76
76
|
_dict = gen.remove_substring_recursively_from_dict(
|
|
77
77
|
ascii_handler.collect_data_from_file(file_name=config_file), substring="\n"
|
|
78
78
|
)
|
|
79
|
-
|
|
80
|
-
configs.append(application)
|
|
79
|
+
configs.extend(_dict.get("applications", []))
|
|
81
80
|
return configs
|
|
82
81
|
|
|
83
82
|
|
|
@@ -204,7 +203,7 @@ def _prepare_test_options(config, output_path, model_version=None):
|
|
|
204
203
|
if model_version and "model_version" in config:
|
|
205
204
|
config.update({"model_version": model_version})
|
|
206
205
|
|
|
207
|
-
for key in ["output_path", "
|
|
206
|
+
for key in ["output_path", "pack_for_grid_register"]:
|
|
208
207
|
if key in config:
|
|
209
208
|
config[key] = str(Path(output_path).joinpath(config[key]))
|
|
210
209
|
|
simtools/testing/helpers.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""Helper functions for integration testing."""
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
from pathlib import Path
|
|
3
|
+
from simtools import settings
|
|
5
4
|
|
|
6
5
|
|
|
7
6
|
def skip_camera_efficiency(config):
|
|
@@ -21,7 +20,7 @@ def _new_testeff_version():
|
|
|
21
20
|
|
|
22
21
|
This test checks if the new version is used.
|
|
23
22
|
"""
|
|
24
|
-
testeff_path =
|
|
23
|
+
testeff_path = settings.config.sim_telarray_path / "testeff.c"
|
|
25
24
|
try:
|
|
26
25
|
with open(testeff_path, encoding="utf-8") as file:
|
|
27
26
|
file_content = file.read()
|
|
@@ -15,7 +15,11 @@ ERROR_PATTERNS = [
|
|
|
15
15
|
re.compile(r"segmentation fault", re.IGNORECASE),
|
|
16
16
|
]
|
|
17
17
|
|
|
18
|
-
IGNORE_PATTERNS = [
|
|
18
|
+
IGNORE_PATTERNS = [
|
|
19
|
+
re.compile(r"Falling back to 'utf-8' with errors='ignore'", re.IGNORECASE),
|
|
20
|
+
re.compile(r"Failed to get user name[^\n]*setting it to UNKNOWN_USER", re.IGNORECASE),
|
|
21
|
+
re.compile(r"adjust_text::Error", re.IGNORECASE),
|
|
22
|
+
]
|
|
19
23
|
|
|
20
24
|
|
|
21
25
|
def inspect(log_text):
|
|
@@ -4,9 +4,9 @@ import logging
|
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
6
6
|
|
|
7
|
+
from simtools.sim_events.file_info import get_corsika_run_number
|
|
7
8
|
from simtools.simtel.simtel_config_reader import SimtelConfigReader
|
|
8
9
|
from simtools.simtel.simtel_config_writer import sim_telarray_random_seeds
|
|
9
|
-
from simtools.simtel.simtel_io_file_info import get_corsika_run_number
|
|
10
10
|
from simtools.simtel.simtel_io_metadata import (
|
|
11
11
|
get_sim_telarray_telescope_id,
|
|
12
12
|
read_sim_telarray_metadata,
|
|
@@ -14,9 +14,35 @@ from simtools.testing import assertions
|
|
|
14
14
|
_logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
def _versions_match(from_command_line, from_config_file):
|
|
18
|
+
"""Return True if validations should run for the given versions.
|
|
19
|
+
|
|
20
|
+
Behavior:
|
|
21
|
+
- If no version is provided from the command line, run validations.
|
|
22
|
+
- If a filter is provided from the command line, run only when it matches
|
|
23
|
+
the version(s) from the config file.
|
|
24
|
+
"""
|
|
25
|
+
if from_command_line is None:
|
|
26
|
+
return True
|
|
27
|
+
|
|
28
|
+
# Normalize to collections for comparison
|
|
29
|
+
cmd_versions = from_command_line if isinstance(from_command_line, list) else [from_command_line]
|
|
30
|
+
cfg_versions = from_config_file if isinstance(from_config_file, list) else [from_config_file]
|
|
31
|
+
|
|
32
|
+
# Consider a match if any overlap exists
|
|
33
|
+
return any(cv in cmd_versions for cv in cfg_versions)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# Keys to ignore when comparing sim_telarray configuration files
|
|
37
|
+
# (e.g., version numbers, system dependent parameters, CORSIKA options)
|
|
38
|
+
cfg_ignore_keys = [
|
|
39
|
+
"config_release",
|
|
40
|
+
"Label",
|
|
41
|
+
"simtools_", # ignore all simtools_ keys - version/build info dependence
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def validate_application_output(config, from_command_line=None, from_config_file=None):
|
|
20
46
|
"""
|
|
21
47
|
Validate application output against expected output.
|
|
22
48
|
|
|
@@ -39,9 +65,13 @@ def validate_application_output(
|
|
|
39
65
|
|
|
40
66
|
for integration_test in config["integration_tests"]:
|
|
41
67
|
_logger.info(f"Testing application output: {integration_test}")
|
|
68
|
+
_logger.debug(
|
|
69
|
+
f"Model version from command line: {from_command_line}, "
|
|
70
|
+
f"from config file: {from_config_file}"
|
|
71
|
+
)
|
|
42
72
|
|
|
43
|
-
if from_command_line
|
|
44
|
-
_validate_output_files(config, integration_test
|
|
73
|
+
if _versions_match(from_command_line, from_config_file):
|
|
74
|
+
_validate_output_files(config, integration_test)
|
|
45
75
|
|
|
46
76
|
if "file_type" in integration_test:
|
|
47
77
|
assert assertions.assert_file_type(
|
|
@@ -53,7 +83,7 @@ def validate_application_output(
|
|
|
53
83
|
_test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file)
|
|
54
84
|
|
|
55
85
|
|
|
56
|
-
def _validate_output_files(config, integration_test
|
|
86
|
+
def _validate_output_files(config, integration_test):
|
|
57
87
|
"""Validate output files."""
|
|
58
88
|
if "reference_output_file" in integration_test:
|
|
59
89
|
_validate_reference_output_file(config, integration_test)
|
|
@@ -65,11 +95,7 @@ def _validate_output_files(config, integration_test, db_config):
|
|
|
65
95
|
[{"path_descriptor": "output_path", "file": integration_test["output_file"]}],
|
|
66
96
|
)
|
|
67
97
|
if "model_parameter_validation" in integration_test:
|
|
68
|
-
_validate_model_parameter_json_file(
|
|
69
|
-
config,
|
|
70
|
-
integration_test["model_parameter_validation"],
|
|
71
|
-
db_config,
|
|
72
|
-
)
|
|
98
|
+
_validate_model_parameter_json_file(config, integration_test["model_parameter_validation"])
|
|
73
99
|
|
|
74
100
|
|
|
75
101
|
def _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file):
|
|
@@ -115,12 +141,20 @@ def _validate_output_path_and_file(config, integration_file_tests):
|
|
|
115
141
|
try:
|
|
116
142
|
assert output_file_path.exists()
|
|
117
143
|
except AssertionError as exc:
|
|
118
|
-
raise AssertionError(
|
|
144
|
+
raise AssertionError(
|
|
145
|
+
f"Output file {output_file_path} does not exist. "
|
|
146
|
+
f"Directory contents: {list(output_file_path.parent.iterdir())}"
|
|
147
|
+
) from exc
|
|
119
148
|
|
|
120
|
-
|
|
149
|
+
if output_file_path.name.endswith(".simtel.zst"):
|
|
150
|
+
assert assertions.check_output_from_sim_telarray(output_file_path, file_test)
|
|
151
|
+
elif output_file_path.name.endswith(".log_hist.tar.gz"):
|
|
152
|
+
assert assertions.check_simulation_logs(output_file_path, file_test)
|
|
153
|
+
elif output_file_path.suffix == ".log":
|
|
154
|
+
assert assertions.check_plain_log(output_file_path, file_test)
|
|
121
155
|
|
|
122
156
|
|
|
123
|
-
def _validate_model_parameter_json_file(config, model_parameter_validation
|
|
157
|
+
def _validate_model_parameter_json_file(config, model_parameter_validation):
|
|
124
158
|
"""
|
|
125
159
|
Validate model parameter json file and compare it with a reference parameter from the database.
|
|
126
160
|
|
|
@@ -135,7 +169,7 @@ def _validate_model_parameter_json_file(config, model_parameter_validation, db_c
|
|
|
135
169
|
|
|
136
170
|
"""
|
|
137
171
|
_logger.info(f"Checking model parameter json file: {model_parameter_validation}")
|
|
138
|
-
db = db_handler.DatabaseHandler(
|
|
172
|
+
db = db_handler.DatabaseHandler()
|
|
139
173
|
|
|
140
174
|
reference_parameter_name = model_parameter_validation.get("reference_parameter_name")
|
|
141
175
|
|
|
@@ -350,8 +384,8 @@ def _compare_simtel_cfg_files(reference_file, test_file):
|
|
|
350
384
|
Compare two sim_telarray configuration files.
|
|
351
385
|
|
|
352
386
|
Line-by-line string comparison. Requires similar sequence of
|
|
353
|
-
parameters in the files. Ignore lines
|
|
354
|
-
(
|
|
387
|
+
parameters in the files. Ignore lines listed in cfg_ignore_keys
|
|
388
|
+
(e.g., simtools package versions or hadronic interaction model strings).
|
|
355
389
|
|
|
356
390
|
Parameters
|
|
357
391
|
----------
|
|
@@ -370,16 +404,28 @@ def _compare_simtel_cfg_files(reference_file, test_file):
|
|
|
370
404
|
reference_cfg = [line.rstrip() for line in f1 if line.strip()]
|
|
371
405
|
test_cfg = [line.rstrip() for line in f2 if line.strip()]
|
|
372
406
|
|
|
373
|
-
|
|
407
|
+
def filter_ignored(cfg_lines, file_label):
|
|
408
|
+
filtered = []
|
|
409
|
+
for line in cfg_lines:
|
|
410
|
+
ignored_key = next((ignore for ignore in cfg_ignore_keys if ignore in line), None)
|
|
411
|
+
if ignored_key:
|
|
412
|
+
_logger.debug(f"Ignoring line in {file_label} due to key '{ignored_key}': {line}")
|
|
413
|
+
continue
|
|
414
|
+
filtered.append(line)
|
|
415
|
+
return filtered
|
|
416
|
+
|
|
417
|
+
reference_cfg_filtered = filter_ignored(reference_cfg, "reference file")
|
|
418
|
+
test_cfg_filtered = filter_ignored(test_cfg, "test file")
|
|
419
|
+
|
|
420
|
+
if len(reference_cfg_filtered) != len(test_cfg_filtered):
|
|
374
421
|
_logger.error(
|
|
375
|
-
f"Line counts differ: {reference_file}
|
|
376
|
-
f"
|
|
422
|
+
f"Line counts differ after filtering: {reference_file} "
|
|
423
|
+
f"({len(reference_cfg_filtered)} lines), "
|
|
424
|
+
f"{test_file} ({len(test_cfg_filtered)} lines)."
|
|
377
425
|
)
|
|
378
426
|
return False
|
|
379
427
|
|
|
380
|
-
for ref_line, test_line in zip(
|
|
381
|
-
if any(ignore in ref_line for ignore in ("config_release", "Label", "simtools_version")):
|
|
382
|
-
continue
|
|
428
|
+
for ref_line, test_line in zip(reference_cfg_filtered, test_cfg_filtered):
|
|
383
429
|
if ref_line != test_line:
|
|
384
430
|
_logger.error(
|
|
385
431
|
f"Configuration files {reference_file} and {test_file} do not match: "
|
simtools/utils/general.py
CHANGED
|
@@ -11,6 +11,7 @@ import urllib.request
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from urllib.parse import urlparse
|
|
13
13
|
|
|
14
|
+
import dotenv
|
|
14
15
|
import numpy as np
|
|
15
16
|
|
|
16
17
|
_logger = logging.getLogger(__name__)
|
|
@@ -850,3 +851,39 @@ def get_list_of_files_from_command_line(file_names, suffix_list):
|
|
|
850
851
|
def now_date_time_in_isoformat():
|
|
851
852
|
"""Return date and time in isoformat and second accuracy."""
|
|
852
853
|
return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def load_environment_variables(env_file=".env", env_list=None):
|
|
857
|
+
"""
|
|
858
|
+
Load environment variables (from a .env file or directly from the environment).
|
|
859
|
+
|
|
860
|
+
Allow to read a specific list of variables or all variables from the .env file.
|
|
861
|
+
|
|
862
|
+
Parameters
|
|
863
|
+
----------
|
|
864
|
+
env_file: str
|
|
865
|
+
Path to the .env file.
|
|
866
|
+
env_list: list, optional
|
|
867
|
+
List of environment variables to be read. If None, all variables are read.
|
|
868
|
+
|
|
869
|
+
Returns
|
|
870
|
+
-------
|
|
871
|
+
dict
|
|
872
|
+
Dictionary mapping environment variable names (lowercase, without the
|
|
873
|
+
``SIMTOOLS_`` prefix) to their cleaned string values.
|
|
874
|
+
"""
|
|
875
|
+
dotenv.load_dotenv(env_file or None)
|
|
876
|
+
keys = (
|
|
877
|
+
list(dotenv.dotenv_values(env_file).keys())
|
|
878
|
+
if env_list is None
|
|
879
|
+
else [f"SIMTOOLS_{s.upper()}" for s in env_list]
|
|
880
|
+
)
|
|
881
|
+
|
|
882
|
+
env_values = {}
|
|
883
|
+
for key in keys:
|
|
884
|
+
env_value = os.environ.get(key)
|
|
885
|
+
if env_value is None:
|
|
886
|
+
continue
|
|
887
|
+
cleaned_value = env_value.split("#")[0].strip().replace('"', "").replace("'", "")
|
|
888
|
+
env_values[key.removeprefix("SIMTOOLS_").lower()] = cleaned_value
|
|
889
|
+
return env_values
|
simtools/utils/geometry.py
CHANGED
|
@@ -1,88 +1,11 @@
|
|
|
1
1
|
"""A collection of functions related to geometrical transformations."""
|
|
2
2
|
|
|
3
|
-
import logging
|
|
4
3
|
import math
|
|
5
4
|
|
|
6
5
|
import astropy.units as u
|
|
7
6
|
import numpy as np
|
|
8
7
|
from astropy.units import UnitsError
|
|
9
8
|
|
|
10
|
-
_logger = logging.getLogger(__name__)
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def convert_2d_to_radial_distr(hist_2d, xaxis, yaxis, bins=50, max_dist=1000):
|
|
14
|
-
"""
|
|
15
|
-
Convert a 2d histogram of positions, e.g. photon positions on the ground, to a 1D distribution.
|
|
16
|
-
|
|
17
|
-
Parameters
|
|
18
|
-
----------
|
|
19
|
-
hist_2d: numpy.ndarray
|
|
20
|
-
The histogram counts.
|
|
21
|
-
xaxis: numpy.array
|
|
22
|
-
The values of the x axis (histogram bin edges) on the ground.
|
|
23
|
-
yaxis: numpy.array
|
|
24
|
-
The values of the y axis (histogram bin edges) on the ground.
|
|
25
|
-
bins: float
|
|
26
|
-
Number of bins in distance.
|
|
27
|
-
max_dist: float
|
|
28
|
-
Maximum distance to consider in the 1D histogram, usually in meters.
|
|
29
|
-
|
|
30
|
-
Returns
|
|
31
|
-
-------
|
|
32
|
-
np.array
|
|
33
|
-
The values of the 1D histogram with size = int(max_dist/bin_size).
|
|
34
|
-
np.array
|
|
35
|
-
The bin edges of the 1D histogram with size = int(max_dist/bin_size) + 1.
|
|
36
|
-
|
|
37
|
-
"""
|
|
38
|
-
# Check if the histogram will make sense
|
|
39
|
-
bins_step = 2 * max_dist / bins # in the 2d array, the positive and negative direction count.
|
|
40
|
-
for axis in [xaxis, yaxis]:
|
|
41
|
-
if (bins_step < np.diff(axis)).any():
|
|
42
|
-
msg = (
|
|
43
|
-
f"The histogram with number of bins {bins} and maximum distance of {max_dist} "
|
|
44
|
-
f"resulted in a bin size smaller than the original array. Please adjust those "
|
|
45
|
-
f"parameters to increase the bin size and avoid nan in the histogram values."
|
|
46
|
-
)
|
|
47
|
-
_logger.warning(msg)
|
|
48
|
-
break
|
|
49
|
-
|
|
50
|
-
grid_2d_x, grid_2d_y = np.meshgrid(xaxis[:-1], yaxis[:-1]) # [:-1], since xaxis and yaxis are
|
|
51
|
-
# the hist bin_edges (n + 1).
|
|
52
|
-
# radial_distance_map maps the distance to the center from each element in a square matrix.
|
|
53
|
-
radial_distance_map = np.sqrt(grid_2d_x**2 + grid_2d_y**2)
|
|
54
|
-
# The sorting and unravel_index give us the two indices for the position of the sorted element
|
|
55
|
-
# in the original 2d matrix
|
|
56
|
-
sorted_indices = np.unravel_index(
|
|
57
|
-
np.argsort(radial_distance_map, axis=None), np.shape(radial_distance_map)
|
|
58
|
-
)
|
|
59
|
-
x_indices_sorted, y_indices_sorted = sorted_indices[0], sorted_indices[1]
|
|
60
|
-
|
|
61
|
-
# We construct a 1D array with the histogram counts sorted according to the distance to the
|
|
62
|
-
# center.
|
|
63
|
-
hist_sorted = np.array(
|
|
64
|
-
[hist_2d[i_x, i_y] for i_x, i_y in zip(x_indices_sorted, y_indices_sorted)]
|
|
65
|
-
)
|
|
66
|
-
distance_sorted = np.sort(radial_distance_map, axis=None)
|
|
67
|
-
|
|
68
|
-
# For larger distances, we have more elements in a slice 'dr' in radius, hence, we need to
|
|
69
|
-
# account for it using weights below.
|
|
70
|
-
|
|
71
|
-
weights, radial_bin_edges = np.histogram(distance_sorted, bins=bins, range=(0, max_dist))
|
|
72
|
-
histogram_1d = np.empty_like(weights, dtype=float)
|
|
73
|
-
|
|
74
|
-
for i_radial, _ in enumerate(radial_bin_edges[:-1]):
|
|
75
|
-
# Here we sum all the events within a radial interval 'dr' and then divide by the number of
|
|
76
|
-
# bins that fit this interval.
|
|
77
|
-
indices_to_sum = (distance_sorted >= radial_bin_edges[i_radial]) * (
|
|
78
|
-
distance_sorted < radial_bin_edges[i_radial + 1]
|
|
79
|
-
)
|
|
80
|
-
if weights[i_radial] != 0:
|
|
81
|
-
histogram_1d[i_radial] = np.sum(hist_sorted[indices_to_sum]) / weights[i_radial]
|
|
82
|
-
else:
|
|
83
|
-
histogram_1d[i_radial] = 0
|
|
84
|
-
return histogram_1d, radial_bin_edges
|
|
85
|
-
|
|
86
9
|
|
|
87
10
|
@u.quantity_input(rotation_angle_phi=u.rad, rotation_angle_theta=u.rad)
|
|
88
11
|
def rotate(x, y, rotation_around_z_axis, rotation_around_y_axis=0):
|
simtools/utils/names.py
CHANGED
|
@@ -189,13 +189,13 @@ def model_parameters(class_key_list=None):
|
|
|
189
189
|
dict
|
|
190
190
|
Model parameters definitions.
|
|
191
191
|
"""
|
|
192
|
-
_parameters = {}
|
|
193
192
|
if class_key_list is None:
|
|
194
193
|
return _load_model_parameters()
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
194
|
+
return {
|
|
195
|
+
key: value
|
|
196
|
+
for key, value in _load_model_parameters().items()
|
|
197
|
+
if value.get("instrument", {}).get("class", "") in class_key_list
|
|
198
|
+
}
|
|
199
199
|
|
|
200
200
|
|
|
201
201
|
def site_parameters():
|
|
@@ -516,6 +516,8 @@ def get_site_from_array_element_name(array_element_name):
|
|
|
516
516
|
Site name(s).
|
|
517
517
|
"""
|
|
518
518
|
try: # e.g. instrument is 'North' as given for the site parameters
|
|
519
|
+
if array_element_name.startswith("OBS"):
|
|
520
|
+
return validate_site_name(array_element_name.split("-")[1])
|
|
519
521
|
return validate_site_name(array_element_name)
|
|
520
522
|
except ValueError: # e.g. instrument is 'LSTN' as given for the array element types
|
|
521
523
|
return array_elements()[get_array_element_type_from_name(array_element_name)]["site"]
|
|
@@ -630,7 +632,6 @@ def get_simulation_software_name_from_parameter_name(
|
|
|
630
632
|
|
|
631
633
|
def simtel_config_file_name(
|
|
632
634
|
site,
|
|
633
|
-
model_version,
|
|
634
635
|
array_name=None,
|
|
635
636
|
telescope_model_name=None,
|
|
636
637
|
label=None,
|
|
@@ -645,8 +646,6 @@ def simtel_config_file_name(
|
|
|
645
646
|
South or North.
|
|
646
647
|
telescope_model_name: str
|
|
647
648
|
LST-1, MST-FlashCam, ...
|
|
648
|
-
model_version: str
|
|
649
|
-
Version of the model.
|
|
650
649
|
label: str
|
|
651
650
|
Instance label.
|
|
652
651
|
extra_label: str
|
|
@@ -661,7 +660,6 @@ def simtel_config_file_name(
|
|
|
661
660
|
name += f"-{array_name}" if array_name is not None else ""
|
|
662
661
|
name += f"-{site}"
|
|
663
662
|
name += f"-{telescope_model_name}" if telescope_model_name is not None else ""
|
|
664
|
-
name += f"-{model_version}"
|
|
665
663
|
name += f"_{label}" if label is not None else ""
|
|
666
664
|
name += f"_{extra_label}" if extra_label is not None else ""
|
|
667
665
|
name += ".cfg"
|
simtools/version.py
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
# which is adapted from https://github.com/astropy/astropy/blob/master/astropy/version.py
|
|
5
5
|
# see https://github.com/astropy/astropy/pull/10774 for a discussion on why this needed.
|
|
6
6
|
|
|
7
|
+
import re
|
|
8
|
+
|
|
7
9
|
from packaging.specifiers import SpecifierSet
|
|
8
10
|
from packaging.version import InvalidVersion, Version
|
|
9
11
|
|
|
@@ -192,6 +194,41 @@ def compare_versions(version_string_1, version_string_2, level=MAJOR_MINOR_PATCH
|
|
|
192
194
|
return (ver1 > ver2) - (ver1 < ver2)
|
|
193
195
|
|
|
194
196
|
|
|
197
|
+
def is_valid_semantic_version(version_string, strict=True):
|
|
198
|
+
"""
|
|
199
|
+
Check if a string is a valid semantic version.
|
|
200
|
+
|
|
201
|
+
Parameters
|
|
202
|
+
----------
|
|
203
|
+
version_string : str
|
|
204
|
+
The version string to validate (e.g., "6.0.2", "1.0.0-alpha").
|
|
205
|
+
strict : bool, optional
|
|
206
|
+
If True, use PEP 440 validation (packaging.version.Version).
|
|
207
|
+
If False, use SemVer 2.0.0 regex pattern (allows more flexible pre-release identifiers).
|
|
208
|
+
|
|
209
|
+
Returns
|
|
210
|
+
-------
|
|
211
|
+
bool
|
|
212
|
+
True if the version string is valid, False otherwise.
|
|
213
|
+
"""
|
|
214
|
+
if not version_string:
|
|
215
|
+
return False
|
|
216
|
+
|
|
217
|
+
if strict:
|
|
218
|
+
try:
|
|
219
|
+
Version(version_string)
|
|
220
|
+
return True
|
|
221
|
+
except InvalidVersion:
|
|
222
|
+
return False
|
|
223
|
+
else:
|
|
224
|
+
semver_regex = (
|
|
225
|
+
r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)" # major.minor.patch
|
|
226
|
+
r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?" # pre-release
|
|
227
|
+
r"(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$" # build metadata
|
|
228
|
+
)
|
|
229
|
+
return bool(re.match(semver_regex, version_string))
|
|
230
|
+
|
|
231
|
+
|
|
195
232
|
def check_version_constraint(version_string, constraint):
|
|
196
233
|
"""
|
|
197
234
|
Check if a version satisfies a constraint.
|