gammasimtools 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/METADATA +2 -2
- {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/RECORD +94 -85
- {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/entry_points.txt +2 -1
- simtools/_version.py +2 -2
- simtools/applications/calculate_trigger_rate.py +15 -38
- simtools/applications/convert_all_model_parameters_from_simtel.py +9 -28
- simtools/applications/convert_geo_coordinates_of_array_elements.py +47 -45
- simtools/applications/convert_model_parameter_from_simtel.py +2 -2
- simtools/applications/db_add_file_to_db.py +1 -2
- simtools/applications/db_add_simulation_model_from_repository_to_db.py +110 -0
- simtools/applications/db_add_value_from_json_to_db.py +1 -2
- simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +6 -6
- simtools/applications/db_get_file_from_db.py +11 -12
- simtools/applications/db_get_parameter_from_db.py +44 -32
- simtools/applications/derive_photon_electron_spectrum.py +99 -0
- simtools/applications/generate_array_config.py +17 -17
- simtools/applications/generate_regular_arrays.py +15 -15
- simtools/applications/generate_simtel_array_histograms.py +11 -48
- simtools/applications/production_generate_simulation_config.py +25 -7
- simtools/applications/production_scale_events.py +2 -2
- simtools/applications/simulate_prod.py +1 -1
- simtools/applications/simulate_prod_htcondor_generator.py +26 -26
- simtools/applications/submit_data_from_external.py +12 -4
- simtools/applications/submit_model_parameter_from_external.py +8 -6
- simtools/applications/validate_camera_efficiency.py +2 -2
- simtools/applications/validate_file_using_schema.py +23 -19
- simtools/camera/single_photon_electron_spectrum.py +168 -0
- simtools/configuration/commandline_parser.py +8 -1
- simtools/constants.py +10 -3
- simtools/corsika/corsika_config.py +8 -7
- simtools/corsika/corsika_histograms.py +1 -1
- simtools/data_model/data_reader.py +0 -3
- simtools/data_model/metadata_collector.py +3 -4
- simtools/data_model/metadata_model.py +8 -124
- simtools/data_model/model_data_writer.py +17 -63
- simtools/data_model/schema.py +213 -0
- simtools/data_model/validate_data.py +9 -44
- simtools/db/db_handler.py +323 -495
- simtools/db/db_model_upload.py +139 -0
- simtools/io_operations/hdf5_handler.py +54 -24
- simtools/layout/array_layout.py +33 -28
- simtools/model/array_model.py +13 -7
- simtools/model/model_parameter.py +22 -54
- simtools/model/site_model.py +2 -2
- simtools/production_configuration/calculate_statistical_errors_grid_point.py +119 -144
- simtools/production_configuration/event_scaler.py +7 -17
- simtools/production_configuration/generate_simulation_config.py +5 -32
- simtools/production_configuration/interpolation_handler.py +8 -11
- simtools/runners/corsika_simtel_runner.py +3 -1
- simtools/schemas/input/MST_mirror_2f_measurements.schema.yml +39 -0
- simtools/schemas/input/single_pe_spectrum.schema.yml +38 -0
- simtools/schemas/integration_tests_config.metaschema.yml +10 -0
- simtools/schemas/model_parameter.metaschema.yml +7 -2
- simtools/schemas/model_parameter_and_data_schema.metaschema.yml +2 -0
- simtools/schemas/model_parameters/array_element_position_utm.schema.yml +1 -1
- simtools/schemas/model_parameters/array_window.schema.yml +37 -0
- simtools/schemas/model_parameters/asum_clipping.schema.yml +0 -4
- simtools/schemas/model_parameters/channels_per_chip.schema.yml +1 -1
- simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml +2 -2
- simtools/schemas/model_parameters/dsum_clipping.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_offset.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_pedsub.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_prescale.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_presum_max.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_shaping.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_threshold.schema.yml +0 -2
- simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +0 -2
- simtools/schemas/model_parameters/fadc_compensate_pedestal.schema.yml +1 -1
- simtools/schemas/model_parameters/fadc_lg_compensate_pedestal.schema.yml +1 -1
- simtools/schemas/model_parameters/fadc_noise.schema.yml +3 -3
- simtools/schemas/model_parameters/fake_mirror_list.schema.yml +33 -0
- simtools/schemas/model_parameters/laser_photons.schema.yml +2 -2
- simtools/schemas/model_parameters/secondary_mirror_degraded_reflection.schema.yml +1 -1
- simtools/schemas/production_configuration_metrics.schema.yml +68 -0
- simtools/schemas/production_tables.schema.yml +41 -0
- simtools/simtel/simtel_config_writer.py +5 -6
- simtools/simtel/simtel_io_histogram.py +32 -67
- simtools/simtel/simtel_io_histograms.py +15 -30
- simtools/simtel/simulator_array.py +2 -1
- simtools/simtel/simulator_camera_efficiency.py +5 -0
- simtools/simtel/simulator_light_emission.py +3 -1
- simtools/simtel/simulator_ray_tracing.py +2 -1
- simtools/testing/helpers.py +6 -13
- simtools/testing/validate_output.py +131 -47
- simtools/utils/general.py +102 -12
- simtools/utils/names.py +24 -20
- simtools/applications/db_add_model_parameters_from_repository_to_db.py +0 -176
- simtools/db/db_array_elements.py +0 -130
- {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/LICENSE +0 -0
- {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/WHEEL +0 -0
- {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/top_level.txt +0 -0
- /simtools/{camera_efficiency.py → camera/camera_efficiency.py} +0 -0
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
#!/usr/bin/python3
|
|
2
2
|
|
|
3
3
|
"""
|
|
4
|
-
|
|
4
|
+
Make a regular array of telescopes and save it as astropy table.
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
The arrays consist of one telescope at the center of the array and or of 4 telescopes
|
|
7
|
+
in a square grid. These arrays are used for trigger rate simulations.
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
The array layout files created will be available at the data/layout directory.
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
Command line arguments
|
|
12
|
+
----------------------
|
|
13
|
+
site (str, required)
|
|
14
|
+
observatory site (e.g., North or South).
|
|
15
|
+
model_version (str, optional)
|
|
16
|
+
Model version to use (e.g., 6.0.0). If not provided, the latest version is used.
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
Example
|
|
19
|
+
-------
|
|
20
|
+
Runtime < 10 s.
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
.. code-block:: console
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
simtools-generate-regular-arrays --site=North
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
27
|
import logging
|
|
@@ -13,8 +13,6 @@ r"""
|
|
|
13
13
|
Name of the histogram files to be plotted.
|
|
14
14
|
It can be given as the histogram file names (more than one option allowed) or as a text
|
|
15
15
|
file with the names of the histogram files in it.
|
|
16
|
-
figure_name (str, required)
|
|
17
|
-
File name for the pdf output (without extension).
|
|
18
16
|
pdf (bool, optional)
|
|
19
17
|
If set, histograms are saved into pdf files.
|
|
20
18
|
One pdf file contains all the histograms found in the file.
|
|
@@ -29,6 +27,8 @@ r"""
|
|
|
29
27
|
If the output output_file_name.hdf5 file already exists and hdf5 is set, the tables
|
|
30
28
|
associated to hdf5 will be overwritten. The remaining tables, if any, will stay
|
|
31
29
|
untouched.
|
|
30
|
+
test: bool
|
|
31
|
+
Test option. Generate only two histograms for testing purposes.
|
|
32
32
|
|
|
33
33
|
Raises
|
|
34
34
|
------
|
|
@@ -108,38 +108,6 @@ def _parse(label, description):
|
|
|
108
108
|
return config_parser
|
|
109
109
|
|
|
110
110
|
|
|
111
|
-
def build_histogram_files(config_parser, logger):
|
|
112
|
-
"""
|
|
113
|
-
Build a list of histogram files from command line arguments.
|
|
114
|
-
|
|
115
|
-
Parameters
|
|
116
|
-
----------
|
|
117
|
-
config_parser: dict
|
|
118
|
-
Parsed command line arguments.
|
|
119
|
-
logger: logging.Logger
|
|
120
|
-
Logger object for logging messages.
|
|
121
|
-
|
|
122
|
-
Returns
|
|
123
|
-
-------
|
|
124
|
-
list
|
|
125
|
-
List of histogram file paths.
|
|
126
|
-
"""
|
|
127
|
-
histogram_files = []
|
|
128
|
-
for one_file in config_parser["hist_file_names"]:
|
|
129
|
-
try:
|
|
130
|
-
if Path(one_file).suffix in [".zst", ".simtel", ".hdata"]:
|
|
131
|
-
histogram_files.append(one_file)
|
|
132
|
-
else:
|
|
133
|
-
with open(one_file, encoding="utf-8") as file:
|
|
134
|
-
for line in file:
|
|
135
|
-
histogram_files.append(line.strip())
|
|
136
|
-
except FileNotFoundError as exc:
|
|
137
|
-
msg = f"{one_file} is not a file."
|
|
138
|
-
logger.error(msg)
|
|
139
|
-
raise FileNotFoundError from exc
|
|
140
|
-
return histogram_files
|
|
141
|
-
|
|
142
|
-
|
|
143
111
|
def check_and_log_overwrite(config_parser, logger):
|
|
144
112
|
"""
|
|
145
113
|
Check if the output hdf5 file already exists and log a warning if it does.
|
|
@@ -197,36 +165,31 @@ def create_pdf(simtel_histograms, output_file_name, config_parser, logger):
|
|
|
197
165
|
logger.info(f"Wrote histograms to the pdf file {output_file_name}.pdf")
|
|
198
166
|
|
|
199
167
|
|
|
200
|
-
def export_to_hdf5(simtel_histograms, output_file_name, overwrite, config_parser, logger):
|
|
201
|
-
"""Export histograms to an HDF5 file."""
|
|
202
|
-
if config_parser["hdf5"]:
|
|
203
|
-
logger.info(f"Wrote histograms to the hdf5 file {output_file_name}.hdf5")
|
|
204
|
-
simtel_histograms.export_histograms(f"{output_file_name}.hdf5", overwrite=overwrite)
|
|
205
|
-
|
|
206
|
-
|
|
207
168
|
def main(): # noqa: D103
|
|
208
169
|
label = Path(__file__).stem
|
|
209
|
-
description = "Display
|
|
170
|
+
description = "Display simtel_array histograms and/or write them into hdf5 format."
|
|
210
171
|
io_handler_instance = io_handler.IOHandler()
|
|
211
172
|
config_parser = _parse(label, description)
|
|
212
173
|
output_path = io_handler_instance.get_output_directory(label, sub_dir="application-plots")
|
|
213
174
|
logger = logging.getLogger()
|
|
214
175
|
logger.setLevel(gen.get_log_level_from_user(config_parser["log_level"]))
|
|
215
|
-
logger.info("Starting the application.")
|
|
216
176
|
|
|
217
|
-
histogram_files =
|
|
177
|
+
histogram_files = gen.get_list_of_files_from_command_line(
|
|
178
|
+
config_parser["hist_file_names"], [".zst", ".simtel", ".hdata"]
|
|
179
|
+
)
|
|
218
180
|
|
|
219
181
|
# If no output name is passed, the tool gets the name of the first histogram of the list
|
|
220
182
|
if config_parser["output_file_name"] is None:
|
|
221
183
|
config_parser["output_file_name"] = Path(histogram_files[0]).absolute().name
|
|
222
184
|
output_file_name = Path(output_path).joinpath(f"{config_parser['output_file_name']}")
|
|
223
185
|
|
|
224
|
-
# If the hdf5 output file already exists, it is overwritten
|
|
225
|
-
overwrite = check_and_log_overwrite(config_parser, logger)
|
|
226
|
-
|
|
227
186
|
simtel_histograms = SimtelIOHistograms(histogram_files)
|
|
228
187
|
create_pdf(simtel_histograms, output_file_name, config_parser, logger)
|
|
229
|
-
|
|
188
|
+
if config_parser["hdf5"]:
|
|
189
|
+
simtel_histograms.export_histograms(
|
|
190
|
+
f"{output_file_name}.hdf5",
|
|
191
|
+
overwrite=check_and_log_overwrite(config_parser, logger),
|
|
192
|
+
)
|
|
230
193
|
|
|
231
194
|
|
|
232
195
|
if __name__ == "__main__":
|
|
@@ -1,11 +1,24 @@
|
|
|
1
1
|
#!/usr/bin/python3
|
|
2
2
|
|
|
3
3
|
r"""
|
|
4
|
-
|
|
4
|
+
Derive simulation configuration parameters for a simulation production.
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
Derived simulation configuration parameters include:
|
|
7
|
+
|
|
8
|
+
* energy range
|
|
9
|
+
* shower core scatter radius
|
|
10
|
+
* view cone radius
|
|
11
|
+
* total number of events to be simulated
|
|
12
|
+
|
|
13
|
+
Configuration parameters depend on characteristics of the observations, especially elevation,
|
|
14
|
+
azimuth, and night sky background.
|
|
15
|
+
|
|
16
|
+
The configuration parameters are derived according to the required precision. The metrics are:
|
|
17
|
+
|
|
18
|
+
* statistical uncertainty on the determination of the effective area as function of primary energy
|
|
19
|
+
* fraction of lost events to the selected core scatter and view cone radius (to be implemented)
|
|
20
|
+
* statistical uncertainty of the energy migration matrix as function of primary energy
|
|
21
|
+
(to be implemented)
|
|
9
22
|
|
|
10
23
|
Command line arguments
|
|
11
24
|
----------------------
|
|
@@ -38,9 +51,9 @@ To run the simulation configuration, execute the script as follows:
|
|
|
38
51
|
--nsb 0.3 --ctao_data_level "A" --science_case "high_precision" \
|
|
39
52
|
--file_path tests/resources/production_dl2_fits/dl2_mc_events_file.fits \
|
|
40
53
|
--file_type "point-like" \
|
|
41
|
-
--metrics_file tests/resources/production_simulation_config_metrics.
|
|
54
|
+
--metrics_file tests/resources/production_simulation_config_metrics.yml --site North
|
|
42
55
|
|
|
43
|
-
The output will show the
|
|
56
|
+
The output will show the derived simulation parameters.
|
|
44
57
|
"""
|
|
45
58
|
|
|
46
59
|
import json
|
|
@@ -51,6 +64,7 @@ import astropy.units as u
|
|
|
51
64
|
|
|
52
65
|
import simtools.utils.general as gen
|
|
53
66
|
from simtools.configuration import configurator
|
|
67
|
+
from simtools.data_model import schema
|
|
54
68
|
from simtools.io_operations import io_handler
|
|
55
69
|
from simtools.production_configuration.generate_simulation_config import (
|
|
56
70
|
SimulationConfig,
|
|
@@ -60,7 +74,8 @@ from simtools.production_configuration.generate_simulation_config import (
|
|
|
60
74
|
def _parse(label):
|
|
61
75
|
"""Parse command-line arguments."""
|
|
62
76
|
config = configurator.Configurator(
|
|
63
|
-
label=label,
|
|
77
|
+
label=label,
|
|
78
|
+
description="Derive simulation configuration parameters for a simulation production.",
|
|
64
79
|
)
|
|
65
80
|
config.parser.add_argument(
|
|
66
81
|
"--azimuth", type=float, required=True, help="Azimuth angle in degrees."
|
|
@@ -128,6 +143,9 @@ def main():
|
|
|
128
143
|
metrics = (
|
|
129
144
|
gen.collect_data_from_file(args_dict["metrics_file"]) if "metrics_file" in args_dict else {}
|
|
130
145
|
)
|
|
146
|
+
schema.validate_dict_using_schema(
|
|
147
|
+
data=metrics, schema_file="production_configuration_metrics.schema.yml"
|
|
148
|
+
)
|
|
131
149
|
|
|
132
150
|
simulation_config = SimulationConfig(
|
|
133
151
|
grid_point=grid_point_config,
|
|
@@ -93,8 +93,8 @@ def _parse(label, description):
|
|
|
93
93
|
config.parser.add_argument(
|
|
94
94
|
"--metrics_file",
|
|
95
95
|
type=str,
|
|
96
|
-
default="production_simulation_config_metrics.
|
|
97
|
-
help="Metrics definition file. (default: production_simulation_config_metrics.
|
|
96
|
+
default="production_simulation_config_metrics.yml",
|
|
97
|
+
help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
|
|
98
98
|
)
|
|
99
99
|
config.parser.add_argument(
|
|
100
100
|
"--science_case", type=str, required=True, help="Science case for the simulation."
|
|
@@ -31,7 +31,7 @@ r"""
|
|
|
31
31
|
Zenith angle in degrees.
|
|
32
32
|
nshow (int, optional)
|
|
33
33
|
Number of showers to simulate.
|
|
34
|
-
The Number of simulated events depends on the number of times a shower is
|
|
34
|
+
The Number of simulated events depends on the number of times a shower is reused in the
|
|
35
35
|
telescope simulation. The number provided here is before any reuse factors.
|
|
36
36
|
start_run (int, required)
|
|
37
37
|
Start run number such that the actual run number will be 'start_run' + 'run'.
|
|
@@ -1,40 +1,40 @@
|
|
|
1
1
|
#!/usr/bin/python3
|
|
2
2
|
|
|
3
3
|
r"""
|
|
4
|
-
|
|
4
|
+
Generate a run script and submit file for HT Condor job submission of a simulation production.
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
This tool facilitates the submission of multiple simulations to the HT Condor batch system,
|
|
7
|
+
enabling:
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
- Execution of simulations using the "simtools-simulate-prod" application.
|
|
10
|
+
- 'number_of_runs' jobs are submitted to the HT Condor batch system.
|
|
11
|
+
- Utilization of an Apptainer image containing the SimPipe simulation software and tools.
|
|
12
|
+
- Packaging of data and histogram files, and writing them to a specified directory.
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
This tool is intended for use in an HT Condor environment. Jobs run in a container universe
|
|
15
|
+
using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
|
|
16
|
+
to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
20
|
+
- Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
|
|
21
|
+
GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
|
|
22
|
+
- Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
|
|
23
|
+
credentials. These should be listed similarly to a '.env' file and copied to
|
|
24
|
+
'output_path/env.txt'. Ensure that the path to the simulation software is correctly set to
|
|
25
|
+
'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
Command line arguments
|
|
29
|
+
----------------------
|
|
30
|
+
output_path (str, required)
|
|
31
|
+
Directory where the output and the simulation data files will be written.
|
|
32
|
+
apptainer_image (str, optional)
|
|
33
|
+
Apptainer image to use for the simulation (full path).
|
|
34
|
+
priority (int, optional)
|
|
35
|
+
Job priority (default: 1).
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
(all other command line arguments are identical to those of :ref:`simulate_prod`).
|
|
38
38
|
|
|
39
39
|
"""
|
|
40
40
|
|
|
@@ -24,7 +24,7 @@ r"""
|
|
|
24
24
|
simtools-submit-data-from-external \\
|
|
25
25
|
--input_meta ./tests/resources/MLTdata-preproduction.meta.yml \\
|
|
26
26
|
--input ./tests/resources/MLTdata-preproduction.ecsv \\
|
|
27
|
-
--schema
|
|
27
|
+
--schema src/simtools/schemas/input/MST_mirror_2f_measurements.schema.yml \\
|
|
28
28
|
--output_file TEST-submit_data_from_external.ecsv
|
|
29
29
|
|
|
30
30
|
Expected final print-out message:
|
|
@@ -83,6 +83,12 @@ def _parse(label, description):
|
|
|
83
83
|
type=str,
|
|
84
84
|
required=False,
|
|
85
85
|
)
|
|
86
|
+
config.parser.add_argument(
|
|
87
|
+
"--ignore_metadata",
|
|
88
|
+
help="Ignore metadata",
|
|
89
|
+
action="store_true",
|
|
90
|
+
required=False,
|
|
91
|
+
)
|
|
86
92
|
return config.initialize(output=True)
|
|
87
93
|
|
|
88
94
|
|
|
@@ -95,16 +101,18 @@ def main(): # noqa: D103
|
|
|
95
101
|
logger = logging.getLogger()
|
|
96
102
|
logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
|
|
97
103
|
|
|
98
|
-
_metadata = MetadataCollector(args_dict
|
|
104
|
+
_metadata = None if args_dict.get("ignore_metadata") else MetadataCollector(args_dict)
|
|
99
105
|
|
|
100
106
|
data_validator = validate_data.DataValidator(
|
|
101
|
-
schema_file=
|
|
107
|
+
schema_file=(
|
|
108
|
+
_metadata.get_data_model_schema_file_name() if _metadata else args_dict.get("schema")
|
|
109
|
+
),
|
|
102
110
|
data_file=args_dict["input"],
|
|
103
111
|
)
|
|
104
112
|
|
|
105
113
|
writer.ModelDataWriter.dump(
|
|
106
114
|
args_dict=args_dict,
|
|
107
|
-
metadata=_metadata.get_top_level_metadata(),
|
|
115
|
+
metadata=_metadata.get_top_level_metadata() if _metadata else None,
|
|
108
116
|
product_data=data_validator.validate_and_transform(),
|
|
109
117
|
)
|
|
110
118
|
|
|
@@ -16,8 +16,8 @@ r"""
|
|
|
16
16
|
instrument name.
|
|
17
17
|
site (str)
|
|
18
18
|
site location.
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
parameter_version (str)
|
|
20
|
+
Parameter version.
|
|
21
21
|
input_meta (str, optional)
|
|
22
22
|
input meta data file (yml format)
|
|
23
23
|
|
|
@@ -33,7 +33,7 @@ r"""
|
|
|
33
33
|
--value 2 \\
|
|
34
34
|
--instrument LSTN-design \\
|
|
35
35
|
--site North \\
|
|
36
|
-
--
|
|
36
|
+
--parameter_version 0.1.0 \\
|
|
37
37
|
--input_meta num_gains.metadata.yml
|
|
38
38
|
|
|
39
39
|
"""
|
|
@@ -70,7 +70,9 @@ def _parse(label, description):
|
|
|
70
70
|
)
|
|
71
71
|
config.parser.add_argument("--instrument", type=str, required=True, help="Instrument name")
|
|
72
72
|
config.parser.add_argument("--site", type=str, required=True, help="Site location")
|
|
73
|
-
config.parser.add_argument(
|
|
73
|
+
config.parser.add_argument(
|
|
74
|
+
"--parameter_version", type=str, required=True, help="Parameter version"
|
|
75
|
+
)
|
|
74
76
|
|
|
75
77
|
config.parser.add_argument(
|
|
76
78
|
"--value",
|
|
@@ -102,7 +104,7 @@ def main(): # noqa: D103
|
|
|
102
104
|
logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
|
|
103
105
|
|
|
104
106
|
output_path = (
|
|
105
|
-
Path(args_dict["output_path"]) / args_dict["
|
|
107
|
+
Path(args_dict["output_path"]) / args_dict["parameter_version"] / args_dict["instrument"]
|
|
106
108
|
if args_dict.get("output_path")
|
|
107
109
|
else None
|
|
108
110
|
)
|
|
@@ -110,7 +112,7 @@ def main(): # noqa: D103
|
|
|
110
112
|
parameter_name=args_dict["parameter"],
|
|
111
113
|
value=args_dict["value"],
|
|
112
114
|
instrument=args_dict["instrument"],
|
|
113
|
-
|
|
115
|
+
parameter_version=args_dict["parameter_version"],
|
|
114
116
|
output_file=Path(args_dict["parameter"]).with_suffix(".json"),
|
|
115
117
|
output_path=output_path,
|
|
116
118
|
use_plain_output_path=args_dict.get("use_plain_output_path"),
|
|
@@ -47,7 +47,7 @@ import logging
|
|
|
47
47
|
from pathlib import Path
|
|
48
48
|
|
|
49
49
|
import simtools.utils.general as gen
|
|
50
|
-
from simtools.camera_efficiency import CameraEfficiency
|
|
50
|
+
from simtools.camera.camera_efficiency import CameraEfficiency
|
|
51
51
|
from simtools.configuration import configurator
|
|
52
52
|
|
|
53
53
|
|
|
@@ -106,7 +106,7 @@ def main(): # noqa: D103
|
|
|
106
106
|
ce = CameraEfficiency(
|
|
107
107
|
db_config=_db_config,
|
|
108
108
|
simtel_path=args_dict["simtel_path"],
|
|
109
|
-
label=label,
|
|
109
|
+
label=args_dict.get("label", label),
|
|
110
110
|
config_data=args_dict,
|
|
111
111
|
)
|
|
112
112
|
ce.simulate()
|
|
@@ -36,12 +36,10 @@ import logging
|
|
|
36
36
|
import re
|
|
37
37
|
from pathlib import Path
|
|
38
38
|
|
|
39
|
-
import jsonschema
|
|
40
|
-
|
|
41
39
|
import simtools.utils.general as gen
|
|
42
40
|
from simtools.configuration import configurator
|
|
43
41
|
from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
|
|
44
|
-
from simtools.data_model import metadata_collector,
|
|
42
|
+
from simtools.data_model import metadata_collector, schema, validate_data
|
|
45
43
|
|
|
46
44
|
|
|
47
45
|
def _parse(label, description):
|
|
@@ -116,7 +114,20 @@ def _get_schema_file_name(args_dict, data_dict=None):
|
|
|
116
114
|
return schema_file
|
|
117
115
|
|
|
118
116
|
|
|
119
|
-
def
|
|
117
|
+
def _get_json_file_list(file_directory=None, file_name=None):
|
|
118
|
+
"""Return list of json files in a directory."""
|
|
119
|
+
file_list = []
|
|
120
|
+
if file_directory is not None:
|
|
121
|
+
file_list = list(Path(file_directory).rglob("*.json"))
|
|
122
|
+
if not file_list:
|
|
123
|
+
raise FileNotFoundError(f"No files found in {file_directory}")
|
|
124
|
+
elif file_name is not None:
|
|
125
|
+
file_list = [file_name]
|
|
126
|
+
|
|
127
|
+
return file_list
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def validate_dict_using_schema(args_dict, logger):
|
|
120
131
|
"""
|
|
121
132
|
Validate a schema file (or several files) given in yaml or json format.
|
|
122
133
|
|
|
@@ -124,33 +135,26 @@ def validate_schema(args_dict, logger):
|
|
|
124
135
|
the metadata section of the data dictionary.
|
|
125
136
|
|
|
126
137
|
"""
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
file_list = [args_dict["file_name"]]
|
|
131
|
-
for file_name in file_list:
|
|
138
|
+
for file_name in _get_json_file_list(
|
|
139
|
+
args_dict.get("file_directory"), args_dict.get("file_name")
|
|
140
|
+
):
|
|
132
141
|
try:
|
|
133
142
|
data = gen.collect_data_from_file(file_name=file_name)
|
|
134
143
|
except FileNotFoundError as exc:
|
|
135
144
|
logger.error(f"Error reading schema file from {file_name}")
|
|
136
145
|
raise exc
|
|
137
|
-
|
|
138
|
-
metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
|
|
139
|
-
except jsonschema.exceptions.ValidationError as exc:
|
|
140
|
-
logger.error(f"Failed validation of file {file_name}")
|
|
141
|
-
raise exc
|
|
146
|
+
schema.validate_dict_using_schema(data, _get_schema_file_name(args_dict, data))
|
|
142
147
|
logger.info(f"Successful validation of file {file_name}")
|
|
143
148
|
|
|
144
149
|
|
|
145
150
|
def validate_data_files(args_dict, logger):
|
|
146
151
|
"""Validate data files."""
|
|
147
|
-
|
|
148
|
-
if file_directory is not None:
|
|
152
|
+
if args_dict.get("file_directory") is not None:
|
|
149
153
|
tmp_args_dict = {}
|
|
150
|
-
for file_name in
|
|
154
|
+
for file_name in _get_json_file_list(args_dict.get("file_directory")):
|
|
151
155
|
tmp_args_dict["file_name"] = file_name
|
|
152
156
|
parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
|
|
153
|
-
schema_file =
|
|
157
|
+
schema_file = schema.get_model_parameter_schema_file(f"{parameter_name}")
|
|
154
158
|
tmp_args_dict["schema"] = schema_file
|
|
155
159
|
tmp_args_dict["data_type"] = "model_parameter"
|
|
156
160
|
tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]
|
|
@@ -192,7 +196,7 @@ def main(): # noqa: D103
|
|
|
192
196
|
if args_dict["data_type"].lower() == "metadata":
|
|
193
197
|
validate_metadata(args_dict, logger)
|
|
194
198
|
elif args_dict["data_type"].lower() == "schema":
|
|
195
|
-
|
|
199
|
+
validate_dict_using_schema(args_dict, logger)
|
|
196
200
|
else:
|
|
197
201
|
validate_data_files(args_dict, logger)
|
|
198
202
|
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Single photon electron spectral analysis."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
import subprocess
|
|
6
|
+
import tempfile
|
|
7
|
+
from io import BytesIO
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from astropy.table import Table
|
|
11
|
+
|
|
12
|
+
import simtools.data_model.model_data_writer as writer
|
|
13
|
+
from simtools.constants import SCHEMA_PATH
|
|
14
|
+
from simtools.data_model import validate_data
|
|
15
|
+
from simtools.data_model.metadata_collector import MetadataCollector
|
|
16
|
+
from simtools.io_operations import io_handler
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SinglePhotonElectronSpectrum:
|
|
20
|
+
"""
|
|
21
|
+
Single photon electron spectral analysis.
|
|
22
|
+
|
|
23
|
+
Parameters
|
|
24
|
+
----------
|
|
25
|
+
args_dict: dict
|
|
26
|
+
Dictionary with input arguments.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
prompt_column = "frequency (prompt)"
|
|
30
|
+
prompt_plus_afterpulse_column = "frequency (prompt+afterpulsing)"
|
|
31
|
+
afterpulse_column = "frequency (afterpulsing)"
|
|
32
|
+
|
|
33
|
+
input_schema = SCHEMA_PATH / "input" / "single_pe_spectrum.schema.yml"
|
|
34
|
+
|
|
35
|
+
def __init__(self, args_dict):
|
|
36
|
+
"""Initialize SinglePhotonElectronSpectrum class."""
|
|
37
|
+
self._logger = logging.getLogger(__name__)
|
|
38
|
+
self._logger.debug("Initialize SinglePhotonElectronSpectrum class.")
|
|
39
|
+
|
|
40
|
+
self.args_dict = args_dict
|
|
41
|
+
# default output is of ecsv format
|
|
42
|
+
self.args_dict["output_file"] = str(
|
|
43
|
+
Path(self.args_dict["output_file"]).with_suffix(".ecsv")
|
|
44
|
+
)
|
|
45
|
+
self.io_handler = io_handler.IOHandler()
|
|
46
|
+
self.data = "" # Single photon electron spectrum data (as string)
|
|
47
|
+
self.metadata = MetadataCollector(args_dict=self.args_dict)
|
|
48
|
+
|
|
49
|
+
def derive_single_pe_spectrum(self):
|
|
50
|
+
"""Derive single photon electron spectrum."""
|
|
51
|
+
if self.args_dict.get("use_norm_spe"):
|
|
52
|
+
return self._derive_spectrum_norm_spe()
|
|
53
|
+
|
|
54
|
+
raise NotImplementedError(
|
|
55
|
+
"Derivation of single photon electron spectrum using a simtool is not yet implemented."
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def write_single_pe_spectrum(self):
|
|
59
|
+
"""
|
|
60
|
+
Write single photon electron spectrum plus metadata to disk.
|
|
61
|
+
|
|
62
|
+
Includes writing in simtel and simtools (ecsv) formats.
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
simtel_file = self.io_handler.get_output_directory() / Path(
|
|
66
|
+
self.args_dict["output_file"]
|
|
67
|
+
).with_suffix(".dat")
|
|
68
|
+
self._logger.debug(f"norm_spe output file: {simtel_file}")
|
|
69
|
+
with open(simtel_file, "w", encoding="utf-8") as simtel:
|
|
70
|
+
simtel.write(self.data)
|
|
71
|
+
|
|
72
|
+
cleaned_data = re.sub(r"%%%.+", "", self.data) # remove norm_spe row metadata
|
|
73
|
+
table = Table.read(
|
|
74
|
+
BytesIO(cleaned_data.encode("utf-8")),
|
|
75
|
+
format="ascii.no_header",
|
|
76
|
+
comment="#",
|
|
77
|
+
delimiter="\t",
|
|
78
|
+
)
|
|
79
|
+
table.rename_columns(
|
|
80
|
+
["col1", "col2", "col3"],
|
|
81
|
+
["amplitude", self.prompt_column, self.prompt_plus_afterpulse_column],
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
writer.ModelDataWriter.dump(
|
|
85
|
+
args_dict=self.args_dict,
|
|
86
|
+
metadata=self.metadata.top_level_meta,
|
|
87
|
+
product_data=table,
|
|
88
|
+
validate_schema_file=None,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
def _derive_spectrum_norm_spe(self):
|
|
92
|
+
"""
|
|
93
|
+
Derive single photon electron spectrum using sim_telarray tool 'norm_spe'.
|
|
94
|
+
|
|
95
|
+
Returns
|
|
96
|
+
-------
|
|
97
|
+
int
|
|
98
|
+
Return code of the executed command
|
|
99
|
+
|
|
100
|
+
Raises
|
|
101
|
+
------
|
|
102
|
+
subprocess.CalledProcessError
|
|
103
|
+
If the command execution fails.
|
|
104
|
+
"""
|
|
105
|
+
tmp_input_file = self._get_input_data(
|
|
106
|
+
input_file=self.args_dict["input_spectrum"],
|
|
107
|
+
frequency_column=self.prompt_column,
|
|
108
|
+
)
|
|
109
|
+
tmp_ap_file = self._get_input_data(
|
|
110
|
+
input_file=self.args_dict.get("afterpulse_spectrum"),
|
|
111
|
+
frequency_column=self.afterpulse_column,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
command = [
|
|
115
|
+
f"{self.args_dict['simtel_path']}/sim_telarray/bin/norm_spe",
|
|
116
|
+
"-r",
|
|
117
|
+
f"{self.args_dict['step_size']},{self.args_dict['max_amplitude']}",
|
|
118
|
+
tmp_input_file.name,
|
|
119
|
+
]
|
|
120
|
+
if tmp_ap_file:
|
|
121
|
+
command.insert(1, "-a")
|
|
122
|
+
command.insert(2, f"{tmp_ap_file.name}")
|
|
123
|
+
|
|
124
|
+
self._logger.debug(f"Running norm_spe command: {' '.join(command)}")
|
|
125
|
+
try:
|
|
126
|
+
result = subprocess.run(command, capture_output=True, text=True, check=True)
|
|
127
|
+
except subprocess.CalledProcessError as exc:
|
|
128
|
+
self._logger.error(f"Error running norm_spe: {exc}")
|
|
129
|
+
self._logger.error(f"stderr: {exc.stderr}")
|
|
130
|
+
raise exc
|
|
131
|
+
finally:
|
|
132
|
+
for tmp_file in [tmp_input_file, tmp_ap_file]:
|
|
133
|
+
try:
|
|
134
|
+
Path(tmp_file.name).unlink()
|
|
135
|
+
except (AttributeError, FileNotFoundError):
|
|
136
|
+
pass
|
|
137
|
+
|
|
138
|
+
self.data = result.stdout
|
|
139
|
+
return result.returncode
|
|
140
|
+
|
|
141
|
+
def _get_input_data(self, input_file, frequency_column):
|
|
142
|
+
"""
|
|
143
|
+
Return input data for norm_spe command.
|
|
144
|
+
|
|
145
|
+
Input data need to be space separated values of the amplitude spectrum.
|
|
146
|
+
"""
|
|
147
|
+
input_data = ""
|
|
148
|
+
if not input_file:
|
|
149
|
+
return None
|
|
150
|
+
input_file = Path(input_file)
|
|
151
|
+
|
|
152
|
+
if input_file.suffix == ".ecsv":
|
|
153
|
+
data_validator = validate_data.DataValidator(
|
|
154
|
+
schema_file=self.input_schema, data_file=input_file
|
|
155
|
+
)
|
|
156
|
+
table = data_validator.validate_and_transform()
|
|
157
|
+
input_data = "\n".join(f"{row['amplitude']} {row[frequency_column]}" for row in table)
|
|
158
|
+
else:
|
|
159
|
+
with open(input_file, encoding="utf-8") as f:
|
|
160
|
+
input_data = (
|
|
161
|
+
f.read().replace(",", " ")
|
|
162
|
+
if frequency_column == self.prompt_column
|
|
163
|
+
else f.read()
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
with tempfile.NamedTemporaryFile(delete=False, mode="w", encoding="utf-8") as tmpfile:
|
|
167
|
+
tmpfile.write(input_data)
|
|
168
|
+
return tmpfile
|