gammasimtools 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/METADATA +2 -2
  2. {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/RECORD +94 -85
  3. {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/entry_points.txt +2 -1
  4. simtools/_version.py +2 -2
  5. simtools/applications/calculate_trigger_rate.py +15 -38
  6. simtools/applications/convert_all_model_parameters_from_simtel.py +9 -28
  7. simtools/applications/convert_geo_coordinates_of_array_elements.py +47 -45
  8. simtools/applications/convert_model_parameter_from_simtel.py +2 -2
  9. simtools/applications/db_add_file_to_db.py +1 -2
  10. simtools/applications/db_add_simulation_model_from_repository_to_db.py +110 -0
  11. simtools/applications/db_add_value_from_json_to_db.py +1 -2
  12. simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +6 -6
  13. simtools/applications/db_get_file_from_db.py +11 -12
  14. simtools/applications/db_get_parameter_from_db.py +44 -32
  15. simtools/applications/derive_photon_electron_spectrum.py +99 -0
  16. simtools/applications/generate_array_config.py +17 -17
  17. simtools/applications/generate_regular_arrays.py +15 -15
  18. simtools/applications/generate_simtel_array_histograms.py +11 -48
  19. simtools/applications/production_generate_simulation_config.py +25 -7
  20. simtools/applications/production_scale_events.py +2 -2
  21. simtools/applications/simulate_prod.py +1 -1
  22. simtools/applications/simulate_prod_htcondor_generator.py +26 -26
  23. simtools/applications/submit_data_from_external.py +12 -4
  24. simtools/applications/submit_model_parameter_from_external.py +8 -6
  25. simtools/applications/validate_camera_efficiency.py +2 -2
  26. simtools/applications/validate_file_using_schema.py +23 -19
  27. simtools/camera/single_photon_electron_spectrum.py +168 -0
  28. simtools/configuration/commandline_parser.py +8 -1
  29. simtools/constants.py +10 -3
  30. simtools/corsika/corsika_config.py +8 -7
  31. simtools/corsika/corsika_histograms.py +1 -1
  32. simtools/data_model/data_reader.py +0 -3
  33. simtools/data_model/metadata_collector.py +3 -4
  34. simtools/data_model/metadata_model.py +8 -124
  35. simtools/data_model/model_data_writer.py +17 -63
  36. simtools/data_model/schema.py +213 -0
  37. simtools/data_model/validate_data.py +9 -44
  38. simtools/db/db_handler.py +323 -495
  39. simtools/db/db_model_upload.py +139 -0
  40. simtools/io_operations/hdf5_handler.py +54 -24
  41. simtools/layout/array_layout.py +33 -28
  42. simtools/model/array_model.py +13 -7
  43. simtools/model/model_parameter.py +22 -54
  44. simtools/model/site_model.py +2 -2
  45. simtools/production_configuration/calculate_statistical_errors_grid_point.py +119 -144
  46. simtools/production_configuration/event_scaler.py +7 -17
  47. simtools/production_configuration/generate_simulation_config.py +5 -32
  48. simtools/production_configuration/interpolation_handler.py +8 -11
  49. simtools/runners/corsika_simtel_runner.py +3 -1
  50. simtools/schemas/input/MST_mirror_2f_measurements.schema.yml +39 -0
  51. simtools/schemas/input/single_pe_spectrum.schema.yml +38 -0
  52. simtools/schemas/integration_tests_config.metaschema.yml +10 -0
  53. simtools/schemas/model_parameter.metaschema.yml +7 -2
  54. simtools/schemas/model_parameter_and_data_schema.metaschema.yml +2 -0
  55. simtools/schemas/model_parameters/array_element_position_utm.schema.yml +1 -1
  56. simtools/schemas/model_parameters/array_window.schema.yml +37 -0
  57. simtools/schemas/model_parameters/asum_clipping.schema.yml +0 -4
  58. simtools/schemas/model_parameters/channels_per_chip.schema.yml +1 -1
  59. simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml +2 -2
  60. simtools/schemas/model_parameters/dsum_clipping.schema.yml +0 -2
  61. simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +0 -2
  62. simtools/schemas/model_parameters/dsum_offset.schema.yml +0 -2
  63. simtools/schemas/model_parameters/dsum_pedsub.schema.yml +0 -2
  64. simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +0 -2
  65. simtools/schemas/model_parameters/dsum_prescale.schema.yml +0 -2
  66. simtools/schemas/model_parameters/dsum_presum_max.schema.yml +0 -2
  67. simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +0 -2
  68. simtools/schemas/model_parameters/dsum_shaping.schema.yml +0 -2
  69. simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +0 -2
  70. simtools/schemas/model_parameters/dsum_threshold.schema.yml +0 -2
  71. simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +0 -2
  72. simtools/schemas/model_parameters/fadc_compensate_pedestal.schema.yml +1 -1
  73. simtools/schemas/model_parameters/fadc_lg_compensate_pedestal.schema.yml +1 -1
  74. simtools/schemas/model_parameters/fadc_noise.schema.yml +3 -3
  75. simtools/schemas/model_parameters/fake_mirror_list.schema.yml +33 -0
  76. simtools/schemas/model_parameters/laser_photons.schema.yml +2 -2
  77. simtools/schemas/model_parameters/secondary_mirror_degraded_reflection.schema.yml +1 -1
  78. simtools/schemas/production_configuration_metrics.schema.yml +68 -0
  79. simtools/schemas/production_tables.schema.yml +41 -0
  80. simtools/simtel/simtel_config_writer.py +5 -6
  81. simtools/simtel/simtel_io_histogram.py +32 -67
  82. simtools/simtel/simtel_io_histograms.py +15 -30
  83. simtools/simtel/simulator_array.py +2 -1
  84. simtools/simtel/simulator_camera_efficiency.py +5 -0
  85. simtools/simtel/simulator_light_emission.py +3 -1
  86. simtools/simtel/simulator_ray_tracing.py +2 -1
  87. simtools/testing/helpers.py +6 -13
  88. simtools/testing/validate_output.py +131 -47
  89. simtools/utils/general.py +102 -12
  90. simtools/utils/names.py +24 -20
  91. simtools/applications/db_add_model_parameters_from_repository_to_db.py +0 -176
  92. simtools/db/db_array_elements.py +0 -130
  93. {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/LICENSE +0 -0
  94. {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/WHEEL +0 -0
  95. {gammasimtools-0.9.0.dist-info → gammasimtools-0.10.0.dist-info}/top_level.txt +0 -0
  96. /simtools/{camera_efficiency.py → camera/camera_efficiency.py} +0 -0
@@ -1,27 +1,27 @@
1
1
  #!/usr/bin/python3
2
2
 
3
3
  """
4
- Make a regular array of telescopes and save it as astropy table.
4
+ Make a regular array of telescopes and save it as astropy table.
5
5
 
6
- The arrays consist of one telescope at the center of the array and or of 4 telescopes
7
- in a square grid. These arrays are used for trigger rate simulations.
6
+ The arrays consist of one telescope at the center of the array and or of 4 telescopes
7
+ in a square grid. These arrays are used for trigger rate simulations.
8
8
 
9
- The array layout files created will be available at the data/layout directory.
9
+ The array layout files created will be available at the data/layout directory.
10
10
 
11
- Command line arguments
12
- ----------------------
13
- site (str, required)
14
- observatory site (e.g., North or South).
15
- model_version (str, optional)
16
- Model version to use (e.g., 6.0.0). If not provided, the latest version is used.
11
+ Command line arguments
12
+ ----------------------
13
+ site (str, required)
14
+ observatory site (e.g., North or South).
15
+ model_version (str, optional)
16
+ Model version to use (e.g., 6.0.0). If not provided, the latest version is used.
17
17
 
18
- Example
19
- -------
20
- Runtime < 10 s.
18
+ Example
19
+ -------
20
+ Runtime < 10 s.
21
21
 
22
- .. code-block:: console
22
+ .. code-block:: console
23
23
 
24
- simtools-generate-regular-arrays --site=North
24
+ simtools-generate-regular-arrays --site=North
25
25
  """
26
26
 
27
27
  import logging
@@ -13,8 +13,6 @@ r"""
13
13
  Name of the histogram files to be plotted.
14
14
  It can be given as the histogram file names (more than one option allowed) or as a text
15
15
  file with the names of the histogram files in it.
16
- figure_name (str, required)
17
- File name for the pdf output (without extension).
18
16
  pdf (bool, optional)
19
17
  If set, histograms are saved into pdf files.
20
18
  One pdf file contains all the histograms found in the file.
@@ -29,6 +27,8 @@ r"""
29
27
  If the output output_file_name.hdf5 file already exists and hdf5 is set, the tables
30
28
  associated to hdf5 will be overwritten. The remaining tables, if any, will stay
31
29
  untouched.
30
+ test: bool
31
+ Test option. Generate only two histograms for testing purposes.
32
32
 
33
33
  Raises
34
34
  ------
@@ -108,38 +108,6 @@ def _parse(label, description):
108
108
  return config_parser
109
109
 
110
110
 
111
- def build_histogram_files(config_parser, logger):
112
- """
113
- Build a list of histogram files from command line arguments.
114
-
115
- Parameters
116
- ----------
117
- config_parser: dict
118
- Parsed command line arguments.
119
- logger: logging.Logger
120
- Logger object for logging messages.
121
-
122
- Returns
123
- -------
124
- list
125
- List of histogram file paths.
126
- """
127
- histogram_files = []
128
- for one_file in config_parser["hist_file_names"]:
129
- try:
130
- if Path(one_file).suffix in [".zst", ".simtel", ".hdata"]:
131
- histogram_files.append(one_file)
132
- else:
133
- with open(one_file, encoding="utf-8") as file:
134
- for line in file:
135
- histogram_files.append(line.strip())
136
- except FileNotFoundError as exc:
137
- msg = f"{one_file} is not a file."
138
- logger.error(msg)
139
- raise FileNotFoundError from exc
140
- return histogram_files
141
-
142
-
143
111
  def check_and_log_overwrite(config_parser, logger):
144
112
  """
145
113
  Check if the output hdf5 file already exists and log a warning if it does.
@@ -197,36 +165,31 @@ def create_pdf(simtel_histograms, output_file_name, config_parser, logger):
197
165
  logger.info(f"Wrote histograms to the pdf file {output_file_name}.pdf")
198
166
 
199
167
 
200
- def export_to_hdf5(simtel_histograms, output_file_name, overwrite, config_parser, logger):
201
- """Export histograms to an HDF5 file."""
202
- if config_parser["hdf5"]:
203
- logger.info(f"Wrote histograms to the hdf5 file {output_file_name}.hdf5")
204
- simtel_histograms.export_histograms(f"{output_file_name}.hdf5", overwrite=overwrite)
205
-
206
-
207
168
  def main(): # noqa: D103
208
169
  label = Path(__file__).stem
209
- description = "Display the simtel_array histograms."
170
+ description = "Display simtel_array histograms and/or write them into hdf5 format."
210
171
  io_handler_instance = io_handler.IOHandler()
211
172
  config_parser = _parse(label, description)
212
173
  output_path = io_handler_instance.get_output_directory(label, sub_dir="application-plots")
213
174
  logger = logging.getLogger()
214
175
  logger.setLevel(gen.get_log_level_from_user(config_parser["log_level"]))
215
- logger.info("Starting the application.")
216
176
 
217
- histogram_files = build_histogram_files(config_parser, logger)
177
+ histogram_files = gen.get_list_of_files_from_command_line(
178
+ config_parser["hist_file_names"], [".zst", ".simtel", ".hdata"]
179
+ )
218
180
 
219
181
  # If no output name is passed, the tool gets the name of the first histogram of the list
220
182
  if config_parser["output_file_name"] is None:
221
183
  config_parser["output_file_name"] = Path(histogram_files[0]).absolute().name
222
184
  output_file_name = Path(output_path).joinpath(f"{config_parser['output_file_name']}")
223
185
 
224
- # If the hdf5 output file already exists, it is overwritten
225
- overwrite = check_and_log_overwrite(config_parser, logger)
226
-
227
186
  simtel_histograms = SimtelIOHistograms(histogram_files)
228
187
  create_pdf(simtel_histograms, output_file_name, config_parser, logger)
229
- export_to_hdf5(simtel_histograms, output_file_name, overwrite, config_parser, logger)
188
+ if config_parser["hdf5"]:
189
+ simtel_histograms.export_histograms(
190
+ f"{output_file_name}.hdf5",
191
+ overwrite=check_and_log_overwrite(config_parser, logger),
192
+ )
230
193
 
231
194
 
232
195
  if __name__ == "__main__":
@@ -1,11 +1,24 @@
1
1
  #!/usr/bin/python3
2
2
 
3
3
  r"""
4
- Configure a simulation based on command-line arguments.
4
+ Derive simulation configuration parameters for a simulation production.
5
5
 
6
- This application configures and
7
- generates simulation parameters for a specific grid point in a statistical uncertainty
8
- evaluation setup.
6
+ Derived simulation configuration parameters include:
7
+
8
+ * energy range
9
+ * shower core scatter radius
10
+ * view cone radius
11
+ * total number of events to be simulated
12
+
13
+ Configuration parameters depend on characteristics of the observations, especially elevation,
14
+ azimuth, and night sky background.
15
+
16
+ The configuration parameters are derived according to the required precision. The metrics are:
17
+
18
+ * statistical uncertainty on the determination of the effective area as function of primary energy
19
+ * fraction of lost events to the selected core scatter and view cone radius (to be implemented)
20
+ * statistical uncertainty of the energy migration matrix as function of primary energy
21
+ (to be implemented)
9
22
 
10
23
  Command line arguments
11
24
  ----------------------
@@ -38,9 +51,9 @@ To run the simulation configuration, execute the script as follows:
38
51
  --nsb 0.3 --ctao_data_level "A" --science_case "high_precision" \
39
52
  --file_path tests/resources/production_dl2_fits/dl2_mc_events_file.fits \
40
53
  --file_type "point-like" \
41
- --metrics_file tests/resources/production_simulation_config_metrics.yaml --site North
54
+ --metrics_file tests/resources/production_simulation_config_metrics.yml --site North
42
55
 
43
- The output will show the configured simulation parameters.
56
+ The output will show the derived simulation parameters.
44
57
  """
45
58
 
46
59
  import json
@@ -51,6 +64,7 @@ import astropy.units as u
51
64
 
52
65
  import simtools.utils.general as gen
53
66
  from simtools.configuration import configurator
67
+ from simtools.data_model import schema
54
68
  from simtools.io_operations import io_handler
55
69
  from simtools.production_configuration.generate_simulation_config import (
56
70
  SimulationConfig,
@@ -60,7 +74,8 @@ from simtools.production_configuration.generate_simulation_config import (
60
74
  def _parse(label):
61
75
  """Parse command-line arguments."""
62
76
  config = configurator.Configurator(
63
- label=label, description="Configure and run a simulation based on input parameters."
77
+ label=label,
78
+ description="Derive simulation configuration parameters for a simulation production.",
64
79
  )
65
80
  config.parser.add_argument(
66
81
  "--azimuth", type=float, required=True, help="Azimuth angle in degrees."
@@ -128,6 +143,9 @@ def main():
128
143
  metrics = (
129
144
  gen.collect_data_from_file(args_dict["metrics_file"]) if "metrics_file" in args_dict else {}
130
145
  )
146
+ schema.validate_dict_using_schema(
147
+ data=metrics, schema_file="production_configuration_metrics.schema.yml"
148
+ )
131
149
 
132
150
  simulation_config = SimulationConfig(
133
151
  grid_point=grid_point_config,
@@ -93,8 +93,8 @@ def _parse(label, description):
93
93
  config.parser.add_argument(
94
94
  "--metrics_file",
95
95
  type=str,
96
- default="production_simulation_config_metrics.yaml",
97
- help="Metrics definition file. (default: production_simulation_config_metrics.yaml)",
96
+ default="production_simulation_config_metrics.yml",
97
+ help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
98
98
  )
99
99
  config.parser.add_argument(
100
100
  "--science_case", type=str, required=True, help="Science case for the simulation."
@@ -31,7 +31,7 @@ r"""
31
31
  Zenith angle in degrees.
32
32
  nshow (int, optional)
33
33
  Number of showers to simulate.
34
- The Number of simulated events depends on the number of times a shower is re-used in the
34
+ The Number of simulated events depends on the number of times a shower is reused in the
35
35
  telescope simulation. The number provided here is before any reuse factors.
36
36
  start_run (int, required)
37
37
  Start run number such that the actual run number will be 'start_run' + 'run'.
@@ -1,40 +1,40 @@
1
1
  #!/usr/bin/python3
2
2
 
3
3
  r"""
4
- Generate a run script and submit file for HT Condor job submission of a simulation production.
4
+ Generate a run script and submit file for HT Condor job submission of a simulation production.
5
5
 
6
- This tool facilitates the submission of multiple simulations to the HT Condor batch system,
7
- enabling:
6
+ This tool facilitates the submission of multiple simulations to the HT Condor batch system,
7
+ enabling:
8
8
 
9
- - Execution of simulations using the "simtools-simulate-prod" application.
10
- - 'number_of_runs' jobs are submitted to the HT Condor batch system.
11
- - Utilization of an Apptainer image containing the SimPipe simulation software and tools.
12
- - Packaging of data and histogram files, and writing them to a specified directory.
9
+ - Execution of simulations using the "simtools-simulate-prod" application.
10
+ - 'number_of_runs' jobs are submitted to the HT Condor batch system.
11
+ - Utilization of an Apptainer image containing the SimPipe simulation software and tools.
12
+ - Packaging of data and histogram files, and writing them to a specified directory.
13
13
 
14
- This tool is intended for use in an HT Condor environment. Jobs run in a container universe
15
- using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
16
- to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
14
+ This tool is intended for use in an HT Condor environment. Jobs run in a container universe
15
+ using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
16
+ to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
17
17
 
18
- Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
18
+ Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
19
19
 
20
- - Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
21
- GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
22
- - Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
23
- credentials. These should be listed similarly to a '.env' file and copied to
24
- 'output_path/env.txt'. Ensure that the path to the simulation software is correctly set to
25
- 'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
20
+ - Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
21
+ GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
22
+ - Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
23
+ credentials. These should be listed similarly to a '.env' file and copied to
24
+ 'output_path/env.txt'. Ensure that the path to the simulation software is correctly set to
25
+ 'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
26
26
 
27
27
 
28
- Command line arguments
29
- ----------------------
30
- output_path (str, required)
31
- Directory where the output and the simulation data files will be written.
32
- apptainer_image (str, optional)
33
- Apptainer image to use for the simulation (full path).
34
- priority (int, optional)
35
- Job priority (default: 1).
28
+ Command line arguments
29
+ ----------------------
30
+ output_path (str, required)
31
+ Directory where the output and the simulation data files will be written.
32
+ apptainer_image (str, optional)
33
+ Apptainer image to use for the simulation (full path).
34
+ priority (int, optional)
35
+ Job priority (default: 1).
36
36
 
37
- (all other command line arguments are identical to those of :ref:`simulate_prod`).
37
+ (all other command line arguments are identical to those of :ref:`simulate_prod`).
38
38
 
39
39
  """
40
40
 
@@ -24,7 +24,7 @@ r"""
24
24
  simtools-submit-data-from-external \\
25
25
  --input_meta ./tests/resources/MLTdata-preproduction.meta.yml \\
26
26
  --input ./tests/resources/MLTdata-preproduction.ecsv \\
27
- --schema ./tests/resources/schema_MST_mirror_2f_measurements.yml \\
27
+ --schema src/simtools/schemas/input/MST_mirror_2f_measurements.schema.yml \\
28
28
  --output_file TEST-submit_data_from_external.ecsv
29
29
 
30
30
  Expected final print-out message:
@@ -83,6 +83,12 @@ def _parse(label, description):
83
83
  type=str,
84
84
  required=False,
85
85
  )
86
+ config.parser.add_argument(
87
+ "--ignore_metadata",
88
+ help="Ignore metadata",
89
+ action="store_true",
90
+ required=False,
91
+ )
86
92
  return config.initialize(output=True)
87
93
 
88
94
 
@@ -95,16 +101,18 @@ def main(): # noqa: D103
95
101
  logger = logging.getLogger()
96
102
  logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
97
103
 
98
- _metadata = MetadataCollector(args_dict=args_dict)
104
+ _metadata = None if args_dict.get("ignore_metadata") else MetadataCollector(args_dict)
99
105
 
100
106
  data_validator = validate_data.DataValidator(
101
- schema_file=_metadata.get_data_model_schema_file_name(),
107
+ schema_file=(
108
+ _metadata.get_data_model_schema_file_name() if _metadata else args_dict.get("schema")
109
+ ),
102
110
  data_file=args_dict["input"],
103
111
  )
104
112
 
105
113
  writer.ModelDataWriter.dump(
106
114
  args_dict=args_dict,
107
- metadata=_metadata.get_top_level_metadata(),
115
+ metadata=_metadata.get_top_level_metadata() if _metadata else None,
108
116
  product_data=data_validator.validate_and_transform(),
109
117
  )
110
118
 
@@ -16,8 +16,8 @@ r"""
16
16
  instrument name.
17
17
  site (str)
18
18
  site location.
19
- model_version (str)
20
- Model version.
19
+ parameter_version (str)
20
+ Parameter version.
21
21
  input_meta (str, optional)
22
22
  input meta data file (yml format)
23
23
 
@@ -33,7 +33,7 @@ r"""
33
33
  --value 2 \\
34
34
  --instrument LSTN-design \\
35
35
  --site North \\
36
- --model_version 6.0.0 \\
36
+ --parameter_version 0.1.0 \\
37
37
  --input_meta num_gains.metadata.yml
38
38
 
39
39
  """
@@ -70,7 +70,9 @@ def _parse(label, description):
70
70
  )
71
71
  config.parser.add_argument("--instrument", type=str, required=True, help="Instrument name")
72
72
  config.parser.add_argument("--site", type=str, required=True, help="Site location")
73
- config.parser.add_argument("--model_version", type=str, required=True, help="Model version")
73
+ config.parser.add_argument(
74
+ "--parameter_version", type=str, required=True, help="Parameter version"
75
+ )
74
76
 
75
77
  config.parser.add_argument(
76
78
  "--value",
@@ -102,7 +104,7 @@ def main(): # noqa: D103
102
104
  logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
103
105
 
104
106
  output_path = (
105
- Path(args_dict["output_path"]) / args_dict["model_version"] / args_dict["instrument"]
107
+ Path(args_dict["output_path"]) / args_dict["parameter_version"] / args_dict["instrument"]
106
108
  if args_dict.get("output_path")
107
109
  else None
108
110
  )
@@ -110,7 +112,7 @@ def main(): # noqa: D103
110
112
  parameter_name=args_dict["parameter"],
111
113
  value=args_dict["value"],
112
114
  instrument=args_dict["instrument"],
113
- model_version=args_dict["model_version"],
115
+ parameter_version=args_dict["parameter_version"],
114
116
  output_file=Path(args_dict["parameter"]).with_suffix(".json"),
115
117
  output_path=output_path,
116
118
  use_plain_output_path=args_dict.get("use_plain_output_path"),
@@ -47,7 +47,7 @@ import logging
47
47
  from pathlib import Path
48
48
 
49
49
  import simtools.utils.general as gen
50
- from simtools.camera_efficiency import CameraEfficiency
50
+ from simtools.camera.camera_efficiency import CameraEfficiency
51
51
  from simtools.configuration import configurator
52
52
 
53
53
 
@@ -106,7 +106,7 @@ def main(): # noqa: D103
106
106
  ce = CameraEfficiency(
107
107
  db_config=_db_config,
108
108
  simtel_path=args_dict["simtel_path"],
109
- label=label,
109
+ label=args_dict.get("label", label),
110
110
  config_data=args_dict,
111
111
  )
112
112
  ce.simulate()
@@ -36,12 +36,10 @@ import logging
36
36
  import re
37
37
  from pathlib import Path
38
38
 
39
- import jsonschema
40
-
41
39
  import simtools.utils.general as gen
42
40
  from simtools.configuration import configurator
43
41
  from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
44
- from simtools.data_model import metadata_collector, metadata_model, validate_data
42
+ from simtools.data_model import metadata_collector, schema, validate_data
45
43
 
46
44
 
47
45
  def _parse(label, description):
@@ -116,7 +114,20 @@ def _get_schema_file_name(args_dict, data_dict=None):
116
114
  return schema_file
117
115
 
118
116
 
119
- def validate_schema(args_dict, logger):
117
+ def _get_json_file_list(file_directory=None, file_name=None):
118
+ """Return list of json files in a directory."""
119
+ file_list = []
120
+ if file_directory is not None:
121
+ file_list = list(Path(file_directory).rglob("*.json"))
122
+ if not file_list:
123
+ raise FileNotFoundError(f"No files found in {file_directory}")
124
+ elif file_name is not None:
125
+ file_list = [file_name]
126
+
127
+ return file_list
128
+
129
+
130
+ def validate_dict_using_schema(args_dict, logger):
120
131
  """
121
132
  Validate a schema file (or several files) given in yaml or json format.
122
133
 
@@ -124,33 +135,26 @@ def validate_schema(args_dict, logger):
124
135
  the metadata section of the data dictionary.
125
136
 
126
137
  """
127
- if args_dict.get("file_directory") is not None:
128
- file_list = list(Path(args_dict["file_directory"]).rglob("*.json"))
129
- else:
130
- file_list = [args_dict["file_name"]]
131
- for file_name in file_list:
138
+ for file_name in _get_json_file_list(
139
+ args_dict.get("file_directory"), args_dict.get("file_name")
140
+ ):
132
141
  try:
133
142
  data = gen.collect_data_from_file(file_name=file_name)
134
143
  except FileNotFoundError as exc:
135
144
  logger.error(f"Error reading schema file from {file_name}")
136
145
  raise exc
137
- try:
138
- metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
139
- except jsonschema.exceptions.ValidationError as exc:
140
- logger.error(f"Failed validation of file {file_name}")
141
- raise exc
146
+ schema.validate_dict_using_schema(data, _get_schema_file_name(args_dict, data))
142
147
  logger.info(f"Successful validation of file {file_name}")
143
148
 
144
149
 
145
150
  def validate_data_files(args_dict, logger):
146
151
  """Validate data files."""
147
- file_directory = args_dict.get("file_directory")
148
- if file_directory is not None:
152
+ if args_dict.get("file_directory") is not None:
149
153
  tmp_args_dict = {}
150
- for file_name in Path(file_directory).rglob("*.json"):
154
+ for file_name in _get_json_file_list(args_dict.get("file_directory")):
151
155
  tmp_args_dict["file_name"] = file_name
152
156
  parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
153
- schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
157
+ schema_file = schema.get_model_parameter_schema_file(f"{parameter_name}")
154
158
  tmp_args_dict["schema"] = schema_file
155
159
  tmp_args_dict["data_type"] = "model_parameter"
156
160
  tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]
@@ -192,7 +196,7 @@ def main(): # noqa: D103
192
196
  if args_dict["data_type"].lower() == "metadata":
193
197
  validate_metadata(args_dict, logger)
194
198
  elif args_dict["data_type"].lower() == "schema":
195
- validate_schema(args_dict, logger)
199
+ validate_dict_using_schema(args_dict, logger)
196
200
  else:
197
201
  validate_data_files(args_dict, logger)
198
202
 
@@ -0,0 +1,168 @@
1
+ """Single photon electron spectral analysis."""
2
+
3
+ import logging
4
+ import re
5
+ import subprocess
6
+ import tempfile
7
+ from io import BytesIO
8
+ from pathlib import Path
9
+
10
+ from astropy.table import Table
11
+
12
+ import simtools.data_model.model_data_writer as writer
13
+ from simtools.constants import SCHEMA_PATH
14
+ from simtools.data_model import validate_data
15
+ from simtools.data_model.metadata_collector import MetadataCollector
16
+ from simtools.io_operations import io_handler
17
+
18
+
19
+ class SinglePhotonElectronSpectrum:
20
+ """
21
+ Single photon electron spectral analysis.
22
+
23
+ Parameters
24
+ ----------
25
+ args_dict: dict
26
+ Dictionary with input arguments.
27
+ """
28
+
29
+ prompt_column = "frequency (prompt)"
30
+ prompt_plus_afterpulse_column = "frequency (prompt+afterpulsing)"
31
+ afterpulse_column = "frequency (afterpulsing)"
32
+
33
+ input_schema = SCHEMA_PATH / "input" / "single_pe_spectrum.schema.yml"
34
+
35
+ def __init__(self, args_dict):
36
+ """Initialize SinglePhotonElectronSpectrum class."""
37
+ self._logger = logging.getLogger(__name__)
38
+ self._logger.debug("Initialize SinglePhotonElectronSpectrum class.")
39
+
40
+ self.args_dict = args_dict
41
+ # default output is of ecsv format
42
+ self.args_dict["output_file"] = str(
43
+ Path(self.args_dict["output_file"]).with_suffix(".ecsv")
44
+ )
45
+ self.io_handler = io_handler.IOHandler()
46
+ self.data = "" # Single photon electron spectrum data (as string)
47
+ self.metadata = MetadataCollector(args_dict=self.args_dict)
48
+
49
+ def derive_single_pe_spectrum(self):
50
+ """Derive single photon electron spectrum."""
51
+ if self.args_dict.get("use_norm_spe"):
52
+ return self._derive_spectrum_norm_spe()
53
+
54
+ raise NotImplementedError(
55
+ "Derivation of single photon electron spectrum using a simtool is not yet implemented."
56
+ )
57
+
58
+ def write_single_pe_spectrum(self):
59
+ """
60
+ Write single photon electron spectrum plus metadata to disk.
61
+
62
+ Includes writing in simtel and simtools (ecsv) formats.
63
+
64
+ """
65
+ simtel_file = self.io_handler.get_output_directory() / Path(
66
+ self.args_dict["output_file"]
67
+ ).with_suffix(".dat")
68
+ self._logger.debug(f"norm_spe output file: {simtel_file}")
69
+ with open(simtel_file, "w", encoding="utf-8") as simtel:
70
+ simtel.write(self.data)
71
+
72
+ cleaned_data = re.sub(r"%%%.+", "", self.data) # remove norm_spe row metadata
73
+ table = Table.read(
74
+ BytesIO(cleaned_data.encode("utf-8")),
75
+ format="ascii.no_header",
76
+ comment="#",
77
+ delimiter="\t",
78
+ )
79
+ table.rename_columns(
80
+ ["col1", "col2", "col3"],
81
+ ["amplitude", self.prompt_column, self.prompt_plus_afterpulse_column],
82
+ )
83
+
84
+ writer.ModelDataWriter.dump(
85
+ args_dict=self.args_dict,
86
+ metadata=self.metadata.top_level_meta,
87
+ product_data=table,
88
+ validate_schema_file=None,
89
+ )
90
+
91
+ def _derive_spectrum_norm_spe(self):
92
+ """
93
+ Derive single photon electron spectrum using sim_telarray tool 'norm_spe'.
94
+
95
+ Returns
96
+ -------
97
+ int
98
+ Return code of the executed command
99
+
100
+ Raises
101
+ ------
102
+ subprocess.CalledProcessError
103
+ If the command execution fails.
104
+ """
105
+ tmp_input_file = self._get_input_data(
106
+ input_file=self.args_dict["input_spectrum"],
107
+ frequency_column=self.prompt_column,
108
+ )
109
+ tmp_ap_file = self._get_input_data(
110
+ input_file=self.args_dict.get("afterpulse_spectrum"),
111
+ frequency_column=self.afterpulse_column,
112
+ )
113
+
114
+ command = [
115
+ f"{self.args_dict['simtel_path']}/sim_telarray/bin/norm_spe",
116
+ "-r",
117
+ f"{self.args_dict['step_size']},{self.args_dict['max_amplitude']}",
118
+ tmp_input_file.name,
119
+ ]
120
+ if tmp_ap_file:
121
+ command.insert(1, "-a")
122
+ command.insert(2, f"{tmp_ap_file.name}")
123
+
124
+ self._logger.debug(f"Running norm_spe command: {' '.join(command)}")
125
+ try:
126
+ result = subprocess.run(command, capture_output=True, text=True, check=True)
127
+ except subprocess.CalledProcessError as exc:
128
+ self._logger.error(f"Error running norm_spe: {exc}")
129
+ self._logger.error(f"stderr: {exc.stderr}")
130
+ raise exc
131
+ finally:
132
+ for tmp_file in [tmp_input_file, tmp_ap_file]:
133
+ try:
134
+ Path(tmp_file.name).unlink()
135
+ except (AttributeError, FileNotFoundError):
136
+ pass
137
+
138
+ self.data = result.stdout
139
+ return result.returncode
140
+
141
+ def _get_input_data(self, input_file, frequency_column):
142
+ """
143
+ Return input data for norm_spe command.
144
+
145
+ Input data need to be space separated values of the amplitude spectrum.
146
+ """
147
+ input_data = ""
148
+ if not input_file:
149
+ return None
150
+ input_file = Path(input_file)
151
+
152
+ if input_file.suffix == ".ecsv":
153
+ data_validator = validate_data.DataValidator(
154
+ schema_file=self.input_schema, data_file=input_file
155
+ )
156
+ table = data_validator.validate_and_transform()
157
+ input_data = "\n".join(f"{row['amplitude']} {row[frequency_column]}" for row in table)
158
+ else:
159
+ with open(input_file, encoding="utf-8") as f:
160
+ input_data = (
161
+ f.read().replace(",", " ")
162
+ if frequency_column == self.prompt_column
163
+ else f.read()
164
+ )
165
+
166
+ with tempfile.NamedTemporaryFile(delete=False, mode="w", encoding="utf-8") as tmpfile:
167
+ tmpfile.write(input_data)
168
+ return tmpfile