gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. {gammasimtools-0.9.0.dist-info → gammasimtools-0.11.0.dist-info}/METADATA +4 -2
  2. {gammasimtools-0.9.0.dist-info → gammasimtools-0.11.0.dist-info}/RECORD +133 -117
  3. {gammasimtools-0.9.0.dist-info → gammasimtools-0.11.0.dist-info}/WHEEL +1 -1
  4. {gammasimtools-0.9.0.dist-info → gammasimtools-0.11.0.dist-info}/entry_points.txt +6 -1
  5. simtools/_version.py +9 -4
  6. simtools/applications/calculate_trigger_rate.py +15 -38
  7. simtools/applications/convert_all_model_parameters_from_simtel.py +9 -29
  8. simtools/applications/convert_geo_coordinates_of_array_elements.py +47 -45
  9. simtools/applications/convert_model_parameter_from_simtel.py +2 -3
  10. simtools/applications/db_add_file_to_db.py +1 -3
  11. simtools/applications/db_add_simulation_model_from_repository_to_db.py +110 -0
  12. simtools/applications/db_add_value_from_json_to_db.py +1 -2
  13. simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +6 -6
  14. simtools/applications/db_get_file_from_db.py +11 -12
  15. simtools/applications/db_get_parameter_from_db.py +26 -35
  16. simtools/applications/derive_mirror_rnda.py +1 -2
  17. simtools/applications/derive_photon_electron_spectrum.py +99 -0
  18. simtools/applications/derive_psf_parameters.py +1 -0
  19. simtools/applications/docs_produce_array_element_report.py +71 -0
  20. simtools/applications/docs_produce_model_parameter_reports.py +63 -0
  21. simtools/applications/generate_array_config.py +17 -17
  22. simtools/applications/generate_corsika_histograms.py +2 -2
  23. simtools/applications/generate_regular_arrays.py +19 -17
  24. simtools/applications/generate_simtel_array_histograms.py +11 -48
  25. simtools/applications/production_derive_limits.py +95 -0
  26. simtools/applications/production_generate_simulation_config.py +37 -33
  27. simtools/applications/production_scale_events.py +4 -9
  28. simtools/applications/run_application.py +165 -0
  29. simtools/applications/simulate_light_emission.py +0 -4
  30. simtools/applications/simulate_prod.py +1 -1
  31. simtools/applications/simulate_prod_htcondor_generator.py +26 -26
  32. simtools/applications/submit_data_from_external.py +12 -4
  33. simtools/applications/submit_model_parameter_from_external.py +18 -11
  34. simtools/applications/validate_camera_efficiency.py +2 -2
  35. simtools/applications/validate_file_using_schema.py +26 -22
  36. simtools/camera/single_photon_electron_spectrum.py +168 -0
  37. simtools/configuration/commandline_parser.py +37 -1
  38. simtools/configuration/configurator.py +8 -10
  39. simtools/constants.py +10 -3
  40. simtools/corsika/corsika_config.py +19 -17
  41. simtools/corsika/corsika_histograms.py +5 -7
  42. simtools/corsika/corsika_histograms_visualize.py +2 -4
  43. simtools/data_model/data_reader.py +0 -3
  44. simtools/data_model/metadata_collector.py +20 -12
  45. simtools/data_model/metadata_model.py +8 -124
  46. simtools/data_model/model_data_writer.py +81 -75
  47. simtools/data_model/schema.py +220 -0
  48. simtools/data_model/validate_data.py +79 -68
  49. simtools/db/db_handler.py +350 -492
  50. simtools/db/db_model_upload.py +139 -0
  51. simtools/dependencies.py +112 -0
  52. simtools/io_operations/hdf5_handler.py +54 -24
  53. simtools/layout/array_layout.py +38 -32
  54. simtools/model/array_model.py +13 -7
  55. simtools/model/model_parameter.py +55 -54
  56. simtools/model/site_model.py +2 -2
  57. simtools/production_configuration/calculate_statistical_errors_grid_point.py +119 -145
  58. simtools/production_configuration/event_scaler.py +9 -35
  59. simtools/production_configuration/generate_simulation_config.py +9 -44
  60. simtools/production_configuration/interpolation_handler.py +9 -15
  61. simtools/production_configuration/limits_calculation.py +202 -0
  62. simtools/reporting/docs_read_parameters.py +310 -0
  63. simtools/runners/corsika_simtel_runner.py +4 -4
  64. simtools/schemas/{integration_tests_config.metaschema.yml → application_workflow.metaschema.yml} +61 -27
  65. simtools/schemas/array_elements.yml +8 -0
  66. simtools/schemas/input/MST_mirror_2f_measurements.schema.yml +39 -0
  67. simtools/schemas/input/single_pe_spectrum.schema.yml +38 -0
  68. simtools/schemas/model_parameter.metaschema.yml +103 -2
  69. simtools/schemas/model_parameter_and_data_schema.metaschema.yml +4 -1
  70. simtools/schemas/model_parameters/array_element_position_utm.schema.yml +1 -1
  71. simtools/schemas/model_parameters/array_window.schema.yml +37 -0
  72. simtools/schemas/model_parameters/asum_clipping.schema.yml +0 -4
  73. simtools/schemas/model_parameters/channels_per_chip.schema.yml +1 -1
  74. simtools/schemas/model_parameters/correct_nsb_spectrum_to_telescope_altitude.schema.yml +1 -1
  75. simtools/schemas/model_parameters/corsika_cherenkov_photon_bunch_size.schema.yml +2 -0
  76. simtools/schemas/model_parameters/corsika_cherenkov_photon_wavelength_range.schema.yml +2 -0
  77. simtools/schemas/model_parameters/corsika_first_interaction_height.schema.yml +2 -0
  78. simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml +4 -2
  79. simtools/schemas/model_parameters/corsika_iact_max_bunches.schema.yml +2 -0
  80. simtools/schemas/model_parameters/corsika_iact_split_auto.schema.yml +2 -0
  81. simtools/schemas/model_parameters/corsika_longitudinal_shower_development.schema.yml +2 -0
  82. simtools/schemas/model_parameters/corsika_particle_kinetic_energy_cutoff.schema.yml +2 -0
  83. simtools/schemas/model_parameters/corsika_starting_grammage.schema.yml +2 -0
  84. simtools/schemas/model_parameters/dsum_clipping.schema.yml +0 -2
  85. simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +0 -2
  86. simtools/schemas/model_parameters/dsum_offset.schema.yml +0 -2
  87. simtools/schemas/model_parameters/dsum_pedsub.schema.yml +0 -2
  88. simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +0 -2
  89. simtools/schemas/model_parameters/dsum_prescale.schema.yml +0 -2
  90. simtools/schemas/model_parameters/dsum_presum_max.schema.yml +0 -2
  91. simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +0 -2
  92. simtools/schemas/model_parameters/dsum_shaping.schema.yml +0 -2
  93. simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +0 -2
  94. simtools/schemas/model_parameters/dsum_threshold.schema.yml +0 -2
  95. simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +0 -2
  96. simtools/schemas/model_parameters/fadc_compensate_pedestal.schema.yml +1 -1
  97. simtools/schemas/model_parameters/fadc_lg_compensate_pedestal.schema.yml +1 -1
  98. simtools/schemas/model_parameters/fadc_noise.schema.yml +3 -3
  99. simtools/schemas/model_parameters/fake_mirror_list.schema.yml +33 -0
  100. simtools/schemas/model_parameters/iobuf_maximum.schema.yml +1 -1
  101. simtools/schemas/model_parameters/iobuf_output_maximum.schema.yml +1 -1
  102. simtools/schemas/model_parameters/laser_photons.schema.yml +2 -2
  103. simtools/schemas/model_parameters/lightguide_efficiency_vs_incidence_angle.schema.yml +1 -1
  104. simtools/schemas/model_parameters/lightguide_efficiency_vs_wavelength.schema.yml +1 -1
  105. simtools/schemas/model_parameters/min_photoelectrons.schema.yml +1 -1
  106. simtools/schemas/model_parameters/min_photons.schema.yml +1 -1
  107. simtools/schemas/model_parameters/random_generator.schema.yml +1 -1
  108. simtools/schemas/model_parameters/sampled_output.schema.yml +1 -1
  109. simtools/schemas/model_parameters/save_pe_with_amplitude.schema.yml +1 -1
  110. simtools/schemas/model_parameters/secondary_mirror_degraded_reflection.schema.yml +1 -1
  111. simtools/schemas/model_parameters/store_photoelectrons.schema.yml +1 -1
  112. simtools/schemas/model_parameters/tailcut_scale.schema.yml +1 -1
  113. simtools/schemas/production_configuration_metrics.schema.yml +68 -0
  114. simtools/schemas/production_tables.schema.yml +41 -0
  115. simtools/simtel/simtel_config_reader.py +1 -2
  116. simtools/simtel/simtel_config_writer.py +6 -8
  117. simtools/simtel/simtel_io_histogram.py +32 -68
  118. simtools/simtel/simtel_io_histograms.py +17 -34
  119. simtools/simtel/simulator_array.py +2 -1
  120. simtools/simtel/simulator_camera_efficiency.py +6 -3
  121. simtools/simtel/simulator_light_emission.py +5 -6
  122. simtools/simtel/simulator_ray_tracing.py +3 -4
  123. simtools/testing/configuration.py +2 -1
  124. simtools/testing/helpers.py +6 -13
  125. simtools/testing/validate_output.py +141 -47
  126. simtools/utils/general.py +114 -14
  127. simtools/utils/names.py +299 -157
  128. simtools/utils/value_conversion.py +17 -13
  129. simtools/version.py +2 -2
  130. simtools/visualization/legend_handlers.py +2 -0
  131. simtools/applications/db_add_model_parameters_from_repository_to_db.py +0 -176
  132. simtools/db/db_array_elements.py +0 -130
  133. {gammasimtools-0.9.0.dist-info → gammasimtools-0.11.0.dist-info}/LICENSE +0 -0
  134. {gammasimtools-0.9.0.dist-info → gammasimtools-0.11.0.dist-info}/top_level.txt +0 -0
  135. /simtools/{camera_efficiency.py → camera/camera_efficiency.py} +0 -0
@@ -87,7 +87,7 @@ def get_value_unit_type(value, unit_str=None):
87
87
  return base_value, _unit_as_string(base_unit), base_type
88
88
 
89
89
 
90
- def split_value_and_unit(value):
90
+ def split_value_and_unit(value, is_integer=False):
91
91
  """
92
92
  Split a value into its value and unit.
93
93
 
@@ -98,6 +98,8 @@ def split_value_and_unit(value):
98
98
  ----------
99
99
  value: str, int, float, bool, u.Quantity
100
100
  Value to be parsed.
101
+ is_integer: bool
102
+ Flag to indicate if the value is an integer.
101
103
 
102
104
  Returns
103
105
  -------
@@ -105,40 +107,42 @@ def split_value_and_unit(value):
105
107
  Value and units as (value, unit), or lists of values and unites
106
108
  """
107
109
  if isinstance(value, u.Quantity):
108
- return _split_value_is_quantity(value)
110
+ return _split_value_is_quantity(value, is_integer)
109
111
  if isinstance(value, str):
110
- return _split_value_is_string(value)
112
+ return _split_value_is_string(value, is_integer)
111
113
  if isinstance(value, list | np.ndarray):
112
- return _split_value_is_list(value)
114
+ return _split_value_is_list(value, is_integer)
113
115
  return value, None
114
116
 
115
117
 
116
- def _split_value_is_quantity(value):
118
+ def _split_value_is_quantity(value, is_integer=False):
117
119
  """Split value and unit for an astropy Quantity."""
118
- if isinstance(value.value, list | np.ndarray): # type [100.0, 200] * u.m,
119
- return list(value.value), [str(value.unit)] * len(value)
120
- return value.value, str(value.unit)
120
+ if isinstance(value.value, list | np.ndarray): # type [100.0, 200] * u.m
121
+ return [int(v) if is_integer else v for v in value.value], [str(value.unit)] * len(value)
122
+ return int(value.value) if is_integer else value.value, str(value.unit)
121
123
 
122
124
 
123
- def _split_value_is_string(value):
125
+ def _split_value_is_string(value, is_integer=False):
124
126
  """Split vale and unit for a string."""
125
127
  if value.isdigit(): # single integer value
126
128
  return int(value), None
127
129
  try: # single value with/without unit
128
- return u.Quantity(value).value, str(u.Quantity(value).unit)
130
+ return int(u.Quantity(value).value) if is_integer else u.Quantity(value).value, str(
131
+ u.Quantity(value).unit
132
+ )
129
133
  except ValueError:
130
- return _split_value_is_list(gen.convert_string_to_list(value))
134
+ return _split_value_is_list(gen.convert_string_to_list(value), is_integer)
131
135
  except TypeError: # string value (not numerical)
132
136
  return value, None
133
137
 
134
138
 
135
- def _split_value_is_list(value):
139
+ def _split_value_is_list(value, is_integer=False):
136
140
  """Split value and unit for a list."""
137
141
  value_list = []
138
142
  unit_list = []
139
143
  for item in value:
140
144
  _value, _unit = split_value_and_unit(item)
141
- value_list.append(_value)
145
+ value_list.append(int(_value) if is_integer else _value)
142
146
  if isinstance(_unit, str):
143
147
  unit_list.append(_unit)
144
148
  else:
simtools/version.py CHANGED
@@ -9,11 +9,11 @@ try:
9
9
  from ._dev_version import version
10
10
  except ImportError:
11
11
  from ._version import version
12
- except Exception:
12
+ except Exception: # pylint: disable=broad-except
13
13
  import warnings
14
14
 
15
15
  warnings.warn("Could not determine simtools version; this indicates a broken installation.")
16
16
  del warnings
17
- version = "0.0.0"
17
+ version = "0.0.0" # pylint: disable=invalid-name
18
18
 
19
19
  __version__ = version
@@ -71,6 +71,7 @@ class TelescopeHandler:
71
71
  def __init__(self, radius=None):
72
72
  self.colors_dict = {
73
73
  "LSTN": "darkorange",
74
+ "MSTx": "dodgerblue",
74
75
  "MSTN": "dodgerblue",
75
76
  "LSTS": "darkorange",
76
77
  "MSTS": "dodgerblue",
@@ -84,6 +85,7 @@ class TelescopeHandler:
84
85
  # hardwired values; this is for plotting purposes only
85
86
  self.radius_dict = {
86
87
  "LSTN": 12.5,
88
+ "MSTx": 9.15,
87
89
  "MSTN": 9.15,
88
90
  "LSTS": 12.5,
89
91
  "MSTS": 9.15,
@@ -1,176 +0,0 @@
1
- #!/usr/bin/python3
2
- r"""
3
- Add parameters found in a model parameter repository to a new database.
4
-
5
- Generates a new database with all required collections.
6
- Follows the structure of the CTAO gitlab model parameters repository.
7
- file as input.
8
-
9
- This is an application for experts and should not be used by the general user.
10
-
11
- Command line arguments
12
-
13
- input_path (str, required)
14
- Path of local copy of model parameter repository.
15
- db_name (str, required)
16
- Name of new DB to be created.
17
- type (str, optional)
18
- Type of data to be uploaded to the DB. Options are: model_parameters
19
-
20
- Examples
21
- --------
22
- Upload model data repository to the DB:
23
-
24
- .. code-block:: console
25
-
26
- simtools-db-add_model-parameters-from-repository-to-db \
27
- --input_path /path/to/repository \
28
- --db_name new_db_name \
29
- --type model_parameters
30
- """
31
-
32
- import logging
33
- from pathlib import Path
34
-
35
- import simtools.utils.general as gen
36
- from simtools.configuration import configurator
37
- from simtools.db import db_handler
38
- from simtools.utils import names
39
-
40
-
41
- def _parse(label=None, description=None):
42
- """
43
- Parse command line configuration.
44
-
45
- Parameters
46
- ----------
47
- label : str
48
- Label describing application.
49
- description : str
50
- Description of application.
51
-
52
- Returns
53
- -------
54
- CommandLineParser
55
- Command line parser object.
56
- """
57
- config = configurator.Configurator(label=label, description=description)
58
- config.parser.add_argument(
59
- "--input_path",
60
- help="Path to model parameter repository.",
61
- type=Path,
62
- required=True,
63
- )
64
- config.parser.add_argument(
65
- "--db_name",
66
- help="Name of the new model parameter database to be created.",
67
- type=str.strip,
68
- required=True,
69
- )
70
- config.parser.add_argument(
71
- "--type",
72
- help="Type of data to be uploaded to the database.",
73
- type=str,
74
- required=False,
75
- default="model_parameters",
76
- choices=["model_parameters"],
77
- )
78
-
79
- args_dict, db_config = config.initialize(
80
- output=True, require_command_line=True, db_config=True, simulation_model="version"
81
- )
82
- db_config["db_simulation_model"] = args_dict["db_name"] # overwrite explicitly DB configuration
83
- return args_dict, db_config
84
-
85
-
86
- def add_values_from_json_to_db(file, collection, db, db_name, file_prefix, logger):
87
- """
88
- Upload data from json files to db.
89
-
90
- Parameters
91
- ----------
92
- file : list
93
- Json file to be uploaded to the DB.
94
- collection : str
95
- The DB collection to which to add the file.
96
- db : DatabaseHandler
97
- Database handler object.
98
- db_name : str
99
- Name of the database to be created.
100
- file_prefix : str
101
- Path to location of all additional files to be uploaded.
102
- logger : logging.Logger
103
- Logger object.
104
- """
105
- par_dict = gen.collect_data_from_file(file_name=file)
106
- logger.info(
107
- f"Adding the following parameter to the DB: {par_dict['parameter']} "
108
- f"(collection {collection} in database {db_name})"
109
- )
110
- db.add_new_parameter(
111
- db_name=db_name,
112
- par_dict=par_dict,
113
- collection_name=collection,
114
- file_prefix=file_prefix,
115
- )
116
-
117
-
118
- def _add_model_parameters_to_db(args_dict, db, logger):
119
- """
120
- Add model parameters to the DB.
121
-
122
- Parameters
123
- ----------
124
- args_dict : dict
125
- Command line arguments.
126
- db : DatabaseHandler
127
- Database handler object.
128
- logger : logging.Logger
129
- Logger object.
130
-
131
- """
132
- input_path = Path(args_dict["input_path"])
133
- array_elements = [d for d in input_path.iterdir() if d.is_dir()]
134
- for element in array_elements:
135
- try:
136
- collection = names.get_collection_name_from_array_element_name(element.name)
137
- except ValueError:
138
- if element.name.startswith("OBS"):
139
- collection = "sites"
140
- elif element.name in {"configuration_sim_telarray", "configuration_corsika"}:
141
- collection = element.name
142
- elif element.name == "Files":
143
- logger.info("Files are uploaded with the corresponding model parameters")
144
- continue
145
- logger.info(f"Reading model parameters for {element.name} into collection {collection}")
146
- files_to_insert = list(Path(element).rglob("*json"))
147
- for file in files_to_insert:
148
- if collection == "files":
149
- logger.info("Not yet implemented files")
150
- else:
151
- add_values_from_json_to_db(
152
- file=file,
153
- collection=collection,
154
- db=db,
155
- db_name=args_dict["db_name"],
156
- file_prefix=input_path / "Files",
157
- logger=logger,
158
- )
159
-
160
-
161
- def main():
162
- """Application main."""
163
- label = Path(__file__).stem
164
- args_dict, db_config = _parse(
165
- label, description="Add or update a model parameter database to the DB"
166
- )
167
- logger = logging.getLogger()
168
- logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
169
-
170
- db = db_handler.DatabaseHandler(mongo_db_config=db_config)
171
-
172
- _add_model_parameters_to_db(args_dict, db, logger)
173
-
174
-
175
- if __name__ == "__main__":
176
- main()
@@ -1,130 +0,0 @@
1
- """Retrieval of array elements from the database."""
2
-
3
- from functools import lru_cache
4
-
5
- from pymongo import ASCENDING
6
-
7
- from simtools.utils import names
8
-
9
-
10
- @lru_cache
11
- def get_array_elements(db_collection, model_version):
12
- """
13
- Get all array element names and their design model for a given DB collection.
14
-
15
- Uses the 'design_model' parameter to determine the design model of the array element.
16
- Assumes that a design model is defined for every array element.
17
-
18
- Parameters
19
- ----------
20
- db_collection:
21
- pymongo.collection.Collection
22
- model_version: str
23
- Model version.
24
-
25
- Returns
26
- -------
27
- dict
28
- Dict with array element names found and their design model
29
-
30
- Raises
31
- ------
32
- ValueError
33
- If query for collection name not implemented.
34
- KeyError
35
- If array element entry in the database is incomplete.
36
-
37
- """
38
- query = {"version": model_version}
39
- results = db_collection.find(query, {"instrument": 1, "value": 1, "parameter": 1}).sort(
40
- "instrument", ASCENDING
41
- )
42
-
43
- _all_available_array_elements = {}
44
- for doc in results:
45
- try:
46
- if doc["parameter"] == "design_model":
47
- _all_available_array_elements[doc["instrument"]] = doc["value"]
48
- except KeyError as exc:
49
- raise KeyError(f"Incomplete array element entry in the database: {doc}.") from exc
50
-
51
- if len(_all_available_array_elements) == 0:
52
- raise ValueError(f"No array elements found in DB collection {db_collection}.")
53
-
54
- return _all_available_array_elements
55
-
56
-
57
- def get_array_element_list_for_db_query(array_element_name, db, model_version, collection):
58
- """
59
- Get array element name and design model for querying the database.
60
-
61
- Return a list of array element names to be used for querying the database for a given array
62
- element. This is in most cases the array element itself and its design model.
63
- In cases of no design model available, the design model of the array element is returned.
64
-
65
- Parameters
66
- ----------
67
- array_element_name: str
68
- Name of the array element model (e.g. MSTN-01).
69
- db: DBHandler
70
- Instance of the database handler
71
- model_version: str
72
- Model version.
73
- collection: str
74
- DB collection to get the array elements from (e.g., telescopes, calibration_devices)
75
-
76
- Returns
77
- -------
78
- list
79
- List of array element model names as used in the DB.
80
-
81
- """
82
- try:
83
- _available_array_elements = get_array_elements(
84
- db.get_collection(db_name=None, collection_name=collection),
85
- db.model_version(model_version),
86
- )
87
- except ValueError:
88
- return [names.get_array_element_type_from_name(array_element_name) + "-design"]
89
- try:
90
- return [_available_array_elements[array_element_name], array_element_name]
91
- except KeyError:
92
- pass
93
-
94
- if array_element_name in _available_array_elements.values():
95
- return [array_element_name]
96
-
97
- raise ValueError(f"Array element {array_element_name} not found in DB.")
98
-
99
-
100
- def get_array_elements_of_type(array_element_type, db, model_version, collection):
101
- """
102
- Get all array elements of a certain type in the specified collection in the DB.
103
-
104
- Return e.g. for array_element_type='MSTS' all MSTS array elements found in the collection.
105
-
106
- Parameters
107
- ----------
108
- array_element_type : str
109
- Type of the array element (e.g. LSTN, MSTS)
110
- model_version : str
111
- Which version to get the array elements of
112
- collection : str
113
- Which collection to get the array elements from:
114
- i.e. telescopes, calibration_devices
115
- db_name : str
116
- Database name
117
-
118
- Returns
119
- -------
120
- list
121
- Sorted list of all array element names found in collection
122
-
123
- """
124
- _available_array_elements = get_array_elements(
125
- db.get_collection(db_name=None, collection_name=collection),
126
- db.model_version(model_version),
127
- )
128
- return sorted(
129
- [entry for entry in _available_array_elements if entry.startswith(array_element_type)]
130
- )