gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (355) hide show
  1. gammasimtools-0.8.2.dist-info/METADATA +173 -0
  2. gammasimtools-0.8.2.dist-info/RECORD +345 -0
  3. {gammasimtools-0.6.1.dist-info → gammasimtools-0.8.2.dist-info}/WHEEL +1 -1
  4. gammasimtools-0.8.2.dist-info/entry_points.txt +31 -0
  5. simtools/_dev_version/__init__.py +9 -0
  6. simtools/_version.py +2 -2
  7. simtools/applications/calculate_trigger_rate.py +210 -0
  8. simtools/applications/convert_all_model_parameters_from_simtel.py +372 -0
  9. simtools/applications/{print_array_elements.py → convert_geo_coordinates_of_array_elements.py} +58 -63
  10. simtools/applications/convert_model_parameter_from_simtel.py +119 -0
  11. simtools/applications/{add_file_to_db.py → db_add_file_to_db.py} +70 -60
  12. simtools/applications/db_add_model_parameters_from_repository_to_db.py +184 -0
  13. simtools/applications/db_add_value_from_json_to_db.py +105 -0
  14. simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +180 -0
  15. simtools/applications/db_get_array_layouts_from_db.py +162 -0
  16. simtools/applications/{get_file_from_db.py → db_get_file_from_db.py} +30 -34
  17. simtools/applications/db_get_parameter_from_db.py +131 -0
  18. simtools/applications/db_inspect_databases.py +52 -0
  19. simtools/applications/derive_mirror_rnda.py +39 -255
  20. simtools/applications/derive_psf_parameters.py +441 -0
  21. simtools/applications/generate_array_config.py +82 -0
  22. simtools/applications/generate_corsika_histograms.py +52 -52
  23. simtools/applications/generate_default_metadata.py +5 -8
  24. simtools/applications/generate_regular_arrays.py +117 -0
  25. simtools/applications/generate_simtel_array_histograms.py +97 -56
  26. simtools/applications/plot_array_layout.py +345 -115
  27. simtools/applications/production_generate_simulation_config.py +158 -0
  28. simtools/applications/production_scale_events.py +168 -0
  29. simtools/applications/simulate_light_emission.py +478 -0
  30. simtools/applications/simulate_prod.py +97 -175
  31. simtools/applications/submit_data_from_external.py +9 -12
  32. simtools/applications/submit_model_parameter_from_external.py +122 -0
  33. simtools/applications/validate_camera_efficiency.py +35 -102
  34. simtools/applications/validate_camera_fov.py +20 -19
  35. simtools/applications/{compare_cumulative_psf.py → validate_cumulative_psf.py} +45 -44
  36. simtools/applications/validate_file_using_schema.py +111 -47
  37. simtools/applications/validate_optics.py +17 -22
  38. simtools/camera_efficiency.py +193 -202
  39. simtools/configuration/commandline_parser.py +384 -96
  40. simtools/configuration/configurator.py +55 -71
  41. simtools/constants.py +5 -5
  42. simtools/corsika/corsika_config.py +482 -342
  43. simtools/corsika/corsika_histograms.py +226 -204
  44. simtools/corsika/corsika_histograms_visualize.py +23 -24
  45. simtools/corsika/primary_particle.py +159 -0
  46. simtools/data_model/data_reader.py +25 -20
  47. simtools/data_model/format_checkers.py +52 -0
  48. simtools/data_model/metadata_collector.py +211 -185
  49. simtools/data_model/metadata_model.py +115 -37
  50. simtools/data_model/model_data_writer.py +335 -26
  51. simtools/data_model/validate_data.py +366 -154
  52. simtools/db/db_array_elements.py +130 -0
  53. simtools/db/db_from_repo_handler.py +106 -0
  54. simtools/db/db_handler.py +1246 -0
  55. simtools/io_operations/hdf5_handler.py +3 -1
  56. simtools/io_operations/io_handler.py +32 -57
  57. simtools/job_execution/job_manager.py +82 -69
  58. simtools/layout/array_layout.py +325 -537
  59. simtools/layout/geo_coordinates.py +8 -11
  60. simtools/layout/telescope_position.py +163 -86
  61. simtools/model/array_model.py +312 -259
  62. simtools/model/calibration_model.py +50 -0
  63. simtools/model/camera.py +277 -523
  64. simtools/model/mirrors.py +68 -49
  65. simtools/model/model_parameter.py +602 -0
  66. simtools/model/model_utils.py +11 -39
  67. simtools/model/site_model.py +161 -0
  68. simtools/model/telescope_model.py +143 -633
  69. simtools/production_configuration/calculate_statistical_errors_grid_point.py +454 -0
  70. simtools/production_configuration/event_scaler.py +146 -0
  71. simtools/production_configuration/generate_simulation_config.py +193 -0
  72. simtools/production_configuration/interpolation_handler.py +197 -0
  73. simtools/ray_tracing/__init__.py +0 -0
  74. simtools/ray_tracing/mirror_panel_psf.py +280 -0
  75. simtools/{psf_analysis.py → ray_tracing/psf_analysis.py} +133 -47
  76. simtools/ray_tracing/ray_tracing.py +646 -0
  77. simtools/runners/__init__.py +0 -0
  78. simtools/runners/corsika_runner.py +240 -0
  79. simtools/runners/corsika_simtel_runner.py +225 -0
  80. simtools/runners/runner_services.py +307 -0
  81. simtools/runners/simtel_runner.py +224 -0
  82. simtools/schemas/array_elements.yml +137 -0
  83. simtools/schemas/integration_tests_config.metaschema.yml +93 -0
  84. simtools/schemas/metadata.metaschema.yml +6 -0
  85. simtools/schemas/model_parameter.metaschema.yml +78 -0
  86. simtools/schemas/{data.metaschema.yml → model_parameter_and_data_schema.metaschema.yml} +27 -44
  87. simtools/schemas/model_parameters/adjust_gain.schema.yml +37 -0
  88. simtools/schemas/model_parameters/altitude.schema.yml +37 -0
  89. simtools/schemas/model_parameters/array_coordinates.schema.yml +33 -0
  90. simtools/schemas/model_parameters/array_coordinates_UTM.schema.yml +77 -0
  91. simtools/schemas/model_parameters/array_element_position_ground.schema.yml +39 -0
  92. simtools/schemas/model_parameters/array_element_position_utm.schema.yml +39 -0
  93. simtools/schemas/model_parameters/array_layouts.schema.yml +48 -0
  94. simtools/schemas/model_parameters/array_triggers.schema.yml +93 -0
  95. simtools/schemas/model_parameters/asum_clipping.schema.yml +38 -0
  96. simtools/schemas/model_parameters/asum_offset.schema.yml +35 -0
  97. simtools/schemas/model_parameters/asum_shaping.schema.yml +35 -0
  98. simtools/schemas/model_parameters/asum_threshold.schema.yml +38 -0
  99. simtools/schemas/model_parameters/atmospheric_profile.schema.yml +32 -0
  100. simtools/schemas/model_parameters/atmospheric_transmission.schema.yml +35 -0
  101. simtools/schemas/model_parameters/axes_offsets.schema.yml +53 -0
  102. simtools/schemas/model_parameters/camera_body_diameter.schema.yml +40 -0
  103. simtools/schemas/model_parameters/camera_body_shape.schema.yml +45 -0
  104. simtools/schemas/model_parameters/camera_config_file.schema.yml +40 -0
  105. simtools/schemas/model_parameters/camera_config_rotate.schema.yml +36 -0
  106. simtools/schemas/model_parameters/camera_degraded_efficiency.schema.yml +43 -0
  107. simtools/schemas/model_parameters/camera_degraded_map.schema.yml +42 -0
  108. simtools/schemas/model_parameters/camera_depth.schema.yml +42 -0
  109. simtools/schemas/model_parameters/camera_filter.schema.yml +45 -0
  110. simtools/schemas/model_parameters/camera_filter_incidence_angle.schema.yml +29 -0
  111. simtools/schemas/model_parameters/camera_pixels.schema.yml +36 -0
  112. simtools/schemas/model_parameters/camera_transmission.schema.yml +41 -0
  113. simtools/schemas/model_parameters/channels_per_chip.schema.yml +36 -0
  114. simtools/schemas/model_parameters/correct_nsb_spectrum_to_telescope_altitude.schema.yml +35 -0
  115. simtools/schemas/model_parameters/corsika_cherenkov_photon_bunch_size.schema.yml +27 -0
  116. simtools/schemas/model_parameters/corsika_cherenkov_photon_wavelength_range.schema.yml +38 -0
  117. simtools/schemas/model_parameters/corsika_first_interaction_height.schema.yml +28 -0
  118. simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml +23 -0
  119. simtools/schemas/model_parameters/corsika_iact_max_bunches.schema.yml +27 -0
  120. simtools/schemas/model_parameters/corsika_iact_split_auto.schema.yml +28 -0
  121. simtools/schemas/model_parameters/corsika_longitudinal_shower_development.schema.yml +27 -0
  122. simtools/schemas/model_parameters/corsika_observation_level.schema.yml +38 -0
  123. simtools/schemas/model_parameters/corsika_particle_kinetic_energy_cutoff.schema.yml +52 -0
  124. simtools/schemas/model_parameters/corsika_starting_grammage.schema.yml +27 -0
  125. simtools/schemas/model_parameters/dark_events.schema.yml +32 -0
  126. simtools/schemas/model_parameters/default_trigger.schema.yml +35 -0
  127. simtools/schemas/model_parameters/design_model.schema.yml +31 -0
  128. simtools/schemas/model_parameters/disc_ac_coupled.schema.yml +32 -0
  129. simtools/schemas/model_parameters/disc_bins.schema.yml +39 -0
  130. simtools/schemas/model_parameters/disc_start.schema.yml +41 -0
  131. simtools/schemas/model_parameters/discriminator_amplitude.schema.yml +42 -0
  132. simtools/schemas/model_parameters/discriminator_fall_time.schema.yml +41 -0
  133. simtools/schemas/model_parameters/discriminator_gate_length.schema.yml +41 -0
  134. simtools/schemas/model_parameters/discriminator_hysteresis.schema.yml +39 -0
  135. simtools/schemas/model_parameters/discriminator_output_amplitude.schema.yml +40 -0
  136. simtools/schemas/model_parameters/discriminator_output_var_percent.schema.yml +41 -0
  137. simtools/schemas/model_parameters/discriminator_pulse_shape.schema.yml +33 -0
  138. simtools/schemas/model_parameters/discriminator_rise_time.schema.yml +42 -0
  139. simtools/schemas/model_parameters/discriminator_scale_threshold.schema.yml +37 -0
  140. simtools/schemas/model_parameters/discriminator_sigsum_over_threshold.schema.yml +44 -0
  141. simtools/schemas/model_parameters/discriminator_threshold.schema.yml +36 -0
  142. simtools/schemas/model_parameters/discriminator_time_over_threshold.schema.yml +45 -0
  143. simtools/schemas/model_parameters/discriminator_var_gate_length.schema.yml +40 -0
  144. simtools/schemas/model_parameters/discriminator_var_sigsum_over_threshold.schema.yml +41 -0
  145. simtools/schemas/model_parameters/discriminator_var_threshold.schema.yml +38 -0
  146. simtools/schemas/model_parameters/discriminator_var_time_over_threshold.schema.yml +38 -0
  147. simtools/schemas/model_parameters/dish_shape_length.schema.yml +41 -0
  148. simtools/schemas/model_parameters/dsum_clipping.schema.yml +38 -0
  149. simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +38 -0
  150. simtools/schemas/model_parameters/dsum_offset.schema.yml +37 -0
  151. simtools/schemas/model_parameters/dsum_pedsub.schema.yml +33 -0
  152. simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +39 -0
  153. simtools/schemas/model_parameters/dsum_prescale.schema.yml +44 -0
  154. simtools/schemas/model_parameters/dsum_presum_max.schema.yml +38 -0
  155. simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +45 -0
  156. simtools/schemas/model_parameters/dsum_shaping.schema.yml +44 -0
  157. simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +32 -0
  158. simtools/schemas/model_parameters/dsum_threshold.schema.yml +43 -0
  159. simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +42 -0
  160. simtools/schemas/model_parameters/effective_focal_length.schema.yml +61 -0
  161. simtools/schemas/model_parameters/epsg_code.schema.yml +37 -0
  162. simtools/schemas/model_parameters/fadc_ac_coupled.schema.yml +35 -0
  163. simtools/schemas/model_parameters/fadc_amplitude.schema.yml +46 -0
  164. simtools/schemas/model_parameters/fadc_bins.schema.yml +40 -0
  165. simtools/schemas/model_parameters/fadc_compensate_pedestal.schema.yml +50 -0
  166. simtools/schemas/model_parameters/fadc_dev_pedestal.schema.yml +38 -0
  167. simtools/schemas/model_parameters/fadc_err_compensate_pedestal.schema.yml +42 -0
  168. simtools/schemas/model_parameters/fadc_err_pedestal.schema.yml +49 -0
  169. simtools/schemas/model_parameters/fadc_lg_amplitude.schema.yml +47 -0
  170. simtools/schemas/model_parameters/fadc_lg_compensate_pedestal.schema.yml +51 -0
  171. simtools/schemas/model_parameters/fadc_lg_dev_pedestal.schema.yml +37 -0
  172. simtools/schemas/model_parameters/fadc_lg_err_compensate_pedestal.schema.yml +43 -0
  173. simtools/schemas/model_parameters/fadc_lg_err_pedestal.schema.yml +49 -0
  174. simtools/schemas/model_parameters/fadc_lg_max_signal.schema.yml +43 -0
  175. simtools/schemas/model_parameters/fadc_lg_max_sum.schema.yml +39 -0
  176. simtools/schemas/model_parameters/fadc_lg_noise.schema.yml +42 -0
  177. simtools/schemas/model_parameters/fadc_lg_pedestal.schema.yml +40 -0
  178. simtools/schemas/model_parameters/fadc_lg_sensitivity.schema.yml +50 -0
  179. simtools/schemas/model_parameters/fadc_lg_sysvar_pedestal.schema.yml +42 -0
  180. simtools/schemas/model_parameters/fadc_lg_var_pedestal.schema.yml +41 -0
  181. simtools/schemas/model_parameters/fadc_lg_var_sensitivity.schema.yml +42 -0
  182. simtools/schemas/model_parameters/fadc_max_signal.schema.yml +43 -0
  183. simtools/schemas/model_parameters/fadc_max_sum.schema.yml +39 -0
  184. simtools/schemas/model_parameters/fadc_mhz.schema.yml +31 -0
  185. simtools/schemas/model_parameters/fadc_noise.schema.yml +41 -0
  186. simtools/schemas/model_parameters/fadc_pedestal.schema.yml +40 -0
  187. simtools/schemas/model_parameters/fadc_pulse_shape.schema.yml +39 -0
  188. simtools/schemas/model_parameters/fadc_sensitivity.schema.yml +50 -0
  189. simtools/schemas/model_parameters/fadc_sum_bins.schema.yml +43 -0
  190. simtools/schemas/model_parameters/fadc_sum_offset.schema.yml +43 -0
  191. simtools/schemas/model_parameters/fadc_sysvar_pedestal.schema.yml +42 -0
  192. simtools/schemas/model_parameters/fadc_var_pedestal.schema.yml +41 -0
  193. simtools/schemas/model_parameters/fadc_var_sensitivity.schema.yml +42 -0
  194. simtools/schemas/model_parameters/flatfielding.schema.yml +37 -0
  195. simtools/schemas/model_parameters/focal_length.schema.yml +45 -0
  196. simtools/schemas/model_parameters/focal_surface_parameters.schema.yml +158 -0
  197. simtools/schemas/model_parameters/focal_surface_ref_radius.schema.yml +29 -0
  198. simtools/schemas/model_parameters/focus_offset.schema.yml +66 -0
  199. simtools/schemas/model_parameters/gain_variation.schema.yml +43 -0
  200. simtools/schemas/model_parameters/geomag_horizontal.schema.yml +34 -0
  201. simtools/schemas/model_parameters/geomag_rotation.schema.yml +37 -0
  202. simtools/schemas/model_parameters/geomag_vertical.schema.yml +34 -0
  203. simtools/schemas/model_parameters/hg_lg_variation.schema.yml +36 -0
  204. simtools/schemas/model_parameters/iobuf_maximum.schema.yml +34 -0
  205. simtools/schemas/model_parameters/iobuf_output_maximum.schema.yml +34 -0
  206. simtools/schemas/model_parameters/laser_events.schema.yml +36 -0
  207. simtools/schemas/model_parameters/laser_external_trigger.schema.yml +35 -0
  208. simtools/schemas/model_parameters/laser_photons.schema.yml +32 -0
  209. simtools/schemas/model_parameters/laser_pulse_exptime.schema.yml +34 -0
  210. simtools/schemas/model_parameters/laser_pulse_offset.schema.yml +34 -0
  211. simtools/schemas/model_parameters/laser_pulse_sigtime.schema.yml +33 -0
  212. simtools/schemas/model_parameters/laser_pulse_twidth.schema.yml +33 -0
  213. simtools/schemas/model_parameters/laser_var_photons.schema.yml +33 -0
  214. simtools/schemas/model_parameters/laser_wavelength.schema.yml +33 -0
  215. simtools/schemas/model_parameters/led_events.schema.yml +34 -0
  216. simtools/schemas/model_parameters/led_photons.schema.yml +34 -0
  217. simtools/schemas/model_parameters/led_pulse_offset.schema.yml +32 -0
  218. simtools/schemas/model_parameters/led_pulse_sigtime.schema.yml +33 -0
  219. simtools/schemas/model_parameters/led_var_photons.schema.yml +34 -0
  220. simtools/schemas/model_parameters/lightguide_efficiency_vs_incidence_angle.schema.yml +41 -0
  221. simtools/schemas/model_parameters/lightguide_efficiency_vs_wavelength.schema.yml +43 -0
  222. simtools/schemas/model_parameters/min_photoelectrons.schema.yml +35 -0
  223. simtools/schemas/model_parameters/min_photons.schema.yml +32 -0
  224. simtools/schemas/model_parameters/mirror_align_random_distance.schema.yml +36 -0
  225. simtools/schemas/model_parameters/mirror_align_random_horizontal.schema.yml +64 -0
  226. simtools/schemas/model_parameters/mirror_align_random_vertical.schema.yml +64 -0
  227. simtools/schemas/model_parameters/mirror_class.schema.yml +41 -0
  228. simtools/schemas/model_parameters/mirror_degraded_reflection.schema.yml +51 -0
  229. simtools/schemas/model_parameters/mirror_focal_length.schema.yml +42 -0
  230. simtools/schemas/model_parameters/mirror_list.schema.yml +38 -0
  231. simtools/schemas/model_parameters/mirror_offset.schema.yml +41 -0
  232. simtools/schemas/model_parameters/mirror_panel_2f_measurements.schema.yml +39 -0
  233. simtools/schemas/model_parameters/mirror_reflection_random_angle.schema.yml +61 -0
  234. simtools/schemas/model_parameters/mirror_reflectivity.schema.yml +40 -0
  235. simtools/schemas/model_parameters/multiplicity_offset.schema.yml +46 -0
  236. simtools/schemas/model_parameters/nsb_autoscale_airmass.schema.yml +51 -0
  237. simtools/schemas/model_parameters/nsb_gain_drop_scale.schema.yml +37 -0
  238. simtools/schemas/model_parameters/nsb_offaxis.schema.yml +79 -0
  239. simtools/schemas/model_parameters/nsb_pixel_rate.schema.yml +47 -0
  240. simtools/schemas/model_parameters/nsb_reference_spectrum.schema.yml +34 -0
  241. simtools/schemas/model_parameters/nsb_reference_value.schema.yml +33 -0
  242. simtools/schemas/model_parameters/nsb_scaling_factor.schema.yml +35 -0
  243. simtools/schemas/model_parameters/nsb_skymap.schema.yml +39 -0
  244. simtools/schemas/model_parameters/nsb_spectrum.schema.yml +50 -0
  245. simtools/schemas/model_parameters/num_gains.schema.yml +34 -0
  246. simtools/schemas/model_parameters/only_triggered_telescopes.schema.yml +33 -0
  247. simtools/schemas/model_parameters/optics_properties.schema.yml +31 -0
  248. simtools/schemas/model_parameters/parabolic_dish.schema.yml +32 -0
  249. simtools/schemas/model_parameters/pedestal_events.schema.yml +32 -0
  250. simtools/schemas/model_parameters/photon_delay.schema.yml +38 -0
  251. simtools/schemas/model_parameters/photons_per_run.schema.yml +33 -0
  252. simtools/schemas/model_parameters/pixel_cells.schema.yml +35 -0
  253. simtools/schemas/model_parameters/pixels_parallel.schema.yml +54 -0
  254. simtools/schemas/model_parameters/pixeltrg_time_step.schema.yml +40 -0
  255. simtools/schemas/model_parameters/pm_average_gain.schema.yml +34 -0
  256. simtools/schemas/model_parameters/pm_collection_efficiency.schema.yml +40 -0
  257. simtools/schemas/model_parameters/pm_gain_index.schema.yml +36 -0
  258. simtools/schemas/model_parameters/pm_photoelectron_spectrum.schema.yml +41 -0
  259. simtools/schemas/model_parameters/pm_transit_time.schema.yml +63 -0
  260. simtools/schemas/model_parameters/pm_voltage_variation.schema.yml +39 -0
  261. simtools/schemas/model_parameters/primary_mirror_degraded_map.schema.yml +42 -0
  262. simtools/schemas/model_parameters/primary_mirror_diameter.schema.yml +33 -0
  263. simtools/schemas/model_parameters/primary_mirror_hole_diameter.schema.yml +33 -0
  264. simtools/schemas/model_parameters/primary_mirror_incidence_angle.schema.yml +29 -0
  265. simtools/schemas/model_parameters/primary_mirror_parameters.schema.yml +168 -0
  266. simtools/schemas/model_parameters/primary_mirror_ref_radius.schema.yml +36 -0
  267. simtools/schemas/model_parameters/primary_mirror_segmentation.schema.yml +34 -0
  268. simtools/schemas/model_parameters/qe_variation.schema.yml +43 -0
  269. simtools/schemas/model_parameters/quantum_efficiency.schema.yml +42 -0
  270. simtools/schemas/model_parameters/random_focal_length.schema.yml +45 -0
  271. simtools/schemas/model_parameters/random_generator.schema.yml +36 -0
  272. simtools/schemas/model_parameters/reference_point_altitude.schema.yml +35 -0
  273. simtools/schemas/model_parameters/reference_point_latitude.schema.yml +36 -0
  274. simtools/schemas/model_parameters/reference_point_longitude.schema.yml +36 -0
  275. simtools/schemas/model_parameters/reference_point_utm_east.schema.yml +34 -0
  276. simtools/schemas/model_parameters/reference_point_utm_north.schema.yml +34 -0
  277. simtools/schemas/model_parameters/sampled_output.schema.yml +31 -0
  278. simtools/schemas/model_parameters/save_pe_with_amplitude.schema.yml +34 -0
  279. simtools/schemas/model_parameters/secondary_mirror_baffle.schema.yml +79 -0
  280. simtools/schemas/model_parameters/secondary_mirror_degraded_map.schema.yml +42 -0
  281. simtools/schemas/model_parameters/secondary_mirror_degraded_reflection.schema.yml +41 -0
  282. simtools/schemas/model_parameters/secondary_mirror_diameter.schema.yml +33 -0
  283. simtools/schemas/model_parameters/secondary_mirror_hole_diameter.schema.yml +36 -0
  284. simtools/schemas/model_parameters/secondary_mirror_incidence_angle.schema.yml +29 -0
  285. simtools/schemas/model_parameters/secondary_mirror_parameters.schema.yml +168 -0
  286. simtools/schemas/model_parameters/secondary_mirror_ref_radius.schema.yml +36 -0
  287. simtools/schemas/model_parameters/secondary_mirror_reflectivity.schema.yml +35 -0
  288. simtools/schemas/model_parameters/secondary_mirror_segmentation.schema.yml +37 -0
  289. simtools/schemas/model_parameters/secondary_mirror_shadow_diameter.schema.yml +40 -0
  290. simtools/schemas/model_parameters/secondary_mirror_shadow_offset.schema.yml +40 -0
  291. simtools/schemas/model_parameters/store_photoelectrons.schema.yml +41 -0
  292. simtools/schemas/model_parameters/tailcut_scale.schema.yml +40 -0
  293. simtools/schemas/model_parameters/telescope_axis_height.schema.yml +31 -0
  294. simtools/schemas/model_parameters/telescope_random_angle.schema.yml +35 -0
  295. simtools/schemas/model_parameters/telescope_random_error.schema.yml +34 -0
  296. simtools/schemas/model_parameters/telescope_sphere_radius.schema.yml +37 -0
  297. simtools/schemas/model_parameters/telescope_transmission.schema.yml +113 -0
  298. simtools/schemas/model_parameters/teltrig_min_sigsum.schema.yml +41 -0
  299. simtools/schemas/model_parameters/teltrig_min_time.schema.yml +36 -0
  300. simtools/schemas/model_parameters/transit_time_calib_error.schema.yml +36 -0
  301. simtools/schemas/model_parameters/transit_time_compensate_error.schema.yml +37 -0
  302. simtools/schemas/model_parameters/transit_time_compensate_step.schema.yml +38 -0
  303. simtools/schemas/model_parameters/transit_time_error.schema.yml +45 -0
  304. simtools/schemas/model_parameters/transit_time_jitter.schema.yml +36 -0
  305. simtools/schemas/model_parameters/trigger_current_limit.schema.yml +32 -0
  306. simtools/schemas/model_parameters/trigger_delay_compensation.schema.yml +53 -0
  307. simtools/schemas/model_parameters/trigger_pixels.schema.yml +40 -0
  308. simtools/simtel/simtel_config_reader.py +353 -0
  309. simtools/simtel/simtel_config_writer.py +244 -63
  310. simtools/simtel/{simtel_events.py → simtel_io_events.py} +26 -25
  311. simtools/simtel/simtel_io_histogram.py +661 -0
  312. simtools/simtel/simtel_io_histograms.py +569 -0
  313. simtools/simtel/simulator_array.py +145 -0
  314. simtools/simtel/{simtel_runner_camera_efficiency.py → simulator_camera_efficiency.py} +76 -52
  315. simtools/simtel/simulator_light_emission.py +473 -0
  316. simtools/simtel/simulator_ray_tracing.py +262 -0
  317. simtools/simulator.py +220 -446
  318. simtools/testing/__init__.py +0 -0
  319. simtools/testing/assertions.py +151 -0
  320. simtools/testing/configuration.py +226 -0
  321. simtools/testing/helpers.py +42 -0
  322. simtools/testing/validate_output.py +240 -0
  323. simtools/utils/general.py +340 -437
  324. simtools/utils/geometry.py +12 -12
  325. simtools/utils/names.py +257 -644
  326. simtools/utils/value_conversion.py +176 -0
  327. simtools/version.py +3 -1
  328. simtools/visualization/legend_handlers.py +135 -152
  329. simtools/visualization/plot_camera.py +379 -0
  330. simtools/visualization/visualize.py +346 -167
  331. gammasimtools-0.6.1.dist-info/METADATA +0 -180
  332. gammasimtools-0.6.1.dist-info/RECORD +0 -91
  333. gammasimtools-0.6.1.dist-info/entry_points.txt +0 -23
  334. simtools/_dev_version/scm_version.py +0 -10
  335. simtools/applications/db_development_tools/add_new_parameter_to_db.py +0 -81
  336. simtools/applications/db_development_tools/add_unit_to_parameter_in_db.py +0 -59
  337. simtools/applications/db_development_tools/mark_non_optics_parameters_non_applicable.py +0 -102
  338. simtools/applications/get_parameter.py +0 -92
  339. simtools/applications/make_regular_arrays.py +0 -160
  340. simtools/applications/produce_array_config.py +0 -136
  341. simtools/applications/production.py +0 -313
  342. simtools/applications/sim_showers_for_trigger_rates.py +0 -187
  343. simtools/applications/tune_psf.py +0 -334
  344. simtools/corsika/corsika_default_config.py +0 -282
  345. simtools/corsika/corsika_runner.py +0 -450
  346. simtools/corsika_simtel/corsika_simtel_runner.py +0 -197
  347. simtools/db_handler.py +0 -1480
  348. simtools/ray_tracing.py +0 -525
  349. simtools/simtel/simtel_histograms.py +0 -414
  350. simtools/simtel/simtel_runner.py +0 -244
  351. simtools/simtel/simtel_runner_array.py +0 -293
  352. simtools/simtel/simtel_runner_ray_tracing.py +0 -277
  353. {gammasimtools-0.6.1.dist-info → gammasimtools-0.8.2.dist-info}/LICENSE +0 -0
  354. {gammasimtools-0.6.1.dist-info → gammasimtools-0.8.2.dist-info}/top_level.txt +0 -0
  355. /simtools/{corsika_simtel → db}/__init__.py +0 -0
@@ -1,25 +1,28 @@
1
+ """Validation of data using schema."""
2
+
1
3
  import logging
2
4
  import os
3
5
  import re
4
6
  from pathlib import Path
5
7
 
8
+ import jsonschema
6
9
  import numpy as np
7
10
  from astropy import units as u
8
- from astropy.table import Table, unique
11
+ from astropy.table import Column, Table, unique
9
12
  from astropy.utils.diff import report_diff_values
10
13
 
11
14
  import simtools.utils.general as gen
15
+ from simtools.data_model import format_checkers
16
+ from simtools.utils import value_conversion
12
17
 
13
18
  __all__ = ["DataValidator"]
14
19
 
15
20
 
16
21
  class DataValidator:
17
22
  """
18
- Validate data for type and units following a describing schema; converts or
19
- transform data if required.
23
+ Validate data for type and units following a describing schema; converts or transform data.
20
24
 
21
- Data can be of table or list format
22
- (internally, all data is converted to astropy tables).
25
+ Data can be of table or dict format (internally, all data is converted to astropy tables).
23
26
 
24
27
  Parameters
25
28
  ----------
@@ -27,26 +30,41 @@ class DataValidator:
27
30
  Schema file describing input data and transformations.
28
31
  data_file: Path
29
32
  Input data file.
33
+ data_table: astropy.table
34
+ Input data table.
35
+ data_dict: dict
36
+ Input data dict.
37
+ check_exact_data_type: bool
38
+ Check for exact data type (default: True).
30
39
 
31
40
  """
32
41
 
33
- def __init__(self, schema_file=None, data_file=None, data_table=None):
34
- """
35
- Initialize validation class and read required reference data columns
36
-
37
- """
38
-
42
+ def __init__(
43
+ self,
44
+ schema_file=None,
45
+ data_file=None,
46
+ data_table=None,
47
+ data_dict=None,
48
+ check_exact_data_type=True,
49
+ ):
50
+ """Initialize validation class and read required reference data columns."""
39
51
  self._logger = logging.getLogger(__name__)
40
52
 
41
53
  self.data_file_name = data_file
42
54
  self.schema_file_name = schema_file
43
- self._reference_data_columns = None
44
- self.data = None
55
+ self._data_description = None
56
+ self.data_dict = data_dict
45
57
  self.data_table = data_table
58
+ self.check_exact_data_type = check_exact_data_type
46
59
 
47
- def validate_and_transform(self):
60
+ def validate_and_transform(self, is_model_parameter=False):
48
61
  """
49
- Data and data file validation.
62
+ Validate data and data file.
63
+
64
+ Parameters
65
+ ----------
66
+ is_model_parameter: bool
67
+ This is a model parameter (add some data preparation)
50
68
 
51
69
  Returns
52
70
  -------
@@ -59,30 +77,28 @@ class DataValidator:
59
77
  if no data or data table is available
60
78
 
61
79
  """
62
-
63
80
  if self.data_file_name:
64
81
  self.validate_data_file()
65
- if isinstance(self.data, dict):
82
+ if isinstance(self.data_dict, dict):
83
+ if is_model_parameter:
84
+ self._prepare_model_parameter()
66
85
  self._validate_data_dict()
67
- elif isinstance(self.data_table, Table):
86
+ return self.data_dict
87
+ if isinstance(self.data_table, Table):
68
88
  self._validate_data_table()
69
- else:
70
- self._logger.error("No data or data table to validate")
71
- raise TypeError
72
-
73
- return self.data_table
89
+ return self.data_table
90
+ self._logger.error("No data or data table to validate")
91
+ raise TypeError
74
92
 
75
93
  def validate_data_file(self):
76
94
  """
77
- Open data file and read data from file
78
- (doing this successfully is understood as
79
- file validation).
95
+ Open data file and read data from file.
80
96
 
97
+ Doing this successfully is understood as file validation.
81
98
  """
82
-
83
99
  try:
84
- if Path(self.data_file_name).suffix in (".yml", ".yaml"):
85
- self.data = gen.collect_data_from_file_or_dict(self.data_file_name, None)
100
+ if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
101
+ self.data_dict = gen.collect_data_from_file(self.data_file_name)
86
102
  self._logger.info(f"Validating data from: {self.data_file_name}")
87
103
  else:
88
104
  self.data_table = Table.read(self.data_file_name, guess=True, delimiter=r"\s")
@@ -90,53 +106,135 @@ class DataValidator:
90
106
  except (AttributeError, TypeError):
91
107
  pass
92
108
 
109
+ def validate_parameter_and_file_name(self):
110
+ """Validate that file name and key 'parameter_name' in data dict are the same."""
111
+ if self.data_dict.get("parameter") != Path(self.data_file_name).stem:
112
+ raise ValueError(
113
+ f"Parameter name in data dict {self.data_dict.get('parameter')} and "
114
+ f"file name {Path(self.data_file_name).stem} do not match."
115
+ )
116
+
93
117
  def _validate_data_dict(self):
94
118
  """
95
- Validate values. Creates first astropy table from data dict and then uses the same
96
- methods as for tabled data.
119
+ Validate values in a dictionary.
120
+
121
+ Handles different types of naming in data dicts (using 'name' or 'parameter'
122
+ keys for name fields).
123
+
124
+ Raises
125
+ ------
126
+ KeyError
127
+ if data dict does not contain a 'name' or 'parameter' key.
128
+
129
+ """
130
+ if not (_name := self.data_dict.get("name") or self.data_dict.get("parameter")):
131
+ raise KeyError("Data dict does not contain a 'name' or 'parameter' key.")
132
+ self._data_description = self._read_validation_schema(self.schema_file_name, _name)
133
+
134
+ value_as_list, unit_as_list = self._get_value_and_units_as_lists()
135
+
136
+ for index, (value, unit) in enumerate(zip(value_as_list, unit_as_list)):
137
+ value_as_list[index], unit_as_list[index] = self._validate_value_and_unit(
138
+ value, unit, index
139
+ )
140
+
141
+ if len(value_as_list) == 1:
142
+ self.data_dict["value"], self.data_dict["unit"] = value_as_list[0], unit_as_list[0]
143
+ else:
144
+ self.data_dict["value"], self.data_dict["unit"] = value_as_list, unit_as_list
97
145
 
146
+ self._check_version_string(self.data_dict.get("version"))
147
+
148
+ def _validate_value_and_unit(self, value, unit, index):
98
149
  """
150
+ Validate value, unit, and perform type checking and conversions.
99
151
 
100
- try:
101
- self._reference_data_columns = self._read_validation_schema(
102
- self.schema_file_name, self.data["name"]
152
+ Take into account different data types and allow to use json_schema for testing.
153
+ """
154
+ if self._get_data_description(index).get("type", None) == "dict":
155
+ self._validate_data_dict_using_json_schema(
156
+ self.data_dict["value"], self._get_data_description(index).get("json_schema")
103
157
  )
104
- _quantities = []
105
- for value, unit in zip(self.data["value"], self.data["units"]):
106
- try:
107
- _quantities.append(value * u.Unit(unit))
108
- except ValueError:
109
- _quantities.append(value)
110
- self.data_table = Table(rows=[_quantities])
111
- self.data_table.meta["name"] = self.data["name"]
112
- except KeyError as exc:
113
- raise KeyError("Data dict does not contain a 'name', 'value', or 'value' key.") from exc
114
-
115
- if self._reference_data_columns is not None:
116
- self._validate_data_columns()
158
+ else:
159
+ self._check_data_type(np.array(value).dtype, index)
117
160
 
118
- def _validate_data_table(self):
161
+ if self.data_dict.get("type") not in ("string", "dict", "file"):
162
+ self._check_for_not_a_number(value, index)
163
+ value, unit = self._check_and_convert_units(value, unit, index)
164
+ for range_type in ("allowed_range", "required_range"):
165
+ self._check_range(index, np.nanmin(value), np.nanmax(value), range_type)
166
+ return value, unit
167
+
168
+ def _get_value_and_units_as_lists(self):
169
+ """
170
+ Convert value and unit to lists if required.
171
+
172
+ Ignore unit field in data_dict if value is a astropy.Quantity.
173
+ Note the complications from astropy.Units, where a single value is of np.ndarray type.
174
+
175
+ Returns
176
+ -------
177
+ list
178
+ value as list
179
+ list
180
+ unit as list
181
+ """
182
+ target_unit = self.data_dict["unit"]
183
+ value, unit = value_conversion.split_value_and_unit(self.data_dict["value"])
184
+
185
+ if not isinstance(value, list | np.ndarray):
186
+ value, unit = [value], [unit]
187
+ if not isinstance(target_unit, list | np.ndarray):
188
+ target_unit = [target_unit] * len(value)
189
+
190
+ target_unit = [None if unit == "null" else unit for unit in target_unit]
191
+ conversion_factor = [
192
+ 1 if v is None else u.Unit(v).to(u.Unit(t)) for v, t in zip(unit, target_unit)
193
+ ]
194
+ return [v * c for v, c in zip(value, conversion_factor)], target_unit
195
+
196
+ def _validate_data_dict_using_json_schema(self, data, json_schema):
119
197
  """
120
- Validate tabulated data.
198
+ Validate a dictionary using a json schema.
121
199
 
200
+ Parameters
201
+ ----------
202
+ data: dict
203
+ Data dictionary
204
+ json_schema: dict
205
+ JSON schema
122
206
  """
207
+ if json_schema is None:
208
+ self._logger.debug("Skipping validation of dict type")
209
+ return
210
+ self._logger.debug("Validation of dict type using JSON schema")
211
+ try:
212
+ jsonschema.validate(data, json_schema, format_checker=format_checkers.format_checker)
213
+ except jsonschema.exceptions.ValidationError as exc:
214
+ self._logger.error(f"Validation error: {exc}")
215
+ raise exc
123
216
 
217
+ def _validate_data_table(self):
218
+ """Validate tabulated data."""
124
219
  try:
125
- self._reference_data_columns = self._read_validation_schema(self.schema_file_name)[
126
- 0
127
- ].get("table_columns", None)
220
+ self._data_description = self._read_validation_schema(self.schema_file_name)[0].get(
221
+ "table_columns", None
222
+ )
128
223
  except IndexError:
129
224
  self._logger.error(f"Error reading validation schema from {self.schema_file_name}")
130
225
  raise
131
226
 
132
- if self._reference_data_columns is not None:
227
+ if self._data_description is not None:
133
228
  self._validate_data_columns()
134
229
  self._check_data_for_duplicates()
135
230
  self._sort_data()
136
231
 
137
232
  def _validate_data_columns(self):
138
233
  """
139
- Validate that
234
+ Validate that data columns.
235
+
236
+ This includes:
237
+
140
238
  - required data columns are available
141
239
  - columns are in the correct units (if necessary apply a unit conversion)
142
240
  - ranges (minimum, maximum) are correct.
@@ -144,18 +242,17 @@ class DataValidator:
144
242
  This is not applied to columns of type 'string'.
145
243
 
146
244
  """
147
-
148
245
  self._check_required_columns()
149
246
 
150
247
  for col_name in self.data_table.colnames:
151
248
  col = self.data_table[col_name]
152
- if not self._get_reference_data_column(col_name, status_test=True):
249
+ if not self._get_data_description(col_name, status_test=True):
153
250
  continue
154
251
  if not np.issubdtype(col.dtype, np.number):
155
252
  continue
156
- self._check_for_not_a_number(col, col_name)
157
- self._check_data_type(col, col_name)
158
- col = self._check_and_convert_units(col, col_name)
253
+ self._check_for_not_a_number(col.data, col_name)
254
+ self._check_data_type(col.dtype, col_name)
255
+ self.data_table[col_name] = col.to(u.Unit(self._get_reference_unit(col_name)))
159
256
  self._check_range(col_name, np.nanmin(col.data), np.nanmax(col.data), "allowed_range")
160
257
  self._check_range(col_name, np.nanmin(col.data), np.nanmax(col.data), "required_range")
161
258
 
@@ -169,8 +266,7 @@ class DataValidator:
169
266
  if a required data column is missing
170
267
 
171
268
  """
172
-
173
- for entry in self._reference_data_columns:
269
+ for entry in self._data_description:
174
270
  if entry.get("required", False):
175
271
  if entry["name"] in self.data_table.columns:
176
272
  self._logger.debug(f"Found required data column {entry['name']}")
@@ -179,8 +275,9 @@ class DataValidator:
179
275
 
180
276
  def _sort_data(self):
181
277
  """
182
- Sort data according to one data column (if required by any column attribute). Data is
183
- either sorted or reverse sorted
278
+ Sort data according to one data column (if required by any column attribute).
279
+
280
+ Data is either sorted or reverse sorted.
184
281
 
185
282
  Raises
186
283
  ------
@@ -188,10 +285,9 @@ class DataValidator:
188
285
  if no table is defined for sorting
189
286
 
190
287
  """
191
-
192
288
  _columns_by_which_to_sort = []
193
289
  _columns_by_which_to_reverse_sort = []
194
- for entry in self._reference_data_columns:
290
+ for entry in self._data_description:
195
291
  if "input_processing" in entry:
196
292
  if "sort" in entry["input_processing"]:
197
293
  _columns_by_which_to_sort.append(entry["name"])
@@ -210,7 +306,7 @@ class DataValidator:
210
306
  try:
211
307
  self.data_table.sort(_columns_by_which_to_reverse_sort, reverse=True)
212
308
  except AttributeError:
213
- self._logger.error("No data table defined for sorting")
309
+ self._logger.error("No data table defined for reverse sorting")
214
310
  raise
215
311
 
216
312
  def _check_data_for_duplicates(self):
@@ -223,7 +319,6 @@ class DataValidator:
223
319
  checked for unique values.
224
320
 
225
321
  """
226
-
227
322
  _column_with_unique_requirement = self._get_unique_column_requirement()
228
323
  if len(_column_with_unique_requirement) == 0:
229
324
  self._logger.debug("No data columns with unique value requirement")
@@ -240,11 +335,10 @@ class DataValidator:
240
335
  ):
241
336
  self.data_table = unique(self.data_table)
242
337
  else:
243
- self._logger.error(
338
+ raise ValueError(
244
339
  "Failed removal of duplication for column "
245
340
  f"{_column_with_unique_requirement}, values are not unique"
246
341
  )
247
- raise ValueError
248
342
 
249
343
  def _get_unique_column_requirement(self):
250
344
  """
@@ -256,10 +350,9 @@ class DataValidator:
256
350
  list of data column with unique value requirement
257
351
 
258
352
  """
259
-
260
353
  _unique_required_column = []
261
354
 
262
- for entry in self._reference_data_columns:
355
+ for entry in self._data_description:
263
356
  if "input_processing" in entry and "remove_duplicates" in entry["input_processing"]:
264
357
  self._logger.debug(f"Removing duplicates for column {entry['name']}")
265
358
  _unique_required_column.append(entry["name"])
@@ -287,21 +380,20 @@ class DataValidator:
287
380
  if column name is not found in reference data columns
288
381
 
289
382
  """
290
-
291
- reference_unit = self._get_reference_data_column(column_name).get("units", None)
292
- if reference_unit == "dimensionless" or reference_unit is None:
383
+ reference_unit = self._get_data_description(column_name).get("unit", None)
384
+ if reference_unit in ("dimensionless", None, ""):
293
385
  return u.dimensionless_unscaled
294
386
 
295
387
  return u.Unit(reference_unit)
296
388
 
297
- def _check_data_type(self, col, column_name):
389
+ def _check_data_type(self, dtype, column_name):
298
390
  """
299
391
  Check column data type.
300
392
 
301
393
  Parameters
302
394
  ----------
303
- col: astropy.column or Quantity
304
- data column to be converted
395
+ dtype: numpy.dtype
396
+ data type
305
397
  column_name: str
306
398
  column name
307
399
 
@@ -311,26 +403,28 @@ class DataValidator:
311
403
  if data type is not correct
312
404
 
313
405
  """
314
-
315
- reference_dtype = self._get_reference_data_column(column_name).get("type", None)
316
-
317
- if not np.issubdtype(col.dtype, reference_dtype):
406
+ reference_dtype = self._get_data_description(column_name).get("type", None)
407
+ if not gen.validate_data_type(
408
+ reference_dtype=reference_dtype,
409
+ value=None,
410
+ dtype=dtype,
411
+ allow_subtypes=(not self.check_exact_data_type),
412
+ ):
318
413
  self._logger.error(
319
414
  f"Invalid data type in column '{column_name}'. "
320
- f"Expected type '{reference_dtype}', found '{col.dtype}'"
415
+ f"Expected type '{reference_dtype}', found '{dtype}' "
416
+ f"(exact type: {self.check_exact_data_type})"
321
417
  )
322
418
  raise TypeError
323
419
 
324
- self._logger.debug(f"Data column '{column_name}' has correct data type")
325
-
326
- def _check_for_not_a_number(self, col, col_name):
420
+ def _check_for_not_a_number(self, data, col_name):
327
421
  """
328
422
  Check that column values are finite and not NaN.
329
423
 
330
424
  Parameters
331
425
  ----------
332
- col: astropy.column or Quantity
333
- data column to be converted
426
+ data: value or numpy.ndarray
427
+ data to be tested
334
428
  col_name: str
335
429
  column name
336
430
 
@@ -345,26 +439,47 @@ class DataValidator:
345
439
  if at least one column value is NaN or Inf.
346
440
 
347
441
  """
442
+ if isinstance(data, str):
443
+ return True
444
+ if not isinstance(data, np.ndarray):
445
+ data = np.array(data)
348
446
 
349
- if np.isnan(col.data).any():
447
+ if np.isnan(data).any():
350
448
  self._logger.info(f"Column {col_name} contains NaN.")
351
- if np.isinf(col.data).any():
449
+ if np.isinf(data).any():
352
450
  self._logger.info(f"Column {col_name} contains infinite value.")
353
451
 
354
- entry = self._get_reference_data_column(col_name)
452
+ entry = self._get_data_description(col_name)
355
453
  if "allow_nan" in entry.get("input_processing", {}):
356
- return np.isnan(col.data).any() or np.isinf(col.data).any()
454
+ return np.isnan(data).any() or np.isinf(data).any()
357
455
 
358
- if np.isnan(col.data).any() or np.isinf(col.data).any():
359
- self._logger.error("NaN or Inf values found in data columns")
360
- raise ValueError
456
+ if np.isnan(data).any() or np.isinf(data).any():
457
+ raise ValueError("NaN or Inf values found in data")
361
458
 
362
459
  return False
363
460
 
364
- def _check_and_convert_units(self, col, col_name):
461
+ @staticmethod
462
+ def _is_dimensionless(unit):
365
463
  """
366
- Check that all columns have an allowed unit. Convert to reference unit (e.g., Angstrom to
367
- nm).
464
+ Check if unit is dimensionless, None, or empty.
465
+
466
+ Parameters
467
+ ----------
468
+ unit: str
469
+ unit of data column
470
+
471
+ Returns
472
+ -------
473
+ bool
474
+ True if unit is dimensionless, None, or empty
475
+ """
476
+ return unit in ("dimensionless", None, "")
477
+
478
+ def _check_and_convert_units(self, data, unit, col_name):
479
+ """
480
+ Check that input data have an allowed unit.
481
+
482
+ Convert to reference unit (e.g., Angstrom to nm).
368
483
 
369
484
  Note on dimensionless columns:
370
485
 
@@ -374,47 +489,91 @@ class DataValidator:
374
489
 
375
490
  Parameters
376
491
  ----------
377
- col: astropy.column or Quantity
378
- data column to be converted
492
+ data: astropy.column, Quantity, list, value
493
+ data to be converted
494
+ unit: str
495
+ unit of data column (read from column or Quantity if possible)
379
496
  col_name: str
380
497
  column name
381
498
 
382
499
  Returns
383
500
  -------
384
- astropy.column
385
- unit-converted data column
501
+ data: astropy.column, Quantity, list, value
502
+ unit-converted data
386
503
 
387
504
  Raises
388
505
  ------
389
506
  u.core.UnitConversionError
390
- If column unit conversions fails
507
+ If unit conversions fails
391
508
 
392
509
  """
393
-
394
510
  self._logger.debug(f"Checking data column '{col_name}'")
395
511
 
512
+ reference_unit = self._get_reference_unit(col_name)
396
513
  try:
397
- reference_unit = self._get_reference_unit(col_name)
398
- if col.unit is None or col.unit == "dimensionless":
399
- col.unit = u.dimensionless_unscaled
400
- return col
401
-
402
- self._logger.debug(
403
- f"Data column '{col_name}' with reference unit "
404
- f"'{reference_unit}' and data unit '{col.unit}'"
405
- )
406
- return u.Quantity(col).to(reference_unit)
407
- except u.core.UnitConversionError:
514
+ column_unit = data.unit
515
+ except AttributeError:
516
+ column_unit = unit
517
+
518
+ if self._is_dimensionless(column_unit) and self._is_dimensionless(reference_unit):
519
+ return data, u.dimensionless_unscaled
520
+
521
+ self._logger.debug(
522
+ f"Data column '{col_name}' with reference unit "
523
+ f"'{reference_unit}' and data unit '{column_unit}'"
524
+ )
525
+ try:
526
+ if isinstance(data, u.Quantity | Column):
527
+ return data.to(reference_unit), reference_unit
528
+
529
+ if isinstance(data, list | np.ndarray):
530
+ return self._check_and_convert_units_for_list(data, column_unit, reference_unit)
531
+
532
+ # ensure that the data type is preserved (e.g., integers)
533
+ return (type(data)(u.Unit(column_unit).to(reference_unit) * data), reference_unit)
534
+ except (u.core.UnitConversionError, ValueError) as exc:
408
535
  self._logger.error(
409
536
  f"Invalid unit in data column '{col_name}'. "
410
- f"Expected type '{reference_unit}', found '{col.unit}'"
537
+ f"Expected type '{reference_unit}', found '{column_unit}'"
411
538
  )
412
- raise
539
+ raise u.core.UnitConversionError from exc
540
+
541
+ def _check_and_convert_units_for_list(self, data, column_unit, reference_unit):
542
+ """
543
+ Check and convert units data in a list or or numpy array.
544
+
545
+ Takes into account that data can be dimensionless (with unit 'None', 'dimensionless'
546
+ or '').
547
+
548
+ Parameters
549
+ ----------
550
+ data: list
551
+ list of data
552
+ column_unit: str
553
+ unit of data column
554
+ reference_unit: str
555
+ reference unit
556
+
557
+ Returns
558
+ -------
559
+ list
560
+ converted data
561
+
562
+ """
563
+ return [
564
+ (
565
+ u.Unit(_to_unit).to(reference_unit) * d
566
+ if _to_unit not in (None, "dimensionless", "")
567
+ else d
568
+ )
569
+ for d, _to_unit in zip(data, column_unit)
570
+ ], reference_unit
413
571
 
414
572
  def _check_range(self, col_name, col_min, col_max, range_type="allowed_range"):
415
573
  """
416
- Check that column data is within allowed range or required range. Assumes that column and
417
- ranges have the same units.
574
+ Check that column data is within allowed range or required range.
575
+
576
+ Assumes that column and ranges have the same units.
418
577
 
419
578
  Parameters
420
579
  ----------
@@ -439,42 +598,32 @@ class DataValidator:
439
598
  """
440
599
  self._logger.debug(f"Checking data in column '{col_name}' for '{range_type}' ")
441
600
 
442
- try:
443
- if range_type not in ("allowed_range", "required_range"):
444
- raise KeyError
445
- except KeyError:
446
- self._logger.error("Allowed range types are 'allowed_range', 'required_range'")
447
- raise
601
+ if range_type not in ("allowed_range", "required_range"):
602
+ raise KeyError("Allowed range types are 'allowed_range', 'required_range'")
448
603
 
449
- _entry = self._get_reference_data_column(col_name)
604
+ _entry = self._get_data_description(col_name)
450
605
  if range_type not in _entry:
451
- return None
606
+ return
452
607
 
453
- try:
454
- if not self._interval_check(
455
- (col_min, col_max),
456
- (_entry[range_type].get("min", np.NINF), _entry[range_type].get("max", np.Inf)),
457
- range_type,
458
- ):
459
- raise ValueError
460
- except KeyError:
461
- self._logger.error(f"Invalid range ('{range_type}') definition for column '{col_name}'")
462
- except ValueError:
463
- self._logger.error(
608
+ if not self._interval_check(
609
+ (col_min, col_max),
610
+ (_entry[range_type].get("min", -np.inf), _entry[range_type].get("max", np.inf)),
611
+ range_type,
612
+ ):
613
+ raise ValueError(
464
614
  f"Value for column '{col_name}' out of range. "
465
615
  f"([{col_min}, {col_max}], {range_type}: "
466
- f"[{_entry[range_type].get('min', np.NINF)}, "
467
- f"{_entry[range_type].get('max', np.Inf)}])"
616
+ f"[{_entry[range_type].get('min', -np.inf)}, "
617
+ f"{_entry[range_type].get('max', np.inf)}])"
468
618
  )
469
- raise
470
-
471
- return None
472
619
 
473
620
  @staticmethod
474
621
  def _interval_check(data, axis_range, range_type):
475
622
  """
476
- Check that values are inside allowed range (range_type='allowed_range') or span at least
477
- the given interval (range_type='required_range').
623
+ Range checking for a given set of data.
624
+
625
+ Check that values are inside allowed range or interval. This(range_type='allowed_range')
626
+ or span at least the given interval (range_type='required_range').
478
627
 
479
628
  Parameters
480
629
  ----------
@@ -491,7 +640,6 @@ class DataValidator:
491
640
  True if range test is passed
492
641
 
493
642
  """
494
-
495
643
  if range_type == "allowed_range":
496
644
  if data[0] >= axis_range[0] and data[1] <= axis_range[1]:
497
645
  return True
@@ -526,21 +674,23 @@ class DataValidator:
526
674
  if 'data' can not be read from dict in schema file
527
675
 
528
676
  """
529
-
530
677
  try:
531
678
  if Path(schema_file).is_dir():
532
- return gen.collect_data_from_file_or_dict(
679
+ return gen.collect_data_from_file(
533
680
  file_name=Path(schema_file) / (parameter + ".schema.yml"),
534
- in_dict=None,
535
681
  )["data"]
536
- return gen.collect_data_from_file_or_dict(file_name=schema_file, in_dict=None)["data"]
682
+ return gen.collect_data_from_file(file_name=schema_file)["data"]
537
683
  except KeyError:
538
684
  self._logger.error(f"Error reading validation schema from {schema_file}")
539
685
  raise
540
686
 
541
- def _get_reference_data_column(self, column_name, status_test=False):
687
+ def _get_data_description(self, column_name=None, status_test=False):
542
688
  """
543
- Return entry in reference data for a given column name.
689
+ Return data description as provided by the schema file.
690
+
691
+ For tables (type: 'data_table'), return the description of
692
+ the column named 'column_name'. For other types, return
693
+ all data descriptions.
544
694
  For columns named 'colX' return the Xth column in the reference data.
545
695
 
546
696
  Parameters
@@ -557,20 +707,38 @@ class DataValidator:
557
707
  bool
558
708
  True if reference column exists (for status_test==True).
559
709
 
560
-
561
710
  Raises
562
711
  ------
563
712
  IndexError
564
713
  If data column is not found.
565
714
 
566
715
  """
716
+ self._logger.debug(
717
+ f"Getting reference data column {column_name} from schema {self._data_description}"
718
+ )
719
+ try:
720
+ return (
721
+ self._data_description[column_name]
722
+ if not status_test
723
+ else (
724
+ self._data_description[column_name] is not None
725
+ and len(self._data_description) > 0
726
+ )
727
+ )
728
+ except IndexError as exc:
729
+ self._logger.error(
730
+ f"Data column '{column_name}' not found in reference column definition"
731
+ )
732
+ raise exc
733
+ except TypeError:
734
+ pass # column_name is not an integer
567
735
 
568
736
  _index = 0
569
737
  if bool(re.match(r"^col\d$", column_name)):
570
738
  _index = int(column_name[3:])
571
- _entry = self._reference_data_columns
739
+ _entry = self._data_description
572
740
  else:
573
- _entry = [item for item in self._reference_data_columns if item["name"] == column_name]
741
+ _entry = [item for item in self._data_description if item["name"] == column_name]
574
742
  if status_test:
575
743
  return len(_entry) > 0
576
744
  try:
@@ -580,3 +748,47 @@ class DataValidator:
580
748
  f"Data column '{column_name}' not found in reference column definition"
581
749
  )
582
750
  raise
751
+
752
+ def _prepare_model_parameter(self):
753
+ """
754
+ Apply data preparation for model parameters.
755
+
756
+ Converts strings to numerical values or lists of values, if required.
757
+
758
+ """
759
+ value = self.data_dict["value"]
760
+ if not isinstance(value, str):
761
+ return
762
+
763
+ # assume float value if type is not defined
764
+ _is_float = self.data_dict.get("type", "float").startswith(("float", "double"))
765
+
766
+ if value.isnumeric():
767
+ self.data_dict["value"] = float(value) if _is_float else int(value)
768
+ else:
769
+ self.data_dict["value"] = gen.convert_string_to_list(value, is_float=_is_float)
770
+
771
+ if self.data_dict["unit"] is not None:
772
+ self.data_dict["unit"] = gen.convert_string_to_list(self.data_dict["unit"])
773
+
774
+ def _check_version_string(self, version):
775
+ """
776
+ Check that version string follows semantic versioning.
777
+
778
+ Parameters
779
+ ----------
780
+ version: str
781
+ version string
782
+
783
+ Raises
784
+ ------
785
+ ValueError
786
+ if version string does not follow semantic versioning
787
+
788
+ """
789
+ if version is None:
790
+ return
791
+ semver_regex = r"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$"
792
+ if not re.match(semver_regex, version):
793
+ raise ValueError(f"Invalid version string '{version}'")
794
+ self._logger.debug(f"Valid version string '{version}'")