gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gammasimtools-0.8.2.dist-info/METADATA +173 -0
- gammasimtools-0.8.2.dist-info/RECORD +345 -0
- {gammasimtools-0.6.1.dist-info → gammasimtools-0.8.2.dist-info}/WHEEL +1 -1
- gammasimtools-0.8.2.dist-info/entry_points.txt +31 -0
- simtools/_dev_version/__init__.py +9 -0
- simtools/_version.py +2 -2
- simtools/applications/calculate_trigger_rate.py +210 -0
- simtools/applications/convert_all_model_parameters_from_simtel.py +372 -0
- simtools/applications/{print_array_elements.py → convert_geo_coordinates_of_array_elements.py} +58 -63
- simtools/applications/convert_model_parameter_from_simtel.py +119 -0
- simtools/applications/{add_file_to_db.py → db_add_file_to_db.py} +70 -60
- simtools/applications/db_add_model_parameters_from_repository_to_db.py +184 -0
- simtools/applications/db_add_value_from_json_to_db.py +105 -0
- simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +180 -0
- simtools/applications/db_get_array_layouts_from_db.py +162 -0
- simtools/applications/{get_file_from_db.py → db_get_file_from_db.py} +30 -34
- simtools/applications/db_get_parameter_from_db.py +131 -0
- simtools/applications/db_inspect_databases.py +52 -0
- simtools/applications/derive_mirror_rnda.py +39 -255
- simtools/applications/derive_psf_parameters.py +441 -0
- simtools/applications/generate_array_config.py +82 -0
- simtools/applications/generate_corsika_histograms.py +52 -52
- simtools/applications/generate_default_metadata.py +5 -8
- simtools/applications/generate_regular_arrays.py +117 -0
- simtools/applications/generate_simtel_array_histograms.py +97 -56
- simtools/applications/plot_array_layout.py +345 -115
- simtools/applications/production_generate_simulation_config.py +158 -0
- simtools/applications/production_scale_events.py +168 -0
- simtools/applications/simulate_light_emission.py +478 -0
- simtools/applications/simulate_prod.py +97 -175
- simtools/applications/submit_data_from_external.py +9 -12
- simtools/applications/submit_model_parameter_from_external.py +122 -0
- simtools/applications/validate_camera_efficiency.py +35 -102
- simtools/applications/validate_camera_fov.py +20 -19
- simtools/applications/{compare_cumulative_psf.py → validate_cumulative_psf.py} +45 -44
- simtools/applications/validate_file_using_schema.py +111 -47
- simtools/applications/validate_optics.py +17 -22
- simtools/camera_efficiency.py +193 -202
- simtools/configuration/commandline_parser.py +384 -96
- simtools/configuration/configurator.py +55 -71
- simtools/constants.py +5 -5
- simtools/corsika/corsika_config.py +482 -342
- simtools/corsika/corsika_histograms.py +226 -204
- simtools/corsika/corsika_histograms_visualize.py +23 -24
- simtools/corsika/primary_particle.py +159 -0
- simtools/data_model/data_reader.py +25 -20
- simtools/data_model/format_checkers.py +52 -0
- simtools/data_model/metadata_collector.py +211 -185
- simtools/data_model/metadata_model.py +115 -37
- simtools/data_model/model_data_writer.py +335 -26
- simtools/data_model/validate_data.py +366 -154
- simtools/db/db_array_elements.py +130 -0
- simtools/db/db_from_repo_handler.py +106 -0
- simtools/db/db_handler.py +1246 -0
- simtools/io_operations/hdf5_handler.py +3 -1
- simtools/io_operations/io_handler.py +32 -57
- simtools/job_execution/job_manager.py +82 -69
- simtools/layout/array_layout.py +325 -537
- simtools/layout/geo_coordinates.py +8 -11
- simtools/layout/telescope_position.py +163 -86
- simtools/model/array_model.py +312 -259
- simtools/model/calibration_model.py +50 -0
- simtools/model/camera.py +277 -523
- simtools/model/mirrors.py +68 -49
- simtools/model/model_parameter.py +602 -0
- simtools/model/model_utils.py +11 -39
- simtools/model/site_model.py +161 -0
- simtools/model/telescope_model.py +143 -633
- simtools/production_configuration/calculate_statistical_errors_grid_point.py +454 -0
- simtools/production_configuration/event_scaler.py +146 -0
- simtools/production_configuration/generate_simulation_config.py +193 -0
- simtools/production_configuration/interpolation_handler.py +197 -0
- simtools/ray_tracing/__init__.py +0 -0
- simtools/ray_tracing/mirror_panel_psf.py +280 -0
- simtools/{psf_analysis.py → ray_tracing/psf_analysis.py} +133 -47
- simtools/ray_tracing/ray_tracing.py +646 -0
- simtools/runners/__init__.py +0 -0
- simtools/runners/corsika_runner.py +240 -0
- simtools/runners/corsika_simtel_runner.py +225 -0
- simtools/runners/runner_services.py +307 -0
- simtools/runners/simtel_runner.py +224 -0
- simtools/schemas/array_elements.yml +137 -0
- simtools/schemas/integration_tests_config.metaschema.yml +93 -0
- simtools/schemas/metadata.metaschema.yml +6 -0
- simtools/schemas/model_parameter.metaschema.yml +78 -0
- simtools/schemas/{data.metaschema.yml → model_parameter_and_data_schema.metaschema.yml} +27 -44
- simtools/schemas/model_parameters/adjust_gain.schema.yml +37 -0
- simtools/schemas/model_parameters/altitude.schema.yml +37 -0
- simtools/schemas/model_parameters/array_coordinates.schema.yml +33 -0
- simtools/schemas/model_parameters/array_coordinates_UTM.schema.yml +77 -0
- simtools/schemas/model_parameters/array_element_position_ground.schema.yml +39 -0
- simtools/schemas/model_parameters/array_element_position_utm.schema.yml +39 -0
- simtools/schemas/model_parameters/array_layouts.schema.yml +48 -0
- simtools/schemas/model_parameters/array_triggers.schema.yml +93 -0
- simtools/schemas/model_parameters/asum_clipping.schema.yml +38 -0
- simtools/schemas/model_parameters/asum_offset.schema.yml +35 -0
- simtools/schemas/model_parameters/asum_shaping.schema.yml +35 -0
- simtools/schemas/model_parameters/asum_threshold.schema.yml +38 -0
- simtools/schemas/model_parameters/atmospheric_profile.schema.yml +32 -0
- simtools/schemas/model_parameters/atmospheric_transmission.schema.yml +35 -0
- simtools/schemas/model_parameters/axes_offsets.schema.yml +53 -0
- simtools/schemas/model_parameters/camera_body_diameter.schema.yml +40 -0
- simtools/schemas/model_parameters/camera_body_shape.schema.yml +45 -0
- simtools/schemas/model_parameters/camera_config_file.schema.yml +40 -0
- simtools/schemas/model_parameters/camera_config_rotate.schema.yml +36 -0
- simtools/schemas/model_parameters/camera_degraded_efficiency.schema.yml +43 -0
- simtools/schemas/model_parameters/camera_degraded_map.schema.yml +42 -0
- simtools/schemas/model_parameters/camera_depth.schema.yml +42 -0
- simtools/schemas/model_parameters/camera_filter.schema.yml +45 -0
- simtools/schemas/model_parameters/camera_filter_incidence_angle.schema.yml +29 -0
- simtools/schemas/model_parameters/camera_pixels.schema.yml +36 -0
- simtools/schemas/model_parameters/camera_transmission.schema.yml +41 -0
- simtools/schemas/model_parameters/channels_per_chip.schema.yml +36 -0
- simtools/schemas/model_parameters/correct_nsb_spectrum_to_telescope_altitude.schema.yml +35 -0
- simtools/schemas/model_parameters/corsika_cherenkov_photon_bunch_size.schema.yml +27 -0
- simtools/schemas/model_parameters/corsika_cherenkov_photon_wavelength_range.schema.yml +38 -0
- simtools/schemas/model_parameters/corsika_first_interaction_height.schema.yml +28 -0
- simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml +23 -0
- simtools/schemas/model_parameters/corsika_iact_max_bunches.schema.yml +27 -0
- simtools/schemas/model_parameters/corsika_iact_split_auto.schema.yml +28 -0
- simtools/schemas/model_parameters/corsika_longitudinal_shower_development.schema.yml +27 -0
- simtools/schemas/model_parameters/corsika_observation_level.schema.yml +38 -0
- simtools/schemas/model_parameters/corsika_particle_kinetic_energy_cutoff.schema.yml +52 -0
- simtools/schemas/model_parameters/corsika_starting_grammage.schema.yml +27 -0
- simtools/schemas/model_parameters/dark_events.schema.yml +32 -0
- simtools/schemas/model_parameters/default_trigger.schema.yml +35 -0
- simtools/schemas/model_parameters/design_model.schema.yml +31 -0
- simtools/schemas/model_parameters/disc_ac_coupled.schema.yml +32 -0
- simtools/schemas/model_parameters/disc_bins.schema.yml +39 -0
- simtools/schemas/model_parameters/disc_start.schema.yml +41 -0
- simtools/schemas/model_parameters/discriminator_amplitude.schema.yml +42 -0
- simtools/schemas/model_parameters/discriminator_fall_time.schema.yml +41 -0
- simtools/schemas/model_parameters/discriminator_gate_length.schema.yml +41 -0
- simtools/schemas/model_parameters/discriminator_hysteresis.schema.yml +39 -0
- simtools/schemas/model_parameters/discriminator_output_amplitude.schema.yml +40 -0
- simtools/schemas/model_parameters/discriminator_output_var_percent.schema.yml +41 -0
- simtools/schemas/model_parameters/discriminator_pulse_shape.schema.yml +33 -0
- simtools/schemas/model_parameters/discriminator_rise_time.schema.yml +42 -0
- simtools/schemas/model_parameters/discriminator_scale_threshold.schema.yml +37 -0
- simtools/schemas/model_parameters/discriminator_sigsum_over_threshold.schema.yml +44 -0
- simtools/schemas/model_parameters/discriminator_threshold.schema.yml +36 -0
- simtools/schemas/model_parameters/discriminator_time_over_threshold.schema.yml +45 -0
- simtools/schemas/model_parameters/discriminator_var_gate_length.schema.yml +40 -0
- simtools/schemas/model_parameters/discriminator_var_sigsum_over_threshold.schema.yml +41 -0
- simtools/schemas/model_parameters/discriminator_var_threshold.schema.yml +38 -0
- simtools/schemas/model_parameters/discriminator_var_time_over_threshold.schema.yml +38 -0
- simtools/schemas/model_parameters/dish_shape_length.schema.yml +41 -0
- simtools/schemas/model_parameters/dsum_clipping.schema.yml +38 -0
- simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +38 -0
- simtools/schemas/model_parameters/dsum_offset.schema.yml +37 -0
- simtools/schemas/model_parameters/dsum_pedsub.schema.yml +33 -0
- simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +39 -0
- simtools/schemas/model_parameters/dsum_prescale.schema.yml +44 -0
- simtools/schemas/model_parameters/dsum_presum_max.schema.yml +38 -0
- simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +45 -0
- simtools/schemas/model_parameters/dsum_shaping.schema.yml +44 -0
- simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +32 -0
- simtools/schemas/model_parameters/dsum_threshold.schema.yml +43 -0
- simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +42 -0
- simtools/schemas/model_parameters/effective_focal_length.schema.yml +61 -0
- simtools/schemas/model_parameters/epsg_code.schema.yml +37 -0
- simtools/schemas/model_parameters/fadc_ac_coupled.schema.yml +35 -0
- simtools/schemas/model_parameters/fadc_amplitude.schema.yml +46 -0
- simtools/schemas/model_parameters/fadc_bins.schema.yml +40 -0
- simtools/schemas/model_parameters/fadc_compensate_pedestal.schema.yml +50 -0
- simtools/schemas/model_parameters/fadc_dev_pedestal.schema.yml +38 -0
- simtools/schemas/model_parameters/fadc_err_compensate_pedestal.schema.yml +42 -0
- simtools/schemas/model_parameters/fadc_err_pedestal.schema.yml +49 -0
- simtools/schemas/model_parameters/fadc_lg_amplitude.schema.yml +47 -0
- simtools/schemas/model_parameters/fadc_lg_compensate_pedestal.schema.yml +51 -0
- simtools/schemas/model_parameters/fadc_lg_dev_pedestal.schema.yml +37 -0
- simtools/schemas/model_parameters/fadc_lg_err_compensate_pedestal.schema.yml +43 -0
- simtools/schemas/model_parameters/fadc_lg_err_pedestal.schema.yml +49 -0
- simtools/schemas/model_parameters/fadc_lg_max_signal.schema.yml +43 -0
- simtools/schemas/model_parameters/fadc_lg_max_sum.schema.yml +39 -0
- simtools/schemas/model_parameters/fadc_lg_noise.schema.yml +42 -0
- simtools/schemas/model_parameters/fadc_lg_pedestal.schema.yml +40 -0
- simtools/schemas/model_parameters/fadc_lg_sensitivity.schema.yml +50 -0
- simtools/schemas/model_parameters/fadc_lg_sysvar_pedestal.schema.yml +42 -0
- simtools/schemas/model_parameters/fadc_lg_var_pedestal.schema.yml +41 -0
- simtools/schemas/model_parameters/fadc_lg_var_sensitivity.schema.yml +42 -0
- simtools/schemas/model_parameters/fadc_max_signal.schema.yml +43 -0
- simtools/schemas/model_parameters/fadc_max_sum.schema.yml +39 -0
- simtools/schemas/model_parameters/fadc_mhz.schema.yml +31 -0
- simtools/schemas/model_parameters/fadc_noise.schema.yml +41 -0
- simtools/schemas/model_parameters/fadc_pedestal.schema.yml +40 -0
- simtools/schemas/model_parameters/fadc_pulse_shape.schema.yml +39 -0
- simtools/schemas/model_parameters/fadc_sensitivity.schema.yml +50 -0
- simtools/schemas/model_parameters/fadc_sum_bins.schema.yml +43 -0
- simtools/schemas/model_parameters/fadc_sum_offset.schema.yml +43 -0
- simtools/schemas/model_parameters/fadc_sysvar_pedestal.schema.yml +42 -0
- simtools/schemas/model_parameters/fadc_var_pedestal.schema.yml +41 -0
- simtools/schemas/model_parameters/fadc_var_sensitivity.schema.yml +42 -0
- simtools/schemas/model_parameters/flatfielding.schema.yml +37 -0
- simtools/schemas/model_parameters/focal_length.schema.yml +45 -0
- simtools/schemas/model_parameters/focal_surface_parameters.schema.yml +158 -0
- simtools/schemas/model_parameters/focal_surface_ref_radius.schema.yml +29 -0
- simtools/schemas/model_parameters/focus_offset.schema.yml +66 -0
- simtools/schemas/model_parameters/gain_variation.schema.yml +43 -0
- simtools/schemas/model_parameters/geomag_horizontal.schema.yml +34 -0
- simtools/schemas/model_parameters/geomag_rotation.schema.yml +37 -0
- simtools/schemas/model_parameters/geomag_vertical.schema.yml +34 -0
- simtools/schemas/model_parameters/hg_lg_variation.schema.yml +36 -0
- simtools/schemas/model_parameters/iobuf_maximum.schema.yml +34 -0
- simtools/schemas/model_parameters/iobuf_output_maximum.schema.yml +34 -0
- simtools/schemas/model_parameters/laser_events.schema.yml +36 -0
- simtools/schemas/model_parameters/laser_external_trigger.schema.yml +35 -0
- simtools/schemas/model_parameters/laser_photons.schema.yml +32 -0
- simtools/schemas/model_parameters/laser_pulse_exptime.schema.yml +34 -0
- simtools/schemas/model_parameters/laser_pulse_offset.schema.yml +34 -0
- simtools/schemas/model_parameters/laser_pulse_sigtime.schema.yml +33 -0
- simtools/schemas/model_parameters/laser_pulse_twidth.schema.yml +33 -0
- simtools/schemas/model_parameters/laser_var_photons.schema.yml +33 -0
- simtools/schemas/model_parameters/laser_wavelength.schema.yml +33 -0
- simtools/schemas/model_parameters/led_events.schema.yml +34 -0
- simtools/schemas/model_parameters/led_photons.schema.yml +34 -0
- simtools/schemas/model_parameters/led_pulse_offset.schema.yml +32 -0
- simtools/schemas/model_parameters/led_pulse_sigtime.schema.yml +33 -0
- simtools/schemas/model_parameters/led_var_photons.schema.yml +34 -0
- simtools/schemas/model_parameters/lightguide_efficiency_vs_incidence_angle.schema.yml +41 -0
- simtools/schemas/model_parameters/lightguide_efficiency_vs_wavelength.schema.yml +43 -0
- simtools/schemas/model_parameters/min_photoelectrons.schema.yml +35 -0
- simtools/schemas/model_parameters/min_photons.schema.yml +32 -0
- simtools/schemas/model_parameters/mirror_align_random_distance.schema.yml +36 -0
- simtools/schemas/model_parameters/mirror_align_random_horizontal.schema.yml +64 -0
- simtools/schemas/model_parameters/mirror_align_random_vertical.schema.yml +64 -0
- simtools/schemas/model_parameters/mirror_class.schema.yml +41 -0
- simtools/schemas/model_parameters/mirror_degraded_reflection.schema.yml +51 -0
- simtools/schemas/model_parameters/mirror_focal_length.schema.yml +42 -0
- simtools/schemas/model_parameters/mirror_list.schema.yml +38 -0
- simtools/schemas/model_parameters/mirror_offset.schema.yml +41 -0
- simtools/schemas/model_parameters/mirror_panel_2f_measurements.schema.yml +39 -0
- simtools/schemas/model_parameters/mirror_reflection_random_angle.schema.yml +61 -0
- simtools/schemas/model_parameters/mirror_reflectivity.schema.yml +40 -0
- simtools/schemas/model_parameters/multiplicity_offset.schema.yml +46 -0
- simtools/schemas/model_parameters/nsb_autoscale_airmass.schema.yml +51 -0
- simtools/schemas/model_parameters/nsb_gain_drop_scale.schema.yml +37 -0
- simtools/schemas/model_parameters/nsb_offaxis.schema.yml +79 -0
- simtools/schemas/model_parameters/nsb_pixel_rate.schema.yml +47 -0
- simtools/schemas/model_parameters/nsb_reference_spectrum.schema.yml +34 -0
- simtools/schemas/model_parameters/nsb_reference_value.schema.yml +33 -0
- simtools/schemas/model_parameters/nsb_scaling_factor.schema.yml +35 -0
- simtools/schemas/model_parameters/nsb_skymap.schema.yml +39 -0
- simtools/schemas/model_parameters/nsb_spectrum.schema.yml +50 -0
- simtools/schemas/model_parameters/num_gains.schema.yml +34 -0
- simtools/schemas/model_parameters/only_triggered_telescopes.schema.yml +33 -0
- simtools/schemas/model_parameters/optics_properties.schema.yml +31 -0
- simtools/schemas/model_parameters/parabolic_dish.schema.yml +32 -0
- simtools/schemas/model_parameters/pedestal_events.schema.yml +32 -0
- simtools/schemas/model_parameters/photon_delay.schema.yml +38 -0
- simtools/schemas/model_parameters/photons_per_run.schema.yml +33 -0
- simtools/schemas/model_parameters/pixel_cells.schema.yml +35 -0
- simtools/schemas/model_parameters/pixels_parallel.schema.yml +54 -0
- simtools/schemas/model_parameters/pixeltrg_time_step.schema.yml +40 -0
- simtools/schemas/model_parameters/pm_average_gain.schema.yml +34 -0
- simtools/schemas/model_parameters/pm_collection_efficiency.schema.yml +40 -0
- simtools/schemas/model_parameters/pm_gain_index.schema.yml +36 -0
- simtools/schemas/model_parameters/pm_photoelectron_spectrum.schema.yml +41 -0
- simtools/schemas/model_parameters/pm_transit_time.schema.yml +63 -0
- simtools/schemas/model_parameters/pm_voltage_variation.schema.yml +39 -0
- simtools/schemas/model_parameters/primary_mirror_degraded_map.schema.yml +42 -0
- simtools/schemas/model_parameters/primary_mirror_diameter.schema.yml +33 -0
- simtools/schemas/model_parameters/primary_mirror_hole_diameter.schema.yml +33 -0
- simtools/schemas/model_parameters/primary_mirror_incidence_angle.schema.yml +29 -0
- simtools/schemas/model_parameters/primary_mirror_parameters.schema.yml +168 -0
- simtools/schemas/model_parameters/primary_mirror_ref_radius.schema.yml +36 -0
- simtools/schemas/model_parameters/primary_mirror_segmentation.schema.yml +34 -0
- simtools/schemas/model_parameters/qe_variation.schema.yml +43 -0
- simtools/schemas/model_parameters/quantum_efficiency.schema.yml +42 -0
- simtools/schemas/model_parameters/random_focal_length.schema.yml +45 -0
- simtools/schemas/model_parameters/random_generator.schema.yml +36 -0
- simtools/schemas/model_parameters/reference_point_altitude.schema.yml +35 -0
- simtools/schemas/model_parameters/reference_point_latitude.schema.yml +36 -0
- simtools/schemas/model_parameters/reference_point_longitude.schema.yml +36 -0
- simtools/schemas/model_parameters/reference_point_utm_east.schema.yml +34 -0
- simtools/schemas/model_parameters/reference_point_utm_north.schema.yml +34 -0
- simtools/schemas/model_parameters/sampled_output.schema.yml +31 -0
- simtools/schemas/model_parameters/save_pe_with_amplitude.schema.yml +34 -0
- simtools/schemas/model_parameters/secondary_mirror_baffle.schema.yml +79 -0
- simtools/schemas/model_parameters/secondary_mirror_degraded_map.schema.yml +42 -0
- simtools/schemas/model_parameters/secondary_mirror_degraded_reflection.schema.yml +41 -0
- simtools/schemas/model_parameters/secondary_mirror_diameter.schema.yml +33 -0
- simtools/schemas/model_parameters/secondary_mirror_hole_diameter.schema.yml +36 -0
- simtools/schemas/model_parameters/secondary_mirror_incidence_angle.schema.yml +29 -0
- simtools/schemas/model_parameters/secondary_mirror_parameters.schema.yml +168 -0
- simtools/schemas/model_parameters/secondary_mirror_ref_radius.schema.yml +36 -0
- simtools/schemas/model_parameters/secondary_mirror_reflectivity.schema.yml +35 -0
- simtools/schemas/model_parameters/secondary_mirror_segmentation.schema.yml +37 -0
- simtools/schemas/model_parameters/secondary_mirror_shadow_diameter.schema.yml +40 -0
- simtools/schemas/model_parameters/secondary_mirror_shadow_offset.schema.yml +40 -0
- simtools/schemas/model_parameters/store_photoelectrons.schema.yml +41 -0
- simtools/schemas/model_parameters/tailcut_scale.schema.yml +40 -0
- simtools/schemas/model_parameters/telescope_axis_height.schema.yml +31 -0
- simtools/schemas/model_parameters/telescope_random_angle.schema.yml +35 -0
- simtools/schemas/model_parameters/telescope_random_error.schema.yml +34 -0
- simtools/schemas/model_parameters/telescope_sphere_radius.schema.yml +37 -0
- simtools/schemas/model_parameters/telescope_transmission.schema.yml +113 -0
- simtools/schemas/model_parameters/teltrig_min_sigsum.schema.yml +41 -0
- simtools/schemas/model_parameters/teltrig_min_time.schema.yml +36 -0
- simtools/schemas/model_parameters/transit_time_calib_error.schema.yml +36 -0
- simtools/schemas/model_parameters/transit_time_compensate_error.schema.yml +37 -0
- simtools/schemas/model_parameters/transit_time_compensate_step.schema.yml +38 -0
- simtools/schemas/model_parameters/transit_time_error.schema.yml +45 -0
- simtools/schemas/model_parameters/transit_time_jitter.schema.yml +36 -0
- simtools/schemas/model_parameters/trigger_current_limit.schema.yml +32 -0
- simtools/schemas/model_parameters/trigger_delay_compensation.schema.yml +53 -0
- simtools/schemas/model_parameters/trigger_pixels.schema.yml +40 -0
- simtools/simtel/simtel_config_reader.py +353 -0
- simtools/simtel/simtel_config_writer.py +244 -63
- simtools/simtel/{simtel_events.py → simtel_io_events.py} +26 -25
- simtools/simtel/simtel_io_histogram.py +661 -0
- simtools/simtel/simtel_io_histograms.py +569 -0
- simtools/simtel/simulator_array.py +145 -0
- simtools/simtel/{simtel_runner_camera_efficiency.py → simulator_camera_efficiency.py} +76 -52
- simtools/simtel/simulator_light_emission.py +473 -0
- simtools/simtel/simulator_ray_tracing.py +262 -0
- simtools/simulator.py +220 -446
- simtools/testing/__init__.py +0 -0
- simtools/testing/assertions.py +151 -0
- simtools/testing/configuration.py +226 -0
- simtools/testing/helpers.py +42 -0
- simtools/testing/validate_output.py +240 -0
- simtools/utils/general.py +340 -437
- simtools/utils/geometry.py +12 -12
- simtools/utils/names.py +257 -644
- simtools/utils/value_conversion.py +176 -0
- simtools/version.py +3 -1
- simtools/visualization/legend_handlers.py +135 -152
- simtools/visualization/plot_camera.py +379 -0
- simtools/visualization/visualize.py +346 -167
- gammasimtools-0.6.1.dist-info/METADATA +0 -180
- gammasimtools-0.6.1.dist-info/RECORD +0 -91
- gammasimtools-0.6.1.dist-info/entry_points.txt +0 -23
- simtools/_dev_version/scm_version.py +0 -10
- simtools/applications/db_development_tools/add_new_parameter_to_db.py +0 -81
- simtools/applications/db_development_tools/add_unit_to_parameter_in_db.py +0 -59
- simtools/applications/db_development_tools/mark_non_optics_parameters_non_applicable.py +0 -102
- simtools/applications/get_parameter.py +0 -92
- simtools/applications/make_regular_arrays.py +0 -160
- simtools/applications/produce_array_config.py +0 -136
- simtools/applications/production.py +0 -313
- simtools/applications/sim_showers_for_trigger_rates.py +0 -187
- simtools/applications/tune_psf.py +0 -334
- simtools/corsika/corsika_default_config.py +0 -282
- simtools/corsika/corsika_runner.py +0 -450
- simtools/corsika_simtel/corsika_simtel_runner.py +0 -197
- simtools/db_handler.py +0 -1480
- simtools/ray_tracing.py +0 -525
- simtools/simtel/simtel_histograms.py +0 -414
- simtools/simtel/simtel_runner.py +0 -244
- simtools/simtel/simtel_runner_array.py +0 -293
- simtools/simtel/simtel_runner_ray_tracing.py +0 -277
- {gammasimtools-0.6.1.dist-info → gammasimtools-0.8.2.dist-info}/LICENSE +0 -0
- {gammasimtools-0.6.1.dist-info → gammasimtools-0.8.2.dist-info}/top_level.txt +0 -0
- /simtools/{corsika_simtel → db}/__init__.py +0 -0
|
@@ -1,25 +1,28 @@
|
|
|
1
|
+
"""Validation of data using schema."""
|
|
2
|
+
|
|
1
3
|
import logging
|
|
2
4
|
import os
|
|
3
5
|
import re
|
|
4
6
|
from pathlib import Path
|
|
5
7
|
|
|
8
|
+
import jsonschema
|
|
6
9
|
import numpy as np
|
|
7
10
|
from astropy import units as u
|
|
8
|
-
from astropy.table import Table, unique
|
|
11
|
+
from astropy.table import Column, Table, unique
|
|
9
12
|
from astropy.utils.diff import report_diff_values
|
|
10
13
|
|
|
11
14
|
import simtools.utils.general as gen
|
|
15
|
+
from simtools.data_model import format_checkers
|
|
16
|
+
from simtools.utils import value_conversion
|
|
12
17
|
|
|
13
18
|
__all__ = ["DataValidator"]
|
|
14
19
|
|
|
15
20
|
|
|
16
21
|
class DataValidator:
|
|
17
22
|
"""
|
|
18
|
-
Validate data for type and units following a describing schema; converts or
|
|
19
|
-
transform data if required.
|
|
23
|
+
Validate data for type and units following a describing schema; converts or transform data.
|
|
20
24
|
|
|
21
|
-
Data can be of table or
|
|
22
|
-
(internally, all data is converted to astropy tables).
|
|
25
|
+
Data can be of table or dict format (internally, all data is converted to astropy tables).
|
|
23
26
|
|
|
24
27
|
Parameters
|
|
25
28
|
----------
|
|
@@ -27,26 +30,41 @@ class DataValidator:
|
|
|
27
30
|
Schema file describing input data and transformations.
|
|
28
31
|
data_file: Path
|
|
29
32
|
Input data file.
|
|
33
|
+
data_table: astropy.table
|
|
34
|
+
Input data table.
|
|
35
|
+
data_dict: dict
|
|
36
|
+
Input data dict.
|
|
37
|
+
check_exact_data_type: bool
|
|
38
|
+
Check for exact data type (default: True).
|
|
30
39
|
|
|
31
40
|
"""
|
|
32
41
|
|
|
33
|
-
def __init__(
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
schema_file=None,
|
|
45
|
+
data_file=None,
|
|
46
|
+
data_table=None,
|
|
47
|
+
data_dict=None,
|
|
48
|
+
check_exact_data_type=True,
|
|
49
|
+
):
|
|
50
|
+
"""Initialize validation class and read required reference data columns."""
|
|
39
51
|
self._logger = logging.getLogger(__name__)
|
|
40
52
|
|
|
41
53
|
self.data_file_name = data_file
|
|
42
54
|
self.schema_file_name = schema_file
|
|
43
|
-
self.
|
|
44
|
-
self.
|
|
55
|
+
self._data_description = None
|
|
56
|
+
self.data_dict = data_dict
|
|
45
57
|
self.data_table = data_table
|
|
58
|
+
self.check_exact_data_type = check_exact_data_type
|
|
46
59
|
|
|
47
|
-
def validate_and_transform(self):
|
|
60
|
+
def validate_and_transform(self, is_model_parameter=False):
|
|
48
61
|
"""
|
|
49
|
-
|
|
62
|
+
Validate data and data file.
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
is_model_parameter: bool
|
|
67
|
+
This is a model parameter (add some data preparation)
|
|
50
68
|
|
|
51
69
|
Returns
|
|
52
70
|
-------
|
|
@@ -59,30 +77,28 @@ class DataValidator:
|
|
|
59
77
|
if no data or data table is available
|
|
60
78
|
|
|
61
79
|
"""
|
|
62
|
-
|
|
63
80
|
if self.data_file_name:
|
|
64
81
|
self.validate_data_file()
|
|
65
|
-
if isinstance(self.
|
|
82
|
+
if isinstance(self.data_dict, dict):
|
|
83
|
+
if is_model_parameter:
|
|
84
|
+
self._prepare_model_parameter()
|
|
66
85
|
self._validate_data_dict()
|
|
67
|
-
|
|
86
|
+
return self.data_dict
|
|
87
|
+
if isinstance(self.data_table, Table):
|
|
68
88
|
self._validate_data_table()
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
return self.data_table
|
|
89
|
+
return self.data_table
|
|
90
|
+
self._logger.error("No data or data table to validate")
|
|
91
|
+
raise TypeError
|
|
74
92
|
|
|
75
93
|
def validate_data_file(self):
|
|
76
94
|
"""
|
|
77
|
-
Open data file and read data from file
|
|
78
|
-
(doing this successfully is understood as
|
|
79
|
-
file validation).
|
|
95
|
+
Open data file and read data from file.
|
|
80
96
|
|
|
97
|
+
Doing this successfully is understood as file validation.
|
|
81
98
|
"""
|
|
82
|
-
|
|
83
99
|
try:
|
|
84
|
-
if Path(self.data_file_name).suffix in (".yml", ".yaml"):
|
|
85
|
-
self.
|
|
100
|
+
if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
|
|
101
|
+
self.data_dict = gen.collect_data_from_file(self.data_file_name)
|
|
86
102
|
self._logger.info(f"Validating data from: {self.data_file_name}")
|
|
87
103
|
else:
|
|
88
104
|
self.data_table = Table.read(self.data_file_name, guess=True, delimiter=r"\s")
|
|
@@ -90,53 +106,135 @@ class DataValidator:
|
|
|
90
106
|
except (AttributeError, TypeError):
|
|
91
107
|
pass
|
|
92
108
|
|
|
109
|
+
def validate_parameter_and_file_name(self):
|
|
110
|
+
"""Validate that file name and key 'parameter_name' in data dict are the same."""
|
|
111
|
+
if self.data_dict.get("parameter") != Path(self.data_file_name).stem:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"Parameter name in data dict {self.data_dict.get('parameter')} and "
|
|
114
|
+
f"file name {Path(self.data_file_name).stem} do not match."
|
|
115
|
+
)
|
|
116
|
+
|
|
93
117
|
def _validate_data_dict(self):
|
|
94
118
|
"""
|
|
95
|
-
Validate values
|
|
96
|
-
|
|
119
|
+
Validate values in a dictionary.
|
|
120
|
+
|
|
121
|
+
Handles different types of naming in data dicts (using 'name' or 'parameter'
|
|
122
|
+
keys for name fields).
|
|
123
|
+
|
|
124
|
+
Raises
|
|
125
|
+
------
|
|
126
|
+
KeyError
|
|
127
|
+
if data dict does not contain a 'name' or 'parameter' key.
|
|
128
|
+
|
|
129
|
+
"""
|
|
130
|
+
if not (_name := self.data_dict.get("name") or self.data_dict.get("parameter")):
|
|
131
|
+
raise KeyError("Data dict does not contain a 'name' or 'parameter' key.")
|
|
132
|
+
self._data_description = self._read_validation_schema(self.schema_file_name, _name)
|
|
133
|
+
|
|
134
|
+
value_as_list, unit_as_list = self._get_value_and_units_as_lists()
|
|
135
|
+
|
|
136
|
+
for index, (value, unit) in enumerate(zip(value_as_list, unit_as_list)):
|
|
137
|
+
value_as_list[index], unit_as_list[index] = self._validate_value_and_unit(
|
|
138
|
+
value, unit, index
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
if len(value_as_list) == 1:
|
|
142
|
+
self.data_dict["value"], self.data_dict["unit"] = value_as_list[0], unit_as_list[0]
|
|
143
|
+
else:
|
|
144
|
+
self.data_dict["value"], self.data_dict["unit"] = value_as_list, unit_as_list
|
|
97
145
|
|
|
146
|
+
self._check_version_string(self.data_dict.get("version"))
|
|
147
|
+
|
|
148
|
+
def _validate_value_and_unit(self, value, unit, index):
|
|
98
149
|
"""
|
|
150
|
+
Validate value, unit, and perform type checking and conversions.
|
|
99
151
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
152
|
+
Take into account different data types and allow to use json_schema for testing.
|
|
153
|
+
"""
|
|
154
|
+
if self._get_data_description(index).get("type", None) == "dict":
|
|
155
|
+
self._validate_data_dict_using_json_schema(
|
|
156
|
+
self.data_dict["value"], self._get_data_description(index).get("json_schema")
|
|
103
157
|
)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
try:
|
|
107
|
-
_quantities.append(value * u.Unit(unit))
|
|
108
|
-
except ValueError:
|
|
109
|
-
_quantities.append(value)
|
|
110
|
-
self.data_table = Table(rows=[_quantities])
|
|
111
|
-
self.data_table.meta["name"] = self.data["name"]
|
|
112
|
-
except KeyError as exc:
|
|
113
|
-
raise KeyError("Data dict does not contain a 'name', 'value', or 'value' key.") from exc
|
|
114
|
-
|
|
115
|
-
if self._reference_data_columns is not None:
|
|
116
|
-
self._validate_data_columns()
|
|
158
|
+
else:
|
|
159
|
+
self._check_data_type(np.array(value).dtype, index)
|
|
117
160
|
|
|
118
|
-
|
|
161
|
+
if self.data_dict.get("type") not in ("string", "dict", "file"):
|
|
162
|
+
self._check_for_not_a_number(value, index)
|
|
163
|
+
value, unit = self._check_and_convert_units(value, unit, index)
|
|
164
|
+
for range_type in ("allowed_range", "required_range"):
|
|
165
|
+
self._check_range(index, np.nanmin(value), np.nanmax(value), range_type)
|
|
166
|
+
return value, unit
|
|
167
|
+
|
|
168
|
+
def _get_value_and_units_as_lists(self):
|
|
169
|
+
"""
|
|
170
|
+
Convert value and unit to lists if required.
|
|
171
|
+
|
|
172
|
+
Ignore unit field in data_dict if value is a astropy.Quantity.
|
|
173
|
+
Note the complications from astropy.Units, where a single value is of np.ndarray type.
|
|
174
|
+
|
|
175
|
+
Returns
|
|
176
|
+
-------
|
|
177
|
+
list
|
|
178
|
+
value as list
|
|
179
|
+
list
|
|
180
|
+
unit as list
|
|
181
|
+
"""
|
|
182
|
+
target_unit = self.data_dict["unit"]
|
|
183
|
+
value, unit = value_conversion.split_value_and_unit(self.data_dict["value"])
|
|
184
|
+
|
|
185
|
+
if not isinstance(value, list | np.ndarray):
|
|
186
|
+
value, unit = [value], [unit]
|
|
187
|
+
if not isinstance(target_unit, list | np.ndarray):
|
|
188
|
+
target_unit = [target_unit] * len(value)
|
|
189
|
+
|
|
190
|
+
target_unit = [None if unit == "null" else unit for unit in target_unit]
|
|
191
|
+
conversion_factor = [
|
|
192
|
+
1 if v is None else u.Unit(v).to(u.Unit(t)) for v, t in zip(unit, target_unit)
|
|
193
|
+
]
|
|
194
|
+
return [v * c for v, c in zip(value, conversion_factor)], target_unit
|
|
195
|
+
|
|
196
|
+
def _validate_data_dict_using_json_schema(self, data, json_schema):
|
|
119
197
|
"""
|
|
120
|
-
Validate
|
|
198
|
+
Validate a dictionary using a json schema.
|
|
121
199
|
|
|
200
|
+
Parameters
|
|
201
|
+
----------
|
|
202
|
+
data: dict
|
|
203
|
+
Data dictionary
|
|
204
|
+
json_schema: dict
|
|
205
|
+
JSON schema
|
|
122
206
|
"""
|
|
207
|
+
if json_schema is None:
|
|
208
|
+
self._logger.debug("Skipping validation of dict type")
|
|
209
|
+
return
|
|
210
|
+
self._logger.debug("Validation of dict type using JSON schema")
|
|
211
|
+
try:
|
|
212
|
+
jsonschema.validate(data, json_schema, format_checker=format_checkers.format_checker)
|
|
213
|
+
except jsonschema.exceptions.ValidationError as exc:
|
|
214
|
+
self._logger.error(f"Validation error: {exc}")
|
|
215
|
+
raise exc
|
|
123
216
|
|
|
217
|
+
def _validate_data_table(self):
|
|
218
|
+
"""Validate tabulated data."""
|
|
124
219
|
try:
|
|
125
|
-
self.
|
|
126
|
-
|
|
127
|
-
|
|
220
|
+
self._data_description = self._read_validation_schema(self.schema_file_name)[0].get(
|
|
221
|
+
"table_columns", None
|
|
222
|
+
)
|
|
128
223
|
except IndexError:
|
|
129
224
|
self._logger.error(f"Error reading validation schema from {self.schema_file_name}")
|
|
130
225
|
raise
|
|
131
226
|
|
|
132
|
-
if self.
|
|
227
|
+
if self._data_description is not None:
|
|
133
228
|
self._validate_data_columns()
|
|
134
229
|
self._check_data_for_duplicates()
|
|
135
230
|
self._sort_data()
|
|
136
231
|
|
|
137
232
|
def _validate_data_columns(self):
|
|
138
233
|
"""
|
|
139
|
-
Validate that
|
|
234
|
+
Validate that data columns.
|
|
235
|
+
|
|
236
|
+
This includes:
|
|
237
|
+
|
|
140
238
|
- required data columns are available
|
|
141
239
|
- columns are in the correct units (if necessary apply a unit conversion)
|
|
142
240
|
- ranges (minimum, maximum) are correct.
|
|
@@ -144,18 +242,17 @@ class DataValidator:
|
|
|
144
242
|
This is not applied to columns of type 'string'.
|
|
145
243
|
|
|
146
244
|
"""
|
|
147
|
-
|
|
148
245
|
self._check_required_columns()
|
|
149
246
|
|
|
150
247
|
for col_name in self.data_table.colnames:
|
|
151
248
|
col = self.data_table[col_name]
|
|
152
|
-
if not self.
|
|
249
|
+
if not self._get_data_description(col_name, status_test=True):
|
|
153
250
|
continue
|
|
154
251
|
if not np.issubdtype(col.dtype, np.number):
|
|
155
252
|
continue
|
|
156
|
-
self._check_for_not_a_number(col, col_name)
|
|
157
|
-
self._check_data_type(col, col_name)
|
|
158
|
-
|
|
253
|
+
self._check_for_not_a_number(col.data, col_name)
|
|
254
|
+
self._check_data_type(col.dtype, col_name)
|
|
255
|
+
self.data_table[col_name] = col.to(u.Unit(self._get_reference_unit(col_name)))
|
|
159
256
|
self._check_range(col_name, np.nanmin(col.data), np.nanmax(col.data), "allowed_range")
|
|
160
257
|
self._check_range(col_name, np.nanmin(col.data), np.nanmax(col.data), "required_range")
|
|
161
258
|
|
|
@@ -169,8 +266,7 @@ class DataValidator:
|
|
|
169
266
|
if a required data column is missing
|
|
170
267
|
|
|
171
268
|
"""
|
|
172
|
-
|
|
173
|
-
for entry in self._reference_data_columns:
|
|
269
|
+
for entry in self._data_description:
|
|
174
270
|
if entry.get("required", False):
|
|
175
271
|
if entry["name"] in self.data_table.columns:
|
|
176
272
|
self._logger.debug(f"Found required data column {entry['name']}")
|
|
@@ -179,8 +275,9 @@ class DataValidator:
|
|
|
179
275
|
|
|
180
276
|
def _sort_data(self):
|
|
181
277
|
"""
|
|
182
|
-
Sort data according to one data column (if required by any column attribute).
|
|
183
|
-
|
|
278
|
+
Sort data according to one data column (if required by any column attribute).
|
|
279
|
+
|
|
280
|
+
Data is either sorted or reverse sorted.
|
|
184
281
|
|
|
185
282
|
Raises
|
|
186
283
|
------
|
|
@@ -188,10 +285,9 @@ class DataValidator:
|
|
|
188
285
|
if no table is defined for sorting
|
|
189
286
|
|
|
190
287
|
"""
|
|
191
|
-
|
|
192
288
|
_columns_by_which_to_sort = []
|
|
193
289
|
_columns_by_which_to_reverse_sort = []
|
|
194
|
-
for entry in self.
|
|
290
|
+
for entry in self._data_description:
|
|
195
291
|
if "input_processing" in entry:
|
|
196
292
|
if "sort" in entry["input_processing"]:
|
|
197
293
|
_columns_by_which_to_sort.append(entry["name"])
|
|
@@ -210,7 +306,7 @@ class DataValidator:
|
|
|
210
306
|
try:
|
|
211
307
|
self.data_table.sort(_columns_by_which_to_reverse_sort, reverse=True)
|
|
212
308
|
except AttributeError:
|
|
213
|
-
self._logger.error("No data table defined for sorting")
|
|
309
|
+
self._logger.error("No data table defined for reverse sorting")
|
|
214
310
|
raise
|
|
215
311
|
|
|
216
312
|
def _check_data_for_duplicates(self):
|
|
@@ -223,7 +319,6 @@ class DataValidator:
|
|
|
223
319
|
checked for unique values.
|
|
224
320
|
|
|
225
321
|
"""
|
|
226
|
-
|
|
227
322
|
_column_with_unique_requirement = self._get_unique_column_requirement()
|
|
228
323
|
if len(_column_with_unique_requirement) == 0:
|
|
229
324
|
self._logger.debug("No data columns with unique value requirement")
|
|
@@ -240,11 +335,10 @@ class DataValidator:
|
|
|
240
335
|
):
|
|
241
336
|
self.data_table = unique(self.data_table)
|
|
242
337
|
else:
|
|
243
|
-
|
|
338
|
+
raise ValueError(
|
|
244
339
|
"Failed removal of duplication for column "
|
|
245
340
|
f"{_column_with_unique_requirement}, values are not unique"
|
|
246
341
|
)
|
|
247
|
-
raise ValueError
|
|
248
342
|
|
|
249
343
|
def _get_unique_column_requirement(self):
|
|
250
344
|
"""
|
|
@@ -256,10 +350,9 @@ class DataValidator:
|
|
|
256
350
|
list of data column with unique value requirement
|
|
257
351
|
|
|
258
352
|
"""
|
|
259
|
-
|
|
260
353
|
_unique_required_column = []
|
|
261
354
|
|
|
262
|
-
for entry in self.
|
|
355
|
+
for entry in self._data_description:
|
|
263
356
|
if "input_processing" in entry and "remove_duplicates" in entry["input_processing"]:
|
|
264
357
|
self._logger.debug(f"Removing duplicates for column {entry['name']}")
|
|
265
358
|
_unique_required_column.append(entry["name"])
|
|
@@ -287,21 +380,20 @@ class DataValidator:
|
|
|
287
380
|
if column name is not found in reference data columns
|
|
288
381
|
|
|
289
382
|
"""
|
|
290
|
-
|
|
291
|
-
reference_unit
|
|
292
|
-
if reference_unit == "dimensionless" or reference_unit is None:
|
|
383
|
+
reference_unit = self._get_data_description(column_name).get("unit", None)
|
|
384
|
+
if reference_unit in ("dimensionless", None, ""):
|
|
293
385
|
return u.dimensionless_unscaled
|
|
294
386
|
|
|
295
387
|
return u.Unit(reference_unit)
|
|
296
388
|
|
|
297
|
-
def _check_data_type(self,
|
|
389
|
+
def _check_data_type(self, dtype, column_name):
|
|
298
390
|
"""
|
|
299
391
|
Check column data type.
|
|
300
392
|
|
|
301
393
|
Parameters
|
|
302
394
|
----------
|
|
303
|
-
|
|
304
|
-
data
|
|
395
|
+
dtype: numpy.dtype
|
|
396
|
+
data type
|
|
305
397
|
column_name: str
|
|
306
398
|
column name
|
|
307
399
|
|
|
@@ -311,26 +403,28 @@ class DataValidator:
|
|
|
311
403
|
if data type is not correct
|
|
312
404
|
|
|
313
405
|
"""
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
406
|
+
reference_dtype = self._get_data_description(column_name).get("type", None)
|
|
407
|
+
if not gen.validate_data_type(
|
|
408
|
+
reference_dtype=reference_dtype,
|
|
409
|
+
value=None,
|
|
410
|
+
dtype=dtype,
|
|
411
|
+
allow_subtypes=(not self.check_exact_data_type),
|
|
412
|
+
):
|
|
318
413
|
self._logger.error(
|
|
319
414
|
f"Invalid data type in column '{column_name}'. "
|
|
320
|
-
f"Expected type '{reference_dtype}', found '{
|
|
415
|
+
f"Expected type '{reference_dtype}', found '{dtype}' "
|
|
416
|
+
f"(exact type: {self.check_exact_data_type})"
|
|
321
417
|
)
|
|
322
418
|
raise TypeError
|
|
323
419
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
def _check_for_not_a_number(self, col, col_name):
|
|
420
|
+
def _check_for_not_a_number(self, data, col_name):
|
|
327
421
|
"""
|
|
328
422
|
Check that column values are finite and not NaN.
|
|
329
423
|
|
|
330
424
|
Parameters
|
|
331
425
|
----------
|
|
332
|
-
|
|
333
|
-
data
|
|
426
|
+
data: value or numpy.ndarray
|
|
427
|
+
data to be tested
|
|
334
428
|
col_name: str
|
|
335
429
|
column name
|
|
336
430
|
|
|
@@ -345,26 +439,47 @@ class DataValidator:
|
|
|
345
439
|
if at least one column value is NaN or Inf.
|
|
346
440
|
|
|
347
441
|
"""
|
|
442
|
+
if isinstance(data, str):
|
|
443
|
+
return True
|
|
444
|
+
if not isinstance(data, np.ndarray):
|
|
445
|
+
data = np.array(data)
|
|
348
446
|
|
|
349
|
-
if np.isnan(
|
|
447
|
+
if np.isnan(data).any():
|
|
350
448
|
self._logger.info(f"Column {col_name} contains NaN.")
|
|
351
|
-
if np.isinf(
|
|
449
|
+
if np.isinf(data).any():
|
|
352
450
|
self._logger.info(f"Column {col_name} contains infinite value.")
|
|
353
451
|
|
|
354
|
-
entry = self.
|
|
452
|
+
entry = self._get_data_description(col_name)
|
|
355
453
|
if "allow_nan" in entry.get("input_processing", {}):
|
|
356
|
-
return np.isnan(
|
|
454
|
+
return np.isnan(data).any() or np.isinf(data).any()
|
|
357
455
|
|
|
358
|
-
if np.isnan(
|
|
359
|
-
|
|
360
|
-
raise ValueError
|
|
456
|
+
if np.isnan(data).any() or np.isinf(data).any():
|
|
457
|
+
raise ValueError("NaN or Inf values found in data")
|
|
361
458
|
|
|
362
459
|
return False
|
|
363
460
|
|
|
364
|
-
|
|
461
|
+
@staticmethod
|
|
462
|
+
def _is_dimensionless(unit):
|
|
365
463
|
"""
|
|
366
|
-
Check
|
|
367
|
-
|
|
464
|
+
Check if unit is dimensionless, None, or empty.
|
|
465
|
+
|
|
466
|
+
Parameters
|
|
467
|
+
----------
|
|
468
|
+
unit: str
|
|
469
|
+
unit of data column
|
|
470
|
+
|
|
471
|
+
Returns
|
|
472
|
+
-------
|
|
473
|
+
bool
|
|
474
|
+
True if unit is dimensionless, None, or empty
|
|
475
|
+
"""
|
|
476
|
+
return unit in ("dimensionless", None, "")
|
|
477
|
+
|
|
478
|
+
def _check_and_convert_units(self, data, unit, col_name):
|
|
479
|
+
"""
|
|
480
|
+
Check that input data have an allowed unit.
|
|
481
|
+
|
|
482
|
+
Convert to reference unit (e.g., Angstrom to nm).
|
|
368
483
|
|
|
369
484
|
Note on dimensionless columns:
|
|
370
485
|
|
|
@@ -374,47 +489,91 @@ class DataValidator:
|
|
|
374
489
|
|
|
375
490
|
Parameters
|
|
376
491
|
----------
|
|
377
|
-
|
|
378
|
-
data
|
|
492
|
+
data: astropy.column, Quantity, list, value
|
|
493
|
+
data to be converted
|
|
494
|
+
unit: str
|
|
495
|
+
unit of data column (read from column or Quantity if possible)
|
|
379
496
|
col_name: str
|
|
380
497
|
column name
|
|
381
498
|
|
|
382
499
|
Returns
|
|
383
500
|
-------
|
|
384
|
-
astropy.column
|
|
385
|
-
unit-converted data
|
|
501
|
+
data: astropy.column, Quantity, list, value
|
|
502
|
+
unit-converted data
|
|
386
503
|
|
|
387
504
|
Raises
|
|
388
505
|
------
|
|
389
506
|
u.core.UnitConversionError
|
|
390
|
-
If
|
|
507
|
+
If unit conversions fails
|
|
391
508
|
|
|
392
509
|
"""
|
|
393
|
-
|
|
394
510
|
self._logger.debug(f"Checking data column '{col_name}'")
|
|
395
511
|
|
|
512
|
+
reference_unit = self._get_reference_unit(col_name)
|
|
396
513
|
try:
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
514
|
+
column_unit = data.unit
|
|
515
|
+
except AttributeError:
|
|
516
|
+
column_unit = unit
|
|
517
|
+
|
|
518
|
+
if self._is_dimensionless(column_unit) and self._is_dimensionless(reference_unit):
|
|
519
|
+
return data, u.dimensionless_unscaled
|
|
520
|
+
|
|
521
|
+
self._logger.debug(
|
|
522
|
+
f"Data column '{col_name}' with reference unit "
|
|
523
|
+
f"'{reference_unit}' and data unit '{column_unit}'"
|
|
524
|
+
)
|
|
525
|
+
try:
|
|
526
|
+
if isinstance(data, u.Quantity | Column):
|
|
527
|
+
return data.to(reference_unit), reference_unit
|
|
528
|
+
|
|
529
|
+
if isinstance(data, list | np.ndarray):
|
|
530
|
+
return self._check_and_convert_units_for_list(data, column_unit, reference_unit)
|
|
531
|
+
|
|
532
|
+
# ensure that the data type is preserved (e.g., integers)
|
|
533
|
+
return (type(data)(u.Unit(column_unit).to(reference_unit) * data), reference_unit)
|
|
534
|
+
except (u.core.UnitConversionError, ValueError) as exc:
|
|
408
535
|
self._logger.error(
|
|
409
536
|
f"Invalid unit in data column '{col_name}'. "
|
|
410
|
-
f"Expected type '{reference_unit}', found '{
|
|
537
|
+
f"Expected type '{reference_unit}', found '{column_unit}'"
|
|
411
538
|
)
|
|
412
|
-
raise
|
|
539
|
+
raise u.core.UnitConversionError from exc
|
|
540
|
+
|
|
541
|
+
def _check_and_convert_units_for_list(self, data, column_unit, reference_unit):
|
|
542
|
+
"""
|
|
543
|
+
Check and convert units data in a list or or numpy array.
|
|
544
|
+
|
|
545
|
+
Takes into account that data can be dimensionless (with unit 'None', 'dimensionless'
|
|
546
|
+
or '').
|
|
547
|
+
|
|
548
|
+
Parameters
|
|
549
|
+
----------
|
|
550
|
+
data: list
|
|
551
|
+
list of data
|
|
552
|
+
column_unit: str
|
|
553
|
+
unit of data column
|
|
554
|
+
reference_unit: str
|
|
555
|
+
reference unit
|
|
556
|
+
|
|
557
|
+
Returns
|
|
558
|
+
-------
|
|
559
|
+
list
|
|
560
|
+
converted data
|
|
561
|
+
|
|
562
|
+
"""
|
|
563
|
+
return [
|
|
564
|
+
(
|
|
565
|
+
u.Unit(_to_unit).to(reference_unit) * d
|
|
566
|
+
if _to_unit not in (None, "dimensionless", "")
|
|
567
|
+
else d
|
|
568
|
+
)
|
|
569
|
+
for d, _to_unit in zip(data, column_unit)
|
|
570
|
+
], reference_unit
|
|
413
571
|
|
|
414
572
|
def _check_range(self, col_name, col_min, col_max, range_type="allowed_range"):
|
|
415
573
|
"""
|
|
416
|
-
Check that column data is within allowed range or required range.
|
|
417
|
-
|
|
574
|
+
Check that column data is within allowed range or required range.
|
|
575
|
+
|
|
576
|
+
Assumes that column and ranges have the same units.
|
|
418
577
|
|
|
419
578
|
Parameters
|
|
420
579
|
----------
|
|
@@ -439,42 +598,32 @@ class DataValidator:
|
|
|
439
598
|
"""
|
|
440
599
|
self._logger.debug(f"Checking data in column '{col_name}' for '{range_type}' ")
|
|
441
600
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
raise KeyError
|
|
445
|
-
except KeyError:
|
|
446
|
-
self._logger.error("Allowed range types are 'allowed_range', 'required_range'")
|
|
447
|
-
raise
|
|
601
|
+
if range_type not in ("allowed_range", "required_range"):
|
|
602
|
+
raise KeyError("Allowed range types are 'allowed_range', 'required_range'")
|
|
448
603
|
|
|
449
|
-
_entry = self.
|
|
604
|
+
_entry = self._get_data_description(col_name)
|
|
450
605
|
if range_type not in _entry:
|
|
451
|
-
return
|
|
606
|
+
return
|
|
452
607
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
raise ValueError
|
|
460
|
-
except KeyError:
|
|
461
|
-
self._logger.error(f"Invalid range ('{range_type}') definition for column '{col_name}'")
|
|
462
|
-
except ValueError:
|
|
463
|
-
self._logger.error(
|
|
608
|
+
if not self._interval_check(
|
|
609
|
+
(col_min, col_max),
|
|
610
|
+
(_entry[range_type].get("min", -np.inf), _entry[range_type].get("max", np.inf)),
|
|
611
|
+
range_type,
|
|
612
|
+
):
|
|
613
|
+
raise ValueError(
|
|
464
614
|
f"Value for column '{col_name}' out of range. "
|
|
465
615
|
f"([{col_min}, {col_max}], {range_type}: "
|
|
466
|
-
f"[{_entry[range_type].get('min', np.
|
|
467
|
-
f"{_entry[range_type].get('max', np.
|
|
616
|
+
f"[{_entry[range_type].get('min', -np.inf)}, "
|
|
617
|
+
f"{_entry[range_type].get('max', np.inf)}])"
|
|
468
618
|
)
|
|
469
|
-
raise
|
|
470
|
-
|
|
471
|
-
return None
|
|
472
619
|
|
|
473
620
|
@staticmethod
|
|
474
621
|
def _interval_check(data, axis_range, range_type):
|
|
475
622
|
"""
|
|
476
|
-
|
|
477
|
-
|
|
623
|
+
Range checking for a given set of data.
|
|
624
|
+
|
|
625
|
+
Check that values are inside allowed range or interval. This(range_type='allowed_range')
|
|
626
|
+
or span at least the given interval (range_type='required_range').
|
|
478
627
|
|
|
479
628
|
Parameters
|
|
480
629
|
----------
|
|
@@ -491,7 +640,6 @@ class DataValidator:
|
|
|
491
640
|
True if range test is passed
|
|
492
641
|
|
|
493
642
|
"""
|
|
494
|
-
|
|
495
643
|
if range_type == "allowed_range":
|
|
496
644
|
if data[0] >= axis_range[0] and data[1] <= axis_range[1]:
|
|
497
645
|
return True
|
|
@@ -526,21 +674,23 @@ class DataValidator:
|
|
|
526
674
|
if 'data' can not be read from dict in schema file
|
|
527
675
|
|
|
528
676
|
"""
|
|
529
|
-
|
|
530
677
|
try:
|
|
531
678
|
if Path(schema_file).is_dir():
|
|
532
|
-
return gen.
|
|
679
|
+
return gen.collect_data_from_file(
|
|
533
680
|
file_name=Path(schema_file) / (parameter + ".schema.yml"),
|
|
534
|
-
in_dict=None,
|
|
535
681
|
)["data"]
|
|
536
|
-
return gen.
|
|
682
|
+
return gen.collect_data_from_file(file_name=schema_file)["data"]
|
|
537
683
|
except KeyError:
|
|
538
684
|
self._logger.error(f"Error reading validation schema from {schema_file}")
|
|
539
685
|
raise
|
|
540
686
|
|
|
541
|
-
def
|
|
687
|
+
def _get_data_description(self, column_name=None, status_test=False):
|
|
542
688
|
"""
|
|
543
|
-
Return
|
|
689
|
+
Return data description as provided by the schema file.
|
|
690
|
+
|
|
691
|
+
For tables (type: 'data_table'), return the description of
|
|
692
|
+
the column named 'column_name'. For other types, return
|
|
693
|
+
all data descriptions.
|
|
544
694
|
For columns named 'colX' return the Xth column in the reference data.
|
|
545
695
|
|
|
546
696
|
Parameters
|
|
@@ -557,20 +707,38 @@ class DataValidator:
|
|
|
557
707
|
bool
|
|
558
708
|
True if reference column exists (for status_test==True).
|
|
559
709
|
|
|
560
|
-
|
|
561
710
|
Raises
|
|
562
711
|
------
|
|
563
712
|
IndexError
|
|
564
713
|
If data column is not found.
|
|
565
714
|
|
|
566
715
|
"""
|
|
716
|
+
self._logger.debug(
|
|
717
|
+
f"Getting reference data column {column_name} from schema {self._data_description}"
|
|
718
|
+
)
|
|
719
|
+
try:
|
|
720
|
+
return (
|
|
721
|
+
self._data_description[column_name]
|
|
722
|
+
if not status_test
|
|
723
|
+
else (
|
|
724
|
+
self._data_description[column_name] is not None
|
|
725
|
+
and len(self._data_description) > 0
|
|
726
|
+
)
|
|
727
|
+
)
|
|
728
|
+
except IndexError as exc:
|
|
729
|
+
self._logger.error(
|
|
730
|
+
f"Data column '{column_name}' not found in reference column definition"
|
|
731
|
+
)
|
|
732
|
+
raise exc
|
|
733
|
+
except TypeError:
|
|
734
|
+
pass # column_name is not an integer
|
|
567
735
|
|
|
568
736
|
_index = 0
|
|
569
737
|
if bool(re.match(r"^col\d$", column_name)):
|
|
570
738
|
_index = int(column_name[3:])
|
|
571
|
-
_entry = self.
|
|
739
|
+
_entry = self._data_description
|
|
572
740
|
else:
|
|
573
|
-
_entry = [item for item in self.
|
|
741
|
+
_entry = [item for item in self._data_description if item["name"] == column_name]
|
|
574
742
|
if status_test:
|
|
575
743
|
return len(_entry) > 0
|
|
576
744
|
try:
|
|
@@ -580,3 +748,47 @@ class DataValidator:
|
|
|
580
748
|
f"Data column '{column_name}' not found in reference column definition"
|
|
581
749
|
)
|
|
582
750
|
raise
|
|
751
|
+
|
|
752
|
+
def _prepare_model_parameter(self):
|
|
753
|
+
"""
|
|
754
|
+
Apply data preparation for model parameters.
|
|
755
|
+
|
|
756
|
+
Converts strings to numerical values or lists of values, if required.
|
|
757
|
+
|
|
758
|
+
"""
|
|
759
|
+
value = self.data_dict["value"]
|
|
760
|
+
if not isinstance(value, str):
|
|
761
|
+
return
|
|
762
|
+
|
|
763
|
+
# assume float value if type is not defined
|
|
764
|
+
_is_float = self.data_dict.get("type", "float").startswith(("float", "double"))
|
|
765
|
+
|
|
766
|
+
if value.isnumeric():
|
|
767
|
+
self.data_dict["value"] = float(value) if _is_float else int(value)
|
|
768
|
+
else:
|
|
769
|
+
self.data_dict["value"] = gen.convert_string_to_list(value, is_float=_is_float)
|
|
770
|
+
|
|
771
|
+
if self.data_dict["unit"] is not None:
|
|
772
|
+
self.data_dict["unit"] = gen.convert_string_to_list(self.data_dict["unit"])
|
|
773
|
+
|
|
774
|
+
def _check_version_string(self, version):
|
|
775
|
+
"""
|
|
776
|
+
Check that version string follows semantic versioning.
|
|
777
|
+
|
|
778
|
+
Parameters
|
|
779
|
+
----------
|
|
780
|
+
version: str
|
|
781
|
+
version string
|
|
782
|
+
|
|
783
|
+
Raises
|
|
784
|
+
------
|
|
785
|
+
ValueError
|
|
786
|
+
if version string does not follow semantic versioning
|
|
787
|
+
|
|
788
|
+
"""
|
|
789
|
+
if version is None:
|
|
790
|
+
return
|
|
791
|
+
semver_regex = r"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$"
|
|
792
|
+
if not re.match(semver_regex, version):
|
|
793
|
+
raise ValueError(f"Invalid version string '{version}'")
|
|
794
|
+
self._logger.debug(f"Valid version string '{version}'")
|