ctao-calibpipe 0.1.0rc9__tar.gz → 0.2.0rc2__tar.gz

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/.gitlab-ci.yml

@@ -1,7 +1,7 @@
 #
 include:
   - project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
-    ref: ff3b2136038df7eb88712f7d523acafdc074f660
+    ref: d29b1cc05e31f9dff21dc01104e4e19ccb54d1f4
     file: 'ci-functions.yml'
 
   - 'aiv-config.yml'
@@ -10,8 +10,8 @@ variables:
   CHART_LOCATION: chart
   CHART_NAME: calibpipe
   CHART_EXTRA_VALUES: "--set dev.client_image_tag=${DOCKER_TAG}"
-  DPPS_AIV_TOOLKIT_DIR: dpps-aiv-toolkit
   CONDA_IMAGE: "quay.io/condaforge/miniforge3:24.11.3-0"
+  CTAPIPE_VERSION: 'latest'
 
 stages:
     - prepare
@@ -22,12 +22,11 @@ stages:
     - report
     - doc
     - publish
+    - changelog
 
 default:
     image: harbor.cta-observatory.org/proxy_cache/python:3.12
 
-variables:
-    CTAPIPE_VERSION: 'latest'
 #
 # Anchors definition
 #
@@ -52,6 +51,22 @@ variables:
     <<: *env_setup
     script:
         - pip install .[test]
+        - >
+          echo "url: https://cds.climate.copernicus.eu/api" > ~/.cdsapirc
+        - >
+          echo "key: $CDS_KEY" >> ~/.cdsapirc
+        - mkdir -p ./src/calibpipe/tests/data/telescope/camera
+        - mkdir -p ./src/calibpipe/tests/data/telescope/throughput
+        - mkdir -p ./src/calibpipe/tests/data/array
+        - curl -o ./src/calibpipe/tests/data/array/cross_calibration_test_dl2.h5 $DL2_PROD5_CTAPIPE_CC
+        - curl -o ./src/calibpipe/tests/data/telescope/throughput/lst_muon_table.h5 $DL1_MUON_SIM_LST
+        - curl -o ./src/calibpipe/tests/data/telescope/throughput/empty_muon_table.h5 $EMPTY_MUON_TABLE
+        - curl -o ./src/calibpipe/tests/data/telescope/camera/flatfield_LST_dark.simtel.gz $CAM_CALIB_TEST_DATA/flasher_LST_dark.simtel.gz
+        - curl -o ./src/calibpipe/tests/data/telescope/camera/pedestal_LST_dark.simtel.gz $CAM_CALIB_TEST_DATA/pedestals_LST_dark.simtel.gz
+        - |
+          for mode in single_chunk same_chunks different_chunks; do
+            curl -o ./src/calibpipe/tests/data/telescope/camera/calibpipe_v0.2.0_statsagg_${mode}.dl1.h5 $CAM_CALIB_TEST_DATA/calibpipe_v0.2.0_statsagg_${mode}.dl1.h5
+          done
         - pytest -v --cov=calibpipe --junitxml=report.xml -m "not gdas and not db and not integration"
     artifacts:
         when: always
@@ -111,6 +126,18 @@ test:
         - pip install -e .[test]
         - pip install coverage-badge
         - mkdir -pv test
+        - mkdir -p ./src/calibpipe/tests/data/telescope/camera
+        - mkdir -p ./src/calibpipe/tests/data/telescope/throughput
+        - mkdir -p ./src/calibpipe/tests/data/array
+        - curl -o ./src/calibpipe/tests/data/array/cross_calibration_test_dl2.h5 $DL2_PROD5_CTAPIPE_CC
+        - curl -o ./src/calibpipe/tests/data/telescope/throughput/lst_muon_table.h5 $DL1_MUON_SIM_LST
+        - curl -o ./src/calibpipe/tests/data/telescope/throughput/empty_muon_table.h5 $EMPTY_MUON_TABLE
+        - curl -o ./src/calibpipe/tests/data/telescope/camera/flatfield_LST_dark.simtel.gz $CAM_CALIB_TEST_DATA/flasher_LST_dark.simtel.gz
+        - curl -o ./src/calibpipe/tests/data/telescope/camera/pedestal_LST_dark.simtel.gz $CAM_CALIB_TEST_DATA/pedestals_LST_dark.simtel.gz
+        - |
+          for mode in single_chunk same_chunks different_chunks; do
+            curl -o ./src/calibpipe/tests/data/telescope/camera/calibpipe_v0.2.0_statsagg_${mode}.dl1.h5 $CAM_CALIB_TEST_DATA/calibpipe_v0.2.0_statsagg_${mode}.dl1.h5
+          done
         - pytest -v --cov=calibpipe --junitxml=test/report.xml  --cov-report=xml:test/coverage.xml
         - coverage-badge
     coverage: '/TOTAL.+ ([0-9]{1,3}%)/'

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ctao-calibpipe
-Version: 0.1.0rc9
+Version: 0.2.0rc2
 Author: Leonid Burmistrov, Mykhailo Dalchenko, Antonio Di Pilato, Gabriel Emery, Tjark Miener, Gregoire Uhlrich, Georgios Voutsinas, Vadym Voitsekhovskyi
 Classifier: License :: OSI Approved :: MIT License
 Requires-Python: >=3.10
@@ -10,8 +10,9 @@ License-File: AUTHORS.md
 Requires-Dist: astral
 Requires-Dist: astropy
 Requires-Dist: cdsapi
-Requires-Dist: ctapipe>=0.18
+Requires-Dist: ctapipe[all]
 Requires-Dist: h5py
+Requires-Dist: iminuit
 Requires-Dist: molecularprofiles>=2.1.0
 Requires-Dist: numpy
 Requires-Dist: pandas

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/aiv-config.yml

@@ -1,14 +1,14 @@
  variables:
   # used to select requirements
   DPPS_UC_GROUPS: CalibPipe
-  DPPS_RELEASE: v0.1
+  DPPS_RELEASE: v0.2
 
   RELEASE_PLAN_FN: dpps-aiv-toolkit/dpps-release-plan/release_development_document/dpps-release-plan.tex
   TRACEABILITY_CSV_FN: dpps-aiv-toolkit/requirements-traceability/DPPS_Requirements_Traceability_Matrix.csv
   APPLICATION_NAME: CalibPipe
   APPLICATION_AUTHOR: CalibPipe Team
   APPLICATION_AUTHOR_ORGANIZATION: Université de Genève - DPNC, CTAO
-  APPLICATION_VERSION: 0.1.0
+  APPLICATION_VERSION: 0.2.0
 
   DOCKER_IMAGE_CONTEXT: $CI_PROJECT_DIR
   CWL_DIR: ${CI_PROJECT_DIR}/workflows

ctao_calibpipe-0.2.0rc2/docs/source/examples/array/cross_calibration_configuration.yaml

@@ -0,0 +1,18 @@
+CalculateCrossCalibration:
+  event_filters: {"min_gammaness": 0.5,
+  "min_energy": 0.0 GeV,
+  "max_distance_asymmetry": 0.05
+}
+  reconstruction_algorithm: "RandomForest"
+  RelativeThroughputFitter:
+    throughput_normalization:
+      - [type, 'MST*', 1.0]
+      - [type, 'SST*', 1.0]
+    reference_telescopes:
+      - [type, 'MST*', 5]
+      - [type, 'SST*', 37]
+  PairFinder:
+    max_impact_distance:
+      - [type, 'LST*', 125.0]
+      - [type, 'MST*', 125.0]
+      - [type, 'SST*', 225.0]

ctao_calibpipe-0.2.0rc2/docs/source/examples/arraycalibration.rst

@@ -0,0 +1,21 @@
+=================
+Array calibration
+=================
+
+Here we describe the configuration parameters of the relative throughput calibration using reconstructed shower energies tool.
+
+* **event_filters**: a dictionary with the event selection filters. The keys are the name of the filters while the values are the thresholds. Currently implemented are:
+* **min_gammaness**: used to select gamma-like showers.
+* **min_energy**: the minimum energy.
+* **max_distance_asymmetry**: used to select equidistant events.
+
+* **reconstruction_algorithm**: name of the reconstruction algorithm.
+
+* **throughput_normalization**: a FloatTelescopeParameter that sets the overall telescope throughput normalization. Dependingon the use case scenario, it could reflect the absolute optical throughput measured by the muon rings / illuminator, if we want to compare or complement these methods. Alternatively it could get an arbitrary number, e.g. one that sets the average throughput to 1, if the user wants to 'flat-field' the array. Finally it could be set to 1, if we want to identify outlier telescopes or study the aging.
+
+* **reference_telescopes**: an IntTelescopeParameter that sets the ID of the telescopes whose throughput kept fixed during the intercalibration minimization.
+
+* **max_impact_distance**: a FloatTelescopeParameter that sets the maximum distance between the telescopes and a shower core in meters. The maximum distance between the telescopes in pair should not exceed the sum of these values for the telescopes in pair.
+
+.. literalinclude:: array/cross_calibration_configuration.yaml
+  :language: YAML

ctao_calibpipe-0.2.0rc2/docs/source/examples/calibprocessing.rst

@@ -0,0 +1,83 @@
+============================
+Calibration event processing
+============================
+
+------
+Common
+------
+
+The common first step in the calibration process is **image extraction**. In this step, pixel-wise quantities such as charges, peak times, and/or variances are computed from the waveforms, which conform to the CTAO DL0 Data Model. This is typically done by integrating or otherwise reducing the waveform data for each pixel. The image extraction is performed using the `ctapipe-process` tool.
+
+* **DataWriter**: Responsible for saving the extracted quantities to a file, following the reference implementation of the ctapipe DL1 data format.
+* **EventTypeFilter**: Used to select the calibration events that will be processed.
+
+The configuration required for image extraction varies depending on the type of calibration event. The following standard fields must be provided (except when using the `VarianceExtractor`):
+
+* **window_shift**: Defines the shift of the integration window relative to the `peak_index` (`peak_index - shift`).
+* **window_width**: Defines the width of the integration window.
+* **apply_integration_correction**: Indicates whether to apply the integration window correction.
+
+-----------------
+Flat-field events
+-----------------
+
+The standard processing of flat-field events for the camera calibration uses
+the **LocalPeakWindowSum** extractor, which sums the signal in
+a window around the peak in each pixel's waveform.
+
+Below is an example configuration for the camera calibration using flat-field events.
+It includes the necessary components for image extraction,
+event type filtering, and  data writing.
+The configuration is designed to handle flat-field events
+and performs the required processing steps using the ``ctapipe-process`` tool.
+
+.. literalinclude:: telescope/camera/configuration/ctapipe_process_flatfield.yaml
+  :language: YAML
+
+-------------------
+Sky pedestal events
+-------------------
+
+The standard processing of sky pedestal events for the camera calibration uses
+the **FixedWindowSum** extractor, which sums the signal within a fixed, user-defined window.
+
+In addition to the standard fields, the **peak_index**, indicating the signal summing reference point, must be provided.
+
+Below is an example configuration for the camera calibration using sky pedestal events. It includes the necessary components for image extraction, event type filtering, and data writing. The configuration is designed to handle sky pedestal events and performs the required processing steps using the ``ctapipe-process`` tool.
+
+.. literalinclude:: telescope/camera/configuration/ctapipe_process_pedestal.yaml
+  :language: YAML
+
+The standard processing of sky pedestal events for the pointing correction uses the
+**VarianceExtractor**, which calculates the signal variance in each waveform.
+
+Below is an example configuration for the pointing correction using sky pedestal events. It includes the necessary components for variance image extraction, event type, and  data writing. The configuration is designed to handle variance images of sky pedestal events and performs the required processing steps using the ``ctapipe-process`` tool.
+
+.. literalinclude:: telescope/pointing/configuration/ctapipe_process_pointing.yaml
+  :language: YAML
+
+-----------
+Muon events
+-----------
+
+The standard processing of muon events for the optical throughput uses the
+**GlobalPeakWindowSum** extractor, which sums signal in a window
+around the peak position of the global average waveform.
+
+In addition to the standard parameters, the **pixel_fraction** is required to identify the position of the integration window.
+
+Below is an example configuration for the calibration pipeline of the muon processing. It includes the necessary components for image extraction, event type filtering, image cleaning, muon processing/fitting, and data writing. The configuration is designed to handle muon events and performs the required processing steps using the ``ctapipe-process`` tool.
+
+.. literalinclude:: telescope/throughput/configuration/ctapipe_process_muon_image.yaml
+  :language: YAML
+
+The processing of muon events goes beyond the standard image extraction and requires additional steps, the extraction of muon parameters. We are also providing two more configuration section, that are mandatory for the muon processing. These sections are:
+
+* **Image cleaning**: The image cleaning step is responsible for removing noise pixels from the extracted images. It uses the ctapipe's ImageCleaner class to apply a specific cleaning algorithm to the images.
+* **Muon processing/fitting**: This step is responsible for processing and fitting the muon events. It uses the ctapipe's MuonProcessor class to perform the muon reconstruction and fitting.
+
+.. literalinclude:: telescope/throughput/configuration/ctapipe_process_muon_fitter.yaml
+  :language: YAML
+
+
+The fields of these two configurations have been already presented above. It is worth noticing that for the muon processing the DL1 images do not have to be stored in the output file for the further analysis. In order to perform the whole muon processing the user has to provide the two standard configuration files or a single concatenated configuration file in the CLI.

ctao_calibpipe-0.2.0rc2/docs/source/examples/cameracalibration.rst

@@ -0,0 +1,48 @@
+==================
+Camera calibration
+==================
+
+-----------------
+Camera Calibrator
+-----------------
+
+The **Camera Calibrator** calculates the final camera calibration coefficients, which includes the pedestal offset per waveform sample and the timing correction for each pixel, using interleaved calibration events. It uses the **FFactor** method to perform the **gain** calibration of the camera pixels. This calibration process is done using the ``calibpipe-calculate-camcalib-coefficients`` tool.
+
+The following parameters shall be provided:
+
+* **input_url**: CTAO HDF5 files for DL1 calibration monitoring.
+* **allowed_tels**: Optional list of allowed telescope IDs, others will be ignored.
+* **faulty_pixels_fraction**: Minimum fraction of faulty camera pixels to identify regions of trouble.
+* **squared_excess_noise_factor**: Excess noise factor squared: :math:`1 + \frac{\text{Var}(\text{Gain})}{\text{Mean}(\text{Gain})^2}`.
+* **window_width**: Width of the window used for the image extraction.
+
+Below is an example configuration for the camera calibration using the **FFactor** method. The configuration is designed to perform the camera calibration using the ``calibpipe-calculate-camcalib-coefficients`` tool.
+
+.. literalinclude:: telescope/camera/configuration/calibpipe_calculate_camcalib_coefficients.yaml
+  :language: YAML
+
+----------------------------
+Additional outlier detection
+----------------------------
+
+The additional outlier detection is used to identify and remove outliers from the calibration data.
+The outlier detection, based on the deviation from the
+expected standard deviation of the number of photoelectrons (**NpeStdOutlierDetector**)
+is included in the ``calibpipe-calculate-camcalib-coefficients`` tool.
+
+To configure it, the following fields are required:
+
+* **n_events**: Number of events used for the chunk-wise aggregation of the statistic values of the calibration data.
+* **relative_qe_dispersion**: Relative (effective) quantum efficiency dispersion of PMs over the camera.
+* **linear_noise_coeff**: Linear noise coefficients [high gain, low gain] or [single gain].
+* **linear_noise_offset**: Linear noise offsets [high gain, low gain] or [single gain].
+* **quadratic_noise_coeff**: Quadratic noise coefficients [high gain, low gain] or [single gain].
+* **quadratic_noise_offset**: Quadratic noise offsets [high gain, low gain] or [single gain].
+* **std_range_factors**: Defines the range of acceptable values (lower, upper) in units of standard deviations.
+
+Below is an example configuration to include the additional outlier detection for the camera calibration. The values have been obtained with a fit of the ``std`` of the LST-1 filter scan taken on 2023/05/10:
+
+.. literalinclude:: telescope/camera/configuration/npe_std_outlier_detector.yaml
+  :language: YAML
+
+In order to perform the camera calibration with the additional outlier detection the user has to provide the two standard configuration files or a single concatenated configuration file in the CLI.

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/docs/source/examples/index.rst

@@ -8,3 +8,7 @@ Examples
     :maxdepth: 1
 
     molecularprofiles
+    calibprocessing
+    pixstatsdescription
+    cameracalibration
+    arraycalibration

ctao_calibpipe-0.2.0rc2/docs/source/examples/pixstatsdescription.rst

@@ -0,0 +1,41 @@
+============================
+Pixel statistics description
+============================
+
+The statistical description is assessed based on the image extracted from the interleaved calibration data. The pixel statistics description is done using the ``ctapipe-calculate-pixel-statistics`` tool. In order to produce test data for the camera calibration tools in the calibpipe package, the utility tool ``calibpipe-produce-camcalib-test-data`` can be used with the configuration files below.
+
+The following configuration parameters are required:
+
+* **allowed_tels**: Optional list of allowed telescope IDs, others will be ignored.
+* **input_column_name**: Column name of the pixel-wise image data to calculate statistics.
+* **output_table_name**: Table name of the output statistics.
+* **output_path**: Output filename.
+
+**PixelStatisticsCalculator** component is used to calculate the statistical description of the calibration events. It needs the following fields:
+
+* **stats_aggregator_type**: Name of the StatisticsAggregator subclass to be used.
+* **outlier_detector_list**: List of dicts containing the OutlierDetector and configuration to be used.
+* **chunk_shift**: Number of samples to shift the aggregation chunk for the calculation of the statistical values.
+* **faulty_pixels_fraction**: Minimum fraction of faulty camera pixels to identify regions of trouble.
+
+Additionally, the user can provide the following fields for the **StatisticsAggregator** component:
+
+* **PlainAggregator**: Compute aggregated statistic values from a chunk of images using numpy functions.
+* **SigmaClippingAggregator**: Compute aggregated statistic values from a chunk of images using astropy's sigma clipping functions.
+
+Below are examples of configurations for assessing the statistical description of interleaved calibration events using the ``ctapipe-calculate-pixel-statistics`` tool.
+
+**Interleaved pedestal events - charge image**
+
+.. literalinclude:: telescope/camera/configuration/ctapipe_calculate_pixel_stats_pedestal_image.yaml
+  :language: YAML
+
+**Interleaved flat-field events - charge image**
+
+.. literalinclude:: telescope/camera/configuration/ctapipe_calculate_pixel_stats_flatfield_image.yaml
+  :language: YAML
+
+**Interleaved flat-field events - peak arrival time**
+
+.. literalinclude:: telescope/camera/configuration/ctapipe_calculate_pixel_stats_flatfield_peak_time.yaml
+  :language: YAML

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/camera/configuration/calibpipe_calculate_camcalib_coefficients.yaml

@@ -0,0 +1,6 @@
+# Camera calibration coefficients calculation configuration
+CameraCalibratorTool:
+  timestamp_tolerance: 1.0 second
+  faulty_pixels_fraction: 0.1
+  squared_excess_noise_factor: 1.222
+  window_width: 12

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/camera/configuration/ctapipe_calculate_pixel_stats_flatfield_image.yaml

@@ -0,0 +1,25 @@
+# Tool configuration for assess statistical description of flatfield events
+PixelStatisticsCalculatorTool:
+  input_column_name: image
+  output_table_name: flatfield_image
+  overwrite: True
+# Pixel statistics configuration
+PixelStatisticsCalculator:
+  stats_aggregator_type:
+    - ["type", "*", "SigmaClippingAggregator"]
+  chunk_shift: 500
+  faulty_pixels_fraction: 0.1
+  outlier_detector_list:
+    -   name: MedianOutlierDetector
+        apply_to: median
+        config:
+            median_range_factors: [-0.9, 8]
+    -   name: StdOutlierDetector
+        apply_to: std
+        config:
+            std_range_factors: [-10, 10]
+# Aggregation configuration
+SigmaClippingAggregator:
+  chunk_size: 2500
+  max_sigma: 4
+  iterations: 5

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/camera/configuration/ctapipe_calculate_pixel_stats_flatfield_peak_time.yaml

@@ -0,0 +1,19 @@
+# Tool configuration for assess statistical description for the time correction
+PixelStatisticsCalculatorTool:
+  input_column_name: peak_time
+  output_table_name: flatfield_peak_time
+  overwrite: True
+# Pixel statistics configuration
+PixelStatisticsCalculator:
+  stats_aggregator_type:
+    - ["type", "*", "PlainAggregator"]
+  chunk_shift: 500
+  faulty_pixels_fraction: 0.1
+  outlier_detector_list:
+    -   name: RangeOutlierDetector
+        apply_to: median
+        config:
+            validity_range: [2, 38]
+# Aggregation configuration
+PlainAggregator:
+  chunk_size: 2500

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/camera/configuration/ctapipe_calculate_pixel_stats_pedestal_image.yaml

@@ -0,0 +1,25 @@
+# Tool configuration for assess statistical description of pedestal events
+PixelStatisticsCalculatorTool:
+  input_column_name: image
+  output_table_name: pedestal_image
+  overwrite: True
+# Pixel statistics configuration
+PixelStatisticsCalculator:
+  stats_aggregator_type:
+    - ["type", "*", "SigmaClippingAggregator"]
+  chunk_shift: 500
+  faulty_pixels_fraction: 0.1
+  outlier_detector_list:
+    -   name: StdOutlierDetector
+        apply_to: median
+        config:
+            std_range_factors: [-10, 10]
+    -   name: StdOutlierDetector
+        apply_to: std
+        config:
+            std_range_factors: [-10, 10]
+# Aggregation configuration
+SigmaClippingAggregator:
+  chunk_size: 2500
+  max_sigma: 4
+  iterations: 5

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/camera/configuration/ctapipe_process_flatfield.yaml

@@ -0,0 +1,21 @@
+# Data writing configuration
+DataWriter:
+  write_dl1_images: true # Required parameter to write flatfield images to DL1
+  write_dl1_parameters: false
+  write_dl2: false
+  transform_image: true
+  transform_peak_time: true
+# Event type filter configuration
+EventTypeFilter:
+  allowed_types: [FLATFIELD]
+# Image extraction configuration
+CameraCalibrator:
+  image_extractor_type:
+    - ['type', '*', 'LocalPeakWindowSum']
+LocalPeakWindowSum:
+  window_shift:
+    - ['type', '*', 5]
+  window_width:
+    - ['type', '*', 12]
+  apply_integration_correction:
+    - ['type', '*', false]

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/camera/configuration/ctapipe_process_pedestal.yaml

@@ -0,0 +1,22 @@
+# Data writing configuration
+DataWriter:
+  write_dl1_images: true # Required parameter to write pedestal images to DL1
+  write_dl1_parameters: false
+  write_dl2: false
+  transform_image: true
+  transform_peak_time: true
+# Event type filter configuration
+EventTypeFilter:
+  allowed_types: [DARK_PEDESTAL, SKY_PEDESTAL]
+CameraCalibrator:
+  image_extractor_type:
+    - ['type', '*', 'FixedWindowSum']
+FixedWindowSum:
+  window_shift:
+    - ['type', '*', 6]
+  window_width:
+    - ['type', '*', 12]
+  peak_index:
+    - ['type', '*', 18]
+  apply_integration_correction:
+    - ['type', '*', false]

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/camera/configuration/npe_std_outlier_detector.yaml

@@ -0,0 +1,9 @@
+# Outlier detector configuration for expected std of the number of photoelectrons
+NpeStdOutlierDetector:
+  n_events: 2500
+  relative_qe_dispersion: 0.7
+  linear_noise_coeff: [1.79717813, 1.72458305]
+  linear_noise_offset: [0.0231544, -0.00162036639]
+  quadratic_noise_coeff: [0.000499670969, 0.00142218]
+  quadratic_noise_offset: [0.0000249034290, 0.0001207]
+  std_range_factors: [-5, 5]

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/pointing/configuration/ctapipe_process_pointing.yaml

@@ -0,0 +1,14 @@
+# Data writing configuration
+DataWriter:
+  write_dl1_images: true
+  write_dl1_parameters: false
+  write_dl2: false
+  transform_image: true
+  transform_peak_time: true
+# Event type filter configuration
+EventTypeFilter:
+  allowed_types: [SKY_PEDESTAL]
+# Image extraction configuration
+CameraCalibrator:
+  image_extractor_type:
+    - ['type', '*', 'VarianceExtractor']

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/throughput/configuration/ctapipe_process_muon_fitter.yaml

@@ -0,0 +1,50 @@
+# Image cleaning configuration
+ImageProcessor:
+  image_cleaner_type:
+    - ['type', '*', 'TailcutsImageCleaner']
+  use_telescope_frame: True
+TailcutsImageCleaner:
+  picture_threshold_pe:
+    - ['type', '*', 10]
+  boundary_threshold_pe:
+    - ['type', '*', 5]
+  min_picture_neighbors:
+    - ['type', '*', 2]
+  keep_isolated_pixels:
+    - ['type', '*', False]
+ImageParameterQuery:
+  quality_criteria: [
+      ["min_pixels", "dl1_params.morphology.n_pixels > 100"],
+      ["min_intensity", "dl1_params.hillas.intensity > 500"]
+    ]
+# Muon processing/fitting configuration
+MuonProcessor:
+  completeness_threshold:
+    - ['type', '*', 30]
+  ratio_width:
+    - ['type', '*', 1.5]
+  pedestal:
+    - ['type', '*', 1.1]
+  ring_integration_width:
+    - ['type', '*', 0.25]
+  outer_ring_width:
+    - ['type', '*', 0.2]
+MuonIntensityFitter:
+  hole_radius_m:
+    - ["type", "LST*", 0.308]
+    - ["type", "MST*", 0.244]
+    - ["type", "SST*", 0.130]
+  spe_width:
+    - ["type", "*", 0.5]
+  min_lambda_m:
+    - ["type", "*", 0.0000003]
+  max_lambda_m:
+    - ["type", "*", 0.0000006]
+  oversampling:
+    - ["type", "*", 3]
+RingQuery:
+  quality_criteria: [
+      ["radius_not_nan", "np.isfinite(ring.radius.value)"],
+      ["min_pixels", "np.count_nonzero(mask) > 50"],
+      ["ring_containment", "parameters.containment > 0.5"]
+    ]

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/throughput/configuration/ctapipe_process_muon_image.yaml

@@ -0,0 +1,27 @@
+# Data writing configuration
+DataWriter:
+  write_dl1_images: true # Optional parameter to write muon images to DL1
+  write_dl1_parameters: true # Optional parameter to write parameters to DL1
+  write_muon_parameters: true # Required parameter to write muon parameters to DL1
+  write_dl2: false
+  transform_image: true
+  transform_peak_time: true
+# Event type filter configuration
+EventTypeFilter:
+  allowed_types: [MUON]
+# Image extraction configuration
+CameraCalibrator:
+  image_extractor_type:
+    - ['type', '*', 'GlobalPeakWindowSum']
+  invalid_pixel_handler_type: "NeighborAverage"
+  apply_waveform_time_shift: False
+  apply_peak_time_shift: True
+GlobalPeakWindowSum:
+  window_shift:
+    - ['type', '*', 3]
+  window_width:
+    - ['type', '*', 7]
+  apply_integration_correction:
+    - ['type', '*', True]
+  pixel_fraction:
+    - ['type', '*', 1]

ctao_calibpipe-0.2.0rc2/docs/source/examples/telescope/throughput/configuration/throughput_muon_configuration.yaml

@@ -0,0 +1,20 @@
+database_configuration:
+    user: TEST_CALIBPIPE_DB_USER
+    password: DUMMY_PSWRD
+    database: TEST_CALIBPIPE_DB
+    host: postgres
+    port: null
+    autocommit: True
+
+CalculateThroughputWithMuons:
+    input_url: None
+    output_url: None
+    output_format: "ascii.ecsv"
+    min_events: 0
+    min_ring_radius: 0.8
+    max_ring_radius: 1.5
+    min_impact_parameter: 0.2
+    max_impact_parameter: 0.9
+    ring_completeness_threshold: 0.3
+    intensity_ratio: 0.5
+    ring_containment_threshold: 0.5

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/docs/source/introduction/diagrams/muon_calibration.mmd

@@ -6,7 +6,7 @@ flowchart BT
     Muon Rings Analysis
     main method):::todo
     2.3(DPPS-UC-120-2.3
-    Shower Energies Analysis):::todo
+    Shower Energies Analysis):::doing
     2.4(DPPS-UC-120-2.4
     Cherenkov Transparency Coefficient):::onhold
 

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/docs/source/introduction/introduction.rst

@@ -50,6 +50,9 @@ Aerosols/clouds calibration
 Array calibration
 -----------------
 
+Relative calibration of the telescope's throughput based on reconstructed gamma-like showers energies. It is based on the expectation that all telescopes observing the same shower should reconstruct the same energy. It is complementary to muon rings calibration, helping to mitigate statistical uncertainties in cases where the muon trigger rate is low, particularly for SSTs. Moreover, unlike muon rings calibration, it uses directly the light from the showers, helping to resolve
+throughput variations with wavelength or crossed airmass. It can be used as an array "flat-fielding" or alternatively as a crosscheck of muon rings calibration. The implementation of the algorithm in based on the study documented in “Cross Calibration of Telescope Optical Throughput Efficiencies using Reconstructed Shower Energies for the Cherenkov Telescope Array”, Astroparticle Physics 75 (2016), pp. 1–7 from Alison Mitchell et al.
+
 ---------------------
 Telescope calibration
 ---------------------

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/pyproject.toml

@@ -24,8 +24,9 @@ dependencies = [
     "astral",
     "astropy",
     "cdsapi",
-    "ctapipe >= 0.18",
+    "ctapipe[all]",
     "h5py",
+    "iminuit",
     "molecularprofiles >= 2.1.0",
     "numpy",
     "pandas",
@@ -82,6 +83,10 @@ calibpipe-create-molecular-atmospheric-model = "calibpipe.tools.molecular_atmosp
 calibpipe-upload-observatory-data = "calibpipe.tools.observatory_data_db_loader:main"
 calibpipe-upload-atmospheric-model-data = "calibpipe.tools.atmospheric_model_db_loader:main"
 calibpipe-init-db = "calibpipe.tools.init_db:main"
+calibpipe-produce-camcalib-test-data = "calibpipe.tools.camcalib_test_data:main"
+calibpipe-calculate-camcalib-coefficients = "calibpipe.tools.camera_calibrator:main"
+calibpipe-calculate-throughput-muon = "calibpipe.tools.muon_throughput_calculator:main"
+calibpipe-cross-calibration = "calibpipe.tools.telescope_cross_calibration_calculator:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/report/manual_testing.tex

@@ -36,4 +36,4 @@ src/calibpipe/tests/unittests/atmosphere/test_meteo_data_handler.py X [100%]
 ========= 8 deselected, 1 xpassed, 3 warnings in 135.75s (0:02:15) =========
 \end{minted}
 
-\ManualTestCase{DPPS-UC-120.1.5}{passed}{Tested manually because GDAS endpoint is unstable. Details are provided in the manual testing section.}
+\ManualTestCase{UC-120-1.5}{passed}{Tested manually because GDAS endpoint is unstable. Details are provided in the manual testing section.}

{ctao_calibpipe-0.1.0rc9 → ctao_calibpipe-0.2.0rc2}/src/calibpipe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.0rc9'
-__version_tuple__ = version_tuple = (0, 1, 0, 'rc9')
+__version__ = version = '0.2.0rc2'
+__version_tuple__ = version_tuple = (0, 2, 0, 'rc2')