gammasimtools 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/runners/corsika_runner.py

@@ -1,7 +1,7 @@
 """Generate run scripts and directories for CORSIKA simulations."""
 
 import logging
-import os
+import stat
 from pathlib import Path
 
 from simtools.io_operations import io_handler
@@ -99,6 +99,15 @@ class CorsikaRunner:
             file_type="config_tmp", run_number=self.corsika_config.run_number
         )
         corsika_input_tmp_file = self._directory["inputs"].joinpath(corsika_input_tmp_name)
+        # CORSIKA log file naming (temporary and final)
+        corsika_log_tmp_file = (
+            self._directory["data"]
+            .joinpath(f"run{self.corsika_config.run_number:06}")
+            .joinpath(f"run{self.corsika_config.run_number}.log")
+        )
+        corsika_log_file = self.get_file_name(
+            file_type="corsika_log", run_number=self.corsika_config.run_number
+        )
 
         if use_pfp:
             pfp_command = self._get_pfp_command(corsika_input_tmp_file, corsika_input_file)
@@ -138,11 +147,13 @@ class CorsikaRunner:
                 file.write(f"cp {corsika_input_file} {corsika_input_tmp_file}")
             file.write("\n# Running corsika_autoinputs\n")
             file.write(autoinputs_command)
+            file.write("\n# Moving log files to the corsika log directory\n")
+            file.write(f"gzip {corsika_log_tmp_file}\n")
+            file.write(f"mv -v {corsika_log_tmp_file}.gz {corsika_log_file}\n")
 
             file.write('\necho "RUNTIME: $SECONDS"\n')
 
-        os.system(f"chmod ug+x {script_file_path}")
-
+        script_file_path.chmod(script_file_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
         return script_file_path
 
     def get_resources(self, run_number=None):

simtools/runners/corsika_simtel_runner.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 from simtools.runners.corsika_runner import CorsikaRunner
 from simtools.simtel.simulator_array import SimulatorArray
+from simtools.utils.general import clear_default_sim_telarray_cfg_directories
 
 __all__ = ["CorsikaSimtelRunner"]
 
@@ -197,7 +198,8 @@ class CorsikaSimtelRunner:
         )
         command += f" | gzip > {_log_file} 2>&1 || exit"
 
-        return command
+        # Remove the default sim_telarray configuration directories
+        return clear_default_sim_telarray_cfg_directories(command)
 
     def get_file_name(self, simulation_software=None, file_type=None, run_number=None, mode=None):
         """

simtools/runners/runner_services.py

@@ -163,6 +163,7 @@ class RunnerServices:
         log_suffixes = {
             "log": ".log.gz",
             "histogram": ".hdata.zst",
+            "corsika_log": ".corsika.log.gz",
         }
         return self.directory["logs"].joinpath(f"{file_name}{log_suffixes[file_type]}")
 
@@ -187,7 +188,6 @@ class RunnerServices:
         data_suffixes = {
             "output": ".zst",
             "corsika_output": ".zst",
-            "corsika_log": ".log",
             "simtel_output": ".simtel.zst",
         }
         run_dir = self._get_run_number_string(run_number)
@@ -245,10 +245,10 @@ class RunnerServices:
         """
         file_name = self._get_file_basename(run_number)
 
-        if file_type in ["log", "histogram"]:
+        if file_type in ["log", "histogram", "corsika_log"]:
             return self._get_log_file_path(file_type, file_name)
 
-        if file_type in ["output", "corsika_output", "corsika_log", "simtel_output"]:
+        if file_type in ["output", "corsika_output", "simtel_output"]:
             return self._get_data_file_path(file_type, file_name, run_number)
 
         if file_type in ("sub_log", "sub_script"):

simtools/runners/simtel_runner.py

@@ -1,7 +1,8 @@
 """Base class for running sim_telarray simulations."""
 
 import logging
-import os
+import stat
+import subprocess
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -96,7 +97,7 @@ class SimtelRunner:
 
             file.write('\necho "RUNTIME: $SECONDS"\n')
 
-        os.system(f"chmod ug+x {script_file_path}")
+        script_file_path.chmod(script_file_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
         return script_file_path
 
     def run(self, test=False, input_file=None, run_number=None):
@@ -114,15 +115,17 @@ class SimtelRunner:
         """
         self._logger.debug("Running sim_telarray")
 
-        command = self._make_run_command(run_number=run_number, input_file=input_file)
+        command, stdout_file, stderr_file = self._make_run_command(
+            run_number=run_number, input_file=input_file
+        )
 
         if test:
             self._logger.info(f"Running (test) with command: {command}")
-            self._run_simtel_and_check_output(command)
+            self._run_simtel_and_check_output(command, stdout_file, stderr_file)
         else:
             self._logger.debug(f"Running ({self.runs_per_set}x) with command: {command}")
             for _ in range(self.runs_per_set):
-                self._run_simtel_and_check_output(command)
+                self._run_simtel_and_check_output(command, stdout_file, stderr_file)
 
         self._check_run_result(run_number=run_number)
 
@@ -148,7 +151,7 @@ class SimtelRunner:
         self._logger.error(msg)
         raise SimtelExecutionError(msg)
 
-    def _run_simtel_and_check_output(self, command):
+    def _run_simtel_and_check_output(self, command, stdout_file, stderr_file):
         """
         Run the sim_telarray command and check the exit code.
 
@@ -157,8 +160,24 @@ class SimtelRunner:
         SimtelExecutionError
             if run was not successful.
         """
-        if os.system(command) != 0:
+        stdout_file = stdout_file if stdout_file else "/dev/null"
+        stderr_file = stderr_file if stderr_file else "/dev/null"
+        with (
+            open(f"{stdout_file}", "w", encoding="utf-8") as stdout,
+            open(f"{stderr_file}", "w", encoding="utf-8") as stderr,
+        ):
+            result = subprocess.run(
+                command,
+                shell=True,
+                text=True,
+                stdout=stdout,
+                stderr=stderr,
+            )
+
+        if result.returncode != 0:
+            self._logger.error(result.stderr)
             self._raise_simtel_error()
+        return result.returncode
 
     def _make_run_command(self, run_number=None, input_file=None):
         self._logger.debug(
@@ -167,7 +186,7 @@ class SimtelRunner:
         )
         input_file = input_file if input_file else "nofile"
         run_number = run_number if run_number else 1
-        return f"{input_file}-{run_number}"
+        return f"{input_file}-{run_number}", None, None
 
     @staticmethod
     def get_config_option(par, value=None, weak_option=False):

simtools/schemas/input/MST_mirror_2f_measurements.schema.yml

@@ -0,0 +1,39 @@
+%YAML 1.2
+---
+title: Schema for mirror panel 2F measurements
+version: 0.1.0
+meta_schema: simpipe-schema
+meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
+meta_schema_version: 0.1.0
+name: mirror_2f_measurement
+description: |-
+  Mirror panel 2F measurements determining
+  panel radius and panel PSF.
+data:
+  - type: data_table
+    table_columns:
+      - name: mirror_panel_id
+        description: |-
+          Mirror panel ID.
+        required: true
+        unit: dimensionless
+        type: string
+      - name: mirror_curvature_radius
+        description: |-
+          Mirror panel curvature radius.
+        required: true
+        unit: cm
+        type: float64
+      - name: psf
+        description: |-
+          Spot size of mirror panel PSF at nominal distance.
+        required: true
+        unit: cm
+        type: float64
+      - name: psf_opt
+        description: |-
+          Spot size of mirror panel PSF
+          (best value, not at nominal distance).
+        required: false
+        unit: cm
+        type: float64

simtools/schemas/input/single_pe_spectrum.schema.yml

@@ -0,0 +1,38 @@
+%YAML 1.2
+---
+title: Schema for single photon electron spectrum
+version: 0.1.0
+meta_schema: simpipe-schema
+meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
+meta_schema_version: 0.1.0
+name: single_pe_spectrum
+description: |-
+  Single photo-electron response as a function of signal amplitude.
+  May include prompt, prompt plus afterpulse, or afterpulse response.
+data:
+  - type: data_table
+    table_columns:
+      - name: amplitude
+        description: |-
+          Signal amplitude (p.e.)
+        required: true
+        unit: dimensionless
+        type: float64
+      - name: frequency (prompt)
+        description: |-
+          Single photo-electron response (prompt)
+        required: false
+        unit: dimensionless
+        type: float64
+      - name: frequency (afterpulsing)
+        description: |-
+          Single photo-electron response (afterpulsing)
+        required: false
+        unit: dimensionless
+        type: float64
+      - name: frequency (prompt+afterpulsing)
+        description: |-
+          Single photo-electron response (prompt+afterpulsing)
+        required: false
+        unit: dimensionless
+        type: float64

simtools/schemas/integration_tests_config.metaschema.yml

@@ -3,7 +3,7 @@ $schema: http://json-schema.org/draft-06/schema#
 $ref: '#/definitions/SimtoolsIntegrationTestConfiguration'
 title: SimPipe integration test configuration metaschema
 description: YAML representation of integration test configuration metaschema
-version: 0.1.0
+version: 0.2.0
 name: integration_tests_config.metaschema
 type: object
 additionalProperties: false
@@ -60,6 +60,20 @@ definitions:
         description: |
           "Path of output file tested to be generated by integration test."
         type: string
+      TEST_OUTPUT_FILES:
+        type: array
+        items:
+          type: object
+          properties:
+            PATH_DESCRIPTOR:
+              type: string
+            FILE:
+              type: string
+            EXPECTED_OUTPUT:
+              description: |
+                "Expected output of integration test."
+              type: object
+          required: ["PATH_DESCRIPTOR", "FILE"]
       FILE_TYPE:
         description: |
           "Expected file type of output file generated by integration test."
@@ -68,10 +82,16 @@ definitions:
         description: |
           "Reference file used for comparison."
         type: string
-      EXPECTED_OUTPUT:
+      TEST_SIMTEL_CFG_FILES:
         description: |
-          "Expected output of integration test."
+          "Reference file used for comparison of sim_telarray configuration files."
         type: object
+        additionalProperties: false
+        patternProperties:
+          "^[0-9]+\\.[0-9]+\\.[0-9]+$":  # Semantic versioning pattern (e.g., "5.0.0", "6.0.0")
+            type: string
+            description: Path to the configuration file for the given version.
+        minProperties: 1
       TOLERANCE:
         description: "Allowed tolerance for floating point comparison."
         type: number

simtools/schemas/model_parameter.metaschema.yml

@@ -2,12 +2,104 @@
 $schema: http://json-schema.org/draft-06/schema#
 $ref: '#/definitions/SimtoolsModelParameters'
 title: SimPipe DB Model Parameter Metaschema
-description: YAML representation of db model parameter metaschema
-version: 0.1.0
+description: |
+  YAML representation of db model parameter metaschema
+  (based on simulation model DB).
+version: 0.2.0
 name: modelparameter.metaschema
 type: object
 additionalProperties: false
 
+definitions:
+  SimtoolsModelParameters:
+    description: ""
+    type: object
+    properties:
+      _id:
+        type: string
+        description: "DB unique identifier."
+      entry_date:
+        type: string
+        description: "Value entry date."
+      file:
+        type: boolean
+        description: "This parameter is a file."
+      instrument:
+        type:
+          - string
+          - "null"
+        description: "Associated instrument."
+      site:
+        type:
+          - string
+          - "null"
+        description: "Associated CTAO site."
+        enum:
+          - North
+          - South
+          - null
+      type:
+        type: string
+        description: "Data type"
+        enum:
+          - boolean
+          - dict
+          - double
+          - file
+          - float64
+          - int
+          - int64
+          - string
+          - uint
+          - uint32
+          - uint64
+      unit:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unit of the parameter."
+      value:
+        anyOf:
+          - type: boolean
+          - type: number
+          - type: string
+          - type: "null"
+          - type: array
+        description: "Value of the parameter."
+      parameter_version:
+        anyOf:
+          - type: string
+        description: "Parameter version."
+      schema_version:
+        anyOf:
+          - type: string
+        description: "Metaschema version."
+      unique_id:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unique ID of parameter definition."
+    required:
+      - file
+      - instrument
+      - parameter_version
+      - schema_version
+      - site
+      - type
+      - unit
+      - value
+...
+---
+$schema: http://json-schema.org/draft-06/schema#
+$ref: '#/definitions/SimtoolsModelParameters'
+title: SimPipe DB Model Parameter Metaschema
+description: |
+  YAML representation of db model parameter metaschema
+  (based on model_parameters DB).
+version: 0.1.0
+name: modelparameter.metaschema
+type: object
+additionalProperties: false
 
 definitions:
   SimtoolsModelParameters:
@@ -48,6 +140,7 @@ definitions:
           - boolean
           - double
           - int
+          - int64
           - string
           - uint
           - file

simtools/schemas/model_parameter_and_data_schema.metaschema.yml

@@ -25,6 +25,7 @@ definitions:
         description: "name of meta schema (e.g. simpipe-schema)"
       meta_schema_url:
         type: string
+        format: uri
         description: "url to meta schema definition"
       meta_schema_version:
         type: string
@@ -322,6 +323,7 @@ definitions:
       - ValidateArrayElementCoordinates
       - ValidateAtmosphericModel
       - ValidateCameraChargeResponse
+      - ValidateCameraEfficiency
       - ValidateCameraGainsAndEfficiency
       - ValidateCameraGeometry
       - ValidateCameraLinearity

simtools/schemas/model_parameters/array_element_position_utm.schema.yml

@@ -7,7 +7,7 @@ meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/si
 meta_schema_version: 0.1.0
 name: array_element_position_utm
 description: |-
-  Position and alitutude of an array element (e.g., a telescope) in UTM coordinates.
+  Position and altitude of an array element (e.g., a telescope) in UTM coordinates.
 short_description: UTM coordinate position of an array element.
 data:
   - type: double

simtools/schemas/model_parameters/array_window.schema.yml

@@ -0,0 +1,37 @@
+%YAML 1.2
+---
+title: Schema for array_window model parameter
+version: 0.1.0
+meta_schema: simpipe-schema
+meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
+meta_schema_version: 0.1.0
+name: array_window
+description: |-
+  Length of a coincidence window of the default stereo trigger,
+  after correction of fixed (cable length, focal length, etc.)
+  and variable (view direction) delays.
+short_description: |-
+  Length of a coincidence window of the default stereo trigger.
+data:
+  - type: double
+    unit: ns
+    default: 1000.
+instrument:
+  class: Camera
+  type:
+    - LSTN
+    - LSTS
+    - MSTN
+    - MSTS
+    - SSTS
+    - SCTS
+activity:
+  setting:
+    - SetParameterFromExternal
+  validation:
+    - ValidateParameterByExpert
+    - ValidateTriggerPerformance
+source:
+  - Initial instrument setup
+simulation_software:
+  - name: sim_telarray

simtools/schemas/model_parameters/asum_clipping.schema.yml

@@ -22,10 +22,6 @@ instrument:
   type:
     - LSTN
     - LSTS
-    - MSTN
-    - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/channels_per_chip.schema.yml

@@ -8,7 +8,7 @@ meta_schema_version: 0.1.0
 name: channels_per_chip
 description: |-
   Number of channels per readout chip.
-  Potentially useful for crosstalk calculations.
+  Potentially useful for cross-talk calculations.
 short_description: Number of channels per readout chip.
 data:
   - type: uint

simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml

@@ -10,8 +10,8 @@ description: |-
   Maximum size of data blocks written by the CORSIKA IACT module.
 data:
   - type: int
-    unit: byte
-    default: 100000000
+    unit: MB
+    default: 1000
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_clipping.schema.yml

@@ -24,8 +24,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_ignore_below.schema.yml

@@ -24,8 +24,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_offset.schema.yml

@@ -23,8 +23,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_pedsub.schema.yml

@@ -20,8 +20,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml

@@ -25,8 +25,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_prescale.schema.yml

@@ -30,8 +30,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_presum_max.schema.yml

@@ -24,8 +24,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_presum_shift.schema.yml

@@ -31,8 +31,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_shaping.schema.yml

@@ -30,8 +30,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml

@@ -18,8 +18,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_threshold.schema.yml

@@ -28,8 +28,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/dsum_zero_clip.schema.yml

@@ -28,8 +28,6 @@ instrument:
   type:
     - MSTN
     - MSTS
-    - SSTS
-    - SCTS
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/effective_focal_length.schema.yml

@@ -33,12 +33,42 @@ short_description: |-
   Effective focal length.
   Only to be used for image analysis, has no effect on the simulation.
 data:
-  - type: double
+  - description: |-
+      Mean effective length for all directions of incidence.
+    type: double
     unit: cm
     default: 0.0
     allowed_range:
       min: 0.0
       max: 10000.0
+  - description: |-
+      Effective length for incidence directions in mirror/camera x-z plane
+      (if non-zero).
+    type: double
+    unit: cm
+    default: 0.0
+    allowed_range:
+      min: 0.0
+      max: 10000.0
+  - description: |-
+      Effective length for incidence directions in mirror/camera y-z plane
+      (if non-zero).
+    type: double
+    unit: cm
+    default: 0.0
+    allowed_range:
+      min: 0.0
+      max: 10000.0
+  - description: |-
+      Any displacement along x in the focal plane from asymmetric PSF behavior.
+    type: double
+    unit: cm
+    default: 0.0
+  - description: |-
+      Any displacement along y in the focal plane from asymmetric PSF behavior.
+    type: double
+    unit: cm
+    default: 0.0
 instrument:
   class: Structure
   type: