gammasimtools 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/applications/plot_tabular_data.py

@@ -9,6 +9,17 @@ Command line arguments
 ----------------------
 config_file (str, required)
     Configuration file name for plotting.
+output_file (str, required)
+    Output file name (without suffix).
+
+Example
+-------
+
+Plot tabular data using a configuration file.
+
+.. code-block:: console
+
+    simtools-plot-tabular-data --plot_config config_file_name --output_file output_file_name
 
 """
 
@@ -20,7 +31,7 @@ from simtools.configuration import configurator
 from simtools.constants import PLOT_CONFIG_SCHEMA
 from simtools.data_model import schema
 from simtools.data_model.metadata_collector import MetadataCollector
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.visualization import plot_tables
 
 
@@ -51,6 +62,12 @@ def _parse(label, description, usage):
         required=True,
         default=None,
     )
+    config.parser.add_argument(
+        "--table_data_path",
+        help="Path to the data files (optional). Expect all files to be in the same directory.",
+        type=str,
+        default=None,
+    )
     config.parser.add_argument(
         "--output_file",
         help="Output file name (without suffix)",
@@ -74,15 +91,16 @@ def main():
 
     plot_config = gen.convert_keys_in_dict_to_lowercase(
         schema.validate_dict_using_schema(
-            gen.collect_data_from_file(args_dict["plot_config"]),
+            ascii_handler.collect_data_from_file(args_dict["plot_config"]),
             PLOT_CONFIG_SCHEMA,
         )
     )
 
     plot_tables.plot(
-        config=plot_config["cta_simpipe"]["plot"],
+        config=plot_config["plot"],
         output_file=io_handler_instance.get_output_file(args_dict["output_file"]),
         db_config=db_config_,
+        data_path=args_dict.get("table_data_path"),
     )
 
     MetadataCollector.dump(

simtools/applications/plot_tabular_data_for_model_parameter.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+r"""
+Plot tabular data for a single model parameter using default plotting configurations.
+
+Uses plotting configurations as defined in the model parameters schema files.
+
+Command line arguments
+----------------------
+parameter (str, required)
+    Model parameter to plot (e.g., 'atmospheric_profile').
+parameter_version (str, required)
+    Version of the model parameter to plot (e.g., '1.0.0').
+site (str, required)
+    Site for which the model parameter is defined (e.g., 'North').
+telescope (str, optional)
+    Telescope for which the model parameter is defined (e.g., 'LSTN-01').
+output_file (str, required)
+    Output file name (without suffix).
+plot_type (str, optional)
+    Type of plot as defined in the schema file.
+    Use '--plot_type all' to plot all types defined in the schema.
+
+Example
+-------
+
+Plot tabular data for a specific type defined in the schema file:
+
+.. code-block:: console
+
+    simtools-plot-tabular-data-for-model-parameter \\
+        --parameter atmospheric_profile \\
+        --parameter_version 1.0.0 \\
+        --site North \\
+        --plot_type refractive_index_vs_altitude
+
+Plot tabular data for all types defined in the schema file:
+
+.. code-block:: console
+
+    simtools-plot-tabular-data-for-model-parameter \\
+        --parameter fadc_pulse_shape
+        --parameter_version 1.0.0 \\
+        --site North \\
+        --telescope LSTN-01 \\
+        --plot_type all
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.io import io_handler
+from simtools.visualization import plot_tables
+
+
+def _parse(label, description):
+    """Parse command line configuration."""
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument("--parameter", type=str, required=True, help="Parameter name.")
+    config.parser.add_argument(
+        "--plot_type",
+        help="Plot type as defined in the schema file.",
+        type=str,
+        required=True,
+        default=None,
+    )
+    return config.initialize(db_config=True, simulation_model=["telescope", "parameter_version"])
+
+
+def main():
+    """Plot tabular data."""
+    args_dict, db_config = _parse(
+        label=Path(__file__).stem,
+        description="Plots tabular data for a model parameter.",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "INFO")))
+    io_handler_instance = io_handler.IOHandler()
+
+    plot_configs, output_files = plot_tables.generate_plot_configurations(
+        parameter=args_dict["parameter"],
+        parameter_version=args_dict["parameter_version"],
+        site=args_dict["site"],
+        telescope=args_dict.get("telescope"),
+        output_path=io_handler_instance.get_output_directory(),
+        plot_type=args_dict["plot_type"],
+        db_config=db_config,
+    )
+
+    for plot_config, output_file in zip(plot_configs, output_files):
+        plot_tables.plot(
+            config=plot_config,
+            output_file=output_file,
+            db_config=db_config,
+        )
+        MetadataCollector.dump(args_dict, output_file=output_file, add_activity_name=True)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/print_version.py

@@ -6,13 +6,12 @@ The versions of simtools, the DB, sim_telarray, and CORSIKA are printed.
 
 """
 
-import json
 import logging
 from pathlib import Path
 
 from simtools import dependencies, version
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.utils import general as gen
 
 
@@ -66,15 +65,15 @@ def main():
 
     version_list = version_string.strip().split("\n")
     for version_entry in version_list:
-        key, value = version_entry.split(": ")
+        key, value = version_entry.split(": ", 1)
         version_dict[key] = value
 
-    with open(
-        io_handler_instance.get_output_file(args_dict["output_file"], label=label),
-        "w",
-        encoding="utf-8",
-    ) as f:
-        json.dump(version_dict, f, indent=4)
+    ascii_handler.write_data_to_file(
+        data=version_dict,
+        output_file=io_handler_instance.get_output_file(
+            args_dict.get("output_file", "simtools_version.json"), label=label
+        ),
+    )
 
 
 if __name__ == "__main__":

simtools/applications/production_derive_corsika_limits.py

@@ -4,34 +4,51 @@ r"""
 Derive CORSIKA configuration limits for energy, core distance, and viewcone radius.
 
 This tool determines configuration limits based on triggered events from broad-range
-simulations. It supports setting:
+simulations. It supports the derivation of the following CORSIKA configuration parameters:
 
-- **ERANGE**: Derives the lower energy limit; upper limit is user-defined.
-- **CSCAT**: Derives the upper core distance; lower limit is user-defined.
-- **VIEWCONE**: Derives the viewcone radius; lower limit is user-defined.
+- **ERANGE**: lower energy limit
+- **CSCAT**: upper core distance
+- **VIEWCONE**: viewcone radius
 
+Broad-range simulations in this context are simulation sets generated with wide-ranging
+definitions for above parameters.
 Limits are computed based on a configurable maximum event loss fraction.
 Results are provided as a table with the following columns:
 
-- particle_type: Particle type (e.g., gamma, proton, electron).
-- telescope_ids: List of telescope IDs used in the simulation.
-- zenith: Zenith angle.
-- azimuth: Azimuth angle.
-- nsb: Night sky background level
-- layout: Layout of the telescope array used in the simulation.
-- lower_energy_limit: Derived lower energy limit.
-- upper_radius_limit: Derived upper radial distance limit.
-- viewcone_radius: Derived upper viewcone radius limit.
++---------------------+-----------+--------+-----------------------------------------------+
+| Field               | Data Type | Units  | Description                                   |
++=====================+===========+========+===============================================+
+| primary_particle    | string    |        | Particle type (e.g., gamma, proton).          |
++---------------------+-----------+--------+-----------------------------------------------+
+| array_name          | string    |        | Array name (custom or as defined in           |
+|                     |           |        | 'array_layouts').                             |
++---------------------+-----------+--------+-----------------------------------------------+
+| telescope_ids       | string    |        | Comma-separated list of telescope IDs         |
+|                     |           |        | of this array.                                |
++---------------------+-----------+--------+-----------------------------------------------+
+| zenith              | float64   | deg    | Direction of array pointing zenith.           |
++---------------------+-----------+--------+-----------------------------------------------+
+| azimuth             | float64   | deg    | Direction of array pointing azimuth.          |
++---------------------+-----------+--------+-----------------------------------------------+
+| nsb_level           | float64   |        | Night sky background level.                   |
++---------------------+-----------+--------+-----------------------------------------------+
+| lower_energy_limit  | float64   | TeV    | Derived lower energy limit (**ERANGE**)       |
++---------------------+-----------+--------+-----------------------------------------------+
+| upper_radius_limit  | float64   | m      | Derived upper core distance limit (**CSCAT**) |
++---------------------+-----------+--------+-----------------------------------------------+
+| viewcone_radius     | float64   | deg    | Derived viewcone radius limit (**VIEWCONE**)  |
++---------------------+-----------+--------+-----------------------------------------------+
 
 The input event data files are generated using the application simtools-generate-simtel-event-data
-and are required for each point in the lookup table.
+and are required for each point in the observational parameter space (e.g., array pointing
+directions, level of night sky background, etc.).
 
 Command line arguments
 ----------------------
-event_data_files (str, required)
-    Path to a file containing event data files derived with 'simtools-generate-simtel-event-data'.
-telescope_ids (str, required)
-    Path to a file containing telescope configurations.
+event_data_file (str, required)
+    Path to reduced event data file.
+telescope_ids (str, optional)
+    Custom array layout file containing telescope IDs.
 loss_fraction (float, required)
     Maximum event-loss fraction for limit computation.
 plot_histograms (bool, optional)
@@ -41,12 +58,24 @@ output_file (str, optional)
 
 Example
 -------
-Derive limits for a given file with a specified loss fraction.
+
+Derive limits for a list of array layouts (use 'all' to derive limits for all layouts):
 
 .. code-block:: console
 
     simtools-production-derive-corsika-limits \\
-        --event_data_files path/to/event_data_files.yaml \\
+        --event_data_file event_dat_file.hdf5 \\
+        --array_layout_name alpha,beta \\
+        --loss_fraction 1e-6 \\
+        --plot_histograms \\
+        --output_file corsika_simulation_limits_lookup.ecsv
+
+Derive limits for a given file for custom defined array layouts:
+
+.. code-block:: console
+
+    simtools-production-derive-corsika-limits \\
+        --event_data_file event_dat_file.hdf5 \\
         --telescope_ids path/to/telescope_configs.yaml \\
         --loss_fraction 1e-6 \\
         --plot_histograms \\
@@ -57,7 +86,7 @@ import logging
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.production_configuration.derive_corsika_limits_grid import (
+from simtools.production_configuration.derive_corsika_limits import (
     generate_corsika_limits_grid,
 )
 
@@ -68,15 +97,15 @@ def _parse():
         description="Derive limits for energy, radial distance, and viewcone."
     )
     config.parser.add_argument(
-        "--event_data_files",
+        "--event_data_file",
         type=str,
         required=True,
-        help="Path to a file containing event data file paths.",
+        help="Event data file containing reduced event data.",
     )
     config.parser.add_argument(
         "--telescope_ids",
         type=str,
-        required=True,
+        required=False,
         help="Path to a file containing telescope configurations.",
     )
     config.parser.add_argument(
@@ -91,17 +120,25 @@ def _parse():
         action="store_true",
         default=False,
     )
-    return config.initialize(db_config=False, output=True)
+    return config.initialize(
+        db_config=True,
+        output=True,
+        simulation_model=[
+            "site",
+            "model_version",
+            "layout",
+        ],
+    )
 
 
 def main():
     """Derive limits for energy, radial distance, and viewcone."""
-    args_dict, _ = _parse()
+    args_dict, db_config = _parse()
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
 
-    generate_corsika_limits_grid(args_dict)
+    generate_corsika_limits_grid(args_dict, db_config)
 
 
 if __name__ == "__main__":

simtools/applications/production_derive_statistics.py

@@ -54,7 +54,7 @@ import logging
 from pathlib import Path
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.production_configuration.derive_production_statistics_handler import (
     ProductionStatisticsHandler,
 )

simtools/applications/production_generate_grid.py

@@ -49,10 +49,10 @@ from astropy.coordinates import EarthLocation
 from astropy.time import Time
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
+from simtools.io.ascii_handler import collect_data_from_file
 from simtools.model.site_model import SiteModel
 from simtools.production_configuration.generate_production_grid import GridGeneration
-from simtools.utils.general import collect_data_from_file
 
 
 def _parse(label, description):

simtools/applications/production_merge_corsika_limits.py

@@ -0,0 +1,214 @@
+#!/usr/bin/python3
+
+r"""
+Merge CORSIKA limit tables from multiple grid points and check grid completeness.
+
+The corsika limit tables are first generated by the application
+'simtools-production-derive-corsika-limits' and contain derived limits for
+energy, radial distance, and viewcone for different observational conditions.
+The individual tables are merged into a single table for each grid point by
+the application 'merge_tables'. This tool finalizes the merging process and
+checks the completeness of the grid against a provided grid definition file.
+
+This tool supports three main use cases:
+
+1. Merge multiple CORSIKA limit tables into a single file and optionally generate
+   plots of the derived limits.
+2. Merge tables and also check for grid completeness against a provided grid
+   definition file. This requires the --grid_definition parameter. Coverage plots
+   can also be generated.
+3. Check grid completeness of an already merged table file. This requires both
+   the --merged_table and --grid_definition parameters.
+
+Command line arguments
+----------------------
+input_files (str)
+    Directory containing corsika_simulation_limits_lookup*.ecsv files to be merged,
+    or a list of specific files to merge. Not used if --merged_table is provided.
+input_files_list (str)
+    Path to a text file containing a list of input files (one file path per line)
+    to be merged. Not used if --merged_table is provided.
+merged_table (str)
+    Path to an already merged table file. Used for checking grid completeness.
+grid_definition (str)
+    Path to a YAML file defining the expected grid points. Required for grid
+    completeness checks and coverage plots.
+output_file (str, optional)
+    Name of the output file for the merged limits table.
+    Default is "merged_corsika_limits.ecsv".
+plot_grid_coverage (bool, optional)
+    Flag to generate plots showing grid coverage. Requires --grid_definition.
+plot_limits (bool, optional)
+    Flag to generate plots showing the derived limits.
+
+Examples
+--------
+1. Merge CORSIKA limit tables from a directory:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --input_files "simtools-output/corsika_limits/" \\
+          --output_file merged_limits.ecsv --plot_limits
+
+2. Merge tables and check grid completeness:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --input_files "simtools-output/corsika_limits/" \\
+          --grid_definition grid_definition.yaml \\
+          --output_file merged_limits.ecsv --plot_grid_coverage
+
+3. Check grid completeness of an existing merged table:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --merged_table merged_limits.ecsv \\
+          --grid_definition grid_definition.yaml --plot_grid_coverage
+
+4. Merge tables using a list of files from a text file:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --input_files_list file_list.txt \\
+          --output_file merged_limits.ecsv
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.data_model import data_reader
+from simtools.io import ascii_handler
+from simtools.production_configuration.merge_corsika_limits import CorsikaMergeLimits
+
+_logger = logging.getLogger(__name__)
+
+
+def _parse():
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        description="Merge CORSIKA limit tables and check grid completeness."
+    )
+    config.parser.add_argument(
+        "--input_files",
+        type=str,
+        default=None,
+        nargs="+",
+        help=(
+            "A list of input files to be merged, or a single directory "
+            "containing the files (*.ecsv)."
+        ),
+    )
+    config.parser.add_argument(
+        "--input_files_list",
+        type=str,
+        default=None,
+        help=(
+            "Path to a text file containing a list of input files (one file path per line) "
+            "to be merged."
+        ),
+    )
+    config.parser.add_argument(
+        "--merged_table",
+        type=str,
+        default=None,
+        help="Path to an already merged table file.",
+    )
+    config.parser.add_argument(
+        "--grid_definition",
+        type=str,
+        default=None,
+        help="Path to YAML file defining the expected grid points.",
+    )
+    config.parser.add_argument(
+        "--plot_grid_coverage",
+        help="Generate plots showing grid coverage.",
+        action="store_true",
+        default=False,
+    )
+    config.parser.add_argument(
+        "--plot_limits",
+        help="Generate plots showing the derived limits.",
+        action="store_true",
+        default=False,
+    )
+    return config.initialize(output=True)
+
+
+def main():
+    """Merge CORSIKA limit tables and check grid completeness."""
+    args_dict, _ = _parse()
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
+
+    merger = CorsikaMergeLimits()
+    grid_definition = (
+        ascii_handler.collect_data_from_file(args_dict["grid_definition"])
+        if args_dict.get("grid_definition")
+        else None
+    )
+
+    if args_dict.get("merged_table"):
+        # Case 3: Check coverage on an existing merged table
+        merged_table_path = Path(args_dict["merged_table"]).expanduser()
+        merged_table = data_reader.read_table_from_file(merged_table_path)
+        input_files = [merged_table_path]
+    elif args_dict.get("input_files") or args_dict.get("input_files_list"):
+        # Case 1 & 2: Merge files
+        input_files = []
+
+        # Process input_files argument
+        if args_dict.get("input_files"):
+            raw_paths = args_dict.get("input_files")
+            if len(raw_paths) == 1 and Path(raw_paths[0]).expanduser().is_dir():
+                input_dir = Path(raw_paths[0]).expanduser()
+                input_files.extend(input_dir.glob("*.ecsv"))
+            else:
+                input_files.extend(Path(f).expanduser() for f in raw_paths)
+
+        # Process input_files_list argument
+        if args_dict.get("input_files_list"):
+            files_from_list = merger.read_file_list(args_dict["input_files_list"])
+            input_files.extend(files_from_list)
+
+        if not input_files:
+            raise FileNotFoundError(
+                "No input files found. Check --input_files or --input_files_list arguments."
+            )
+        merged_table = merger.merge_tables(input_files)
+    else:
+        raise ValueError(
+            "Either --input_files, --input_files_list, or --merged_table must be provided."
+        )
+
+    is_complete, grid_completeness = merger.check_grid_completeness(merged_table, grid_definition)
+
+    if args_dict.get("plot_grid_coverage"):
+        merger.plot_grid_coverage(merged_table, grid_definition)
+
+    if args_dict.get("plot_limits"):
+        merger.plot_limits(merged_table)
+
+    if not args_dict.get("merged_table"):
+        # Write output file only when merging
+        output_file = merger.output_dir / args_dict["output_file"]
+        merger.write_merged_table(
+            merged_table,
+            output_file,
+            input_files,
+            {
+                "is_complete": is_complete,
+                "expected": grid_completeness.get("expected", 0),
+                "found": grid_completeness.get("found", 0),
+                "missing": grid_completeness.get("missing", []),
+            },
+        )
+
+
+if __name__ == "__main__":
+    main()