gammasimtools 0.18.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

simtools/visualization/plot_tables.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 """Plot tabular data."""
 
+import logging
 from pathlib import Path
 
 import numpy as np
@@ -9,11 +10,13 @@ from astropy.table import Table
 import simtools.utils.general as gen
 from simtools.constants import SCHEMA_PATH
 from simtools.db import db_handler
-from simtools.io_operations import legacy_data_handler
+from simtools.io import ascii_handler, legacy_data_handler
 from simtools.visualization import visualize
 
+_logger = logging.getLogger(__name__)
 
-def plot(config, output_file, db_config=None):
+
+def plot(config, output_file, db_config=None, data_path=None):
     """
     Plot tabular data from data or from model parameter files.
 
@@ -25,8 +28,10 @@ def plot(config, output_file, db_config=None):
         Output file.
     db_config: dict, optional
         Database configuration dictionary for accessing the model parameter database.
+    data_path: Path or str, optional
+        Path to the data files (optional). Expect all files to be in the same directory.
     """
-    data = read_table_data(config, db_config)
+    data = read_table_data(config, db_config, data_path)
 
     fig = visualize.plot_1d(
         data,
@@ -37,7 +42,7 @@ def plot(config, output_file, db_config=None):
     return output_file
 
 
-def read_table_data(config, db_config):
+def read_table_data(config, db_config, data_path=None):
     """
     Read table data from file or parameter database.
 
@@ -45,6 +50,10 @@ def read_table_data(config, db_config):
     ----------
     config: dict
         Configuration dictionary for plotting.
+    db_config: dict
+        Database configuration dictionary for accessing the model parameter database.
+    data_path: Path or str, optional
+        Path to the data files (optional). Expect all files to be in the same directory.
 
     Returns
     -------
@@ -57,36 +66,55 @@ def read_table_data(config, db_config):
         if "parameter" in _config:
             table = _read_table_from_model_database(_config, db_config)
         elif "file_name" in _config:
+            file_name = (
+                _config["file_name"]
+                if data_path is None or _config.get("ignore_table_data_path", False)
+                else Path(data_path) / _config["file_name"]
+            )
+            _logger.info(f"Reading tabular data from {file_name}")
             if "legacy" in _config.get("type", ""):
-                table = legacy_data_handler.read_legacy_data_as_table(
-                    _config["file_name"], _config["type"]
-                )
+                table = legacy_data_handler.read_legacy_data_as_table(file_name, _config["type"])
             else:
-                table = Table.read(_config["file_name"], format="ascii.ecsv")
+                table = Table.read(file_name, format="ascii.ecsv")
         else:
             raise ValueError("No table data defined in configuration.")
 
-        if _config.get("normalize_y"):
-            table[_config["column_y"]] = (
-                table[_config["column_y"]] / table[_config["column_y"]].max()
-            )
-        if _config.get("select_values"):
-            table = _select_values_from_table(
-                table,
-                _config["select_values"]["column_name"],
-                _config["select_values"]["value"],
-            )
-        label = _config.get("label", f"{_config.get('column_x')} vs {_config.get('column_y')}")
-        data[label] = gen.get_structure_array_from_table(
+        data[_get_plotting_label(_config, data)] = _process_table_data(table, _config)
+
+    return data
+
+
+def _get_plotting_label(config, data):
+    """Get a label for plotting based on the configuration."""
+    label = config.get("label", f"{config.get('column_x')} vs {config.get('column_y')}")
+    if label in data:
+        index = 1
+        while f"{label} ({index})" in data:
+            index += 1
+        label = f"{label} ({index})"
+    return label
+
+
+def _process_table_data(table, _config):
+    """Process table data based on configuration."""
+    if _config.get("normalize_y"):
+        table[_config["column_y"]] = table[_config["column_y"]] / table[_config["column_y"]].max()
+    if _config.get("select_values"):
+        table = _select_values_from_table(
             table,
-            [
-                _config["column_x"],
-                _config["column_y"],
-                _config.get("column_x_err"),
-                _config.get("column_y_err"),
-            ],
+            _config["select_values"]["column_name"],
+            _config["select_values"]["value"],
         )
-    return data
+
+    return gen.get_structure_array_from_table(
+        table,
+        [
+            _config["column_x"],
+            _config["column_y"],
+            _config.get("column_x_err"),
+            _config.get("column_y_err"),
+        ],
+    )
 
 
 def _read_table_from_model_database(table_config, db_config):
@@ -119,8 +147,38 @@ def _select_values_from_table(table, column_name, value):
     return table[np.isclose(table[column_name], value)]
 
 
+def _filter_config_by_plot_type(config, plot_type):
+    """Filter a configuration based on plot type."""
+    if plot_type != "all" and config.get("type") != plot_type:
+        return False
+    return True
+
+
+def _validate_config_columns(config, valid_columns, logger):
+    """Validate that all required columns in a config exist and have valid data."""
+    for table_config in config.get("tables", []):
+        required_cols = [table_config.get("column_x"), table_config.get("column_y")]
+        if not all(col in valid_columns for col in required_cols if col):
+            missing_cols = [col for col in required_cols if col not in valid_columns]
+            logger.info(
+                f"Skipping plot config {config.get('type')}: "
+                f"Missing valid data in columns: {missing_cols}"
+            )
+            return False
+    return True
+
+
+def _get_valid_columns(table):
+    """Return columns that exist and have valid data (not all NaN)."""
+    valid_columns = []
+    for col in table.colnames:
+        if not all(np.isnan(table[col])):
+            valid_columns.append(col)
+    return valid_columns
+
+
 def generate_plot_configurations(
-    parameter, parameter_version, site, telescope, output_path, plot_type
+    parameter, parameter_version, site, telescope, output_path, plot_type, db_config
 ):
     """
     Generate plot configurations for a model parameter from schema files.
@@ -129,6 +187,18 @@ def generate_plot_configurations(
     ----------
     parameter: str
         Model parameter name.
+    parameter_version: str
+        Parameter version.
+    site: str
+        Site name.
+    telescope: str
+        Telescope name.
+    output_path: str or Path
+        Output path for the plots.
+    plot_type: str
+        Plot type or "all" for all plots.
+    db_config: dict
+        Database configuration.
 
     Returns
     -------
@@ -136,26 +206,52 @@ def generate_plot_configurations(
         Tuple containing a list of plot configurations and a list of output file names.
         Return None, if no plot configurations are found.
     """
+    logger = logging.getLogger(__name__)
+
+    # Get schema configuration
     schema = gen.change_dict_keys_case(
-        gen.collect_data_from_file(
+        ascii_handler.collect_data_from_file(
             file_name=SCHEMA_PATH / "model_parameters" / f"{parameter}.schema.yml"
         )
     )
     configs = schema.get("plot_configuration")
     if not configs:
         return None
-    if plot_type != "all":
-        configs = [config for config in configs if config.get("type") == plot_type]
-        if not configs:
-            raise ValueError(
-                f"No plot configuration found for type '{plot_type}' in parameter '{parameter}'."
-            )
 
+    # Get data table and determine valid columns
+    table = _read_table_from_model_database(
+        {
+            "parameter": parameter,
+            "site": site,
+            "telescope": telescope,
+            "parameter_version": parameter_version,
+        },
+        db_config=db_config,
+    )
+    valid_columns = _get_valid_columns(table)
+
+    # Filter configs based on plot type and column validity
+    valid_configs = []
+    for config in configs:
+        if not _filter_config_by_plot_type(config, plot_type):
+            continue
+
+        if _validate_config_columns(config, valid_columns, logger):
+            valid_configs.append(config)
+
+    if not valid_configs:
+        if plot_type != "all":
+            logger.warning("No valid plot config found.")
+        return None
+
+    # Generate output files
     output_files = []
-    for _config in configs:
+    for _config in valid_configs:
         for _table in _config.get("tables", []):
             _table["parameter_version"] = parameter_version
             _table["site"] = site
+            _table["telescope"] = telescope
+
         output_files.append(
             _generate_output_file_name(
                 parameter=parameter,
@@ -167,7 +263,7 @@ def generate_plot_configurations(
             )
         )
 
-    return configs, output_files
+    return valid_configs, output_files
 
 
 def _generate_output_file_name(

simtools/visualization/visualize.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python3
 """Module for visualization."""
 
-import copy
 import logging
 import re
 from collections import OrderedDict
@@ -24,6 +23,7 @@ __all__ = [
     "set_style",
 ]
 
+
 COLORS = {}
 COLORS["classic"] = [
     "#ba2c54",
@@ -627,105 +627,6 @@ def plot_hist_2d(data, **kwargs):
     return fig
 
 
-def plot_simtel_ctapipe(filename, cleaning_args, distance, return_cleaned=False):
-    """
-    Read in a sim_telarray file and plots reconstructed photoelectrons via ctapipe.
-
-    Parameters
-    ----------
-    filename : str
-        Path to the sim_telarray file.
-    cleaning_args : tuple, optional
-        Cleaning parameters as (boundary_thresh, picture_thresh, min_number_picture_neighbors).
-    distance : astropy Quantity, optional
-        Distance to the target.
-    return_cleaned : bool, optional
-        If True, apply cleaning to the image.
-
-    Returns
-    -------
-    fig : matplotlib.figure.Figure
-        The matplotlib figure containing the plot.
-    """
-    # pylint:disable=import-outside-toplevel
-    import numpy as np
-    from ctapipe.calib import CameraCalibrator
-    from ctapipe.image import tailcuts_clean
-    from ctapipe.io import EventSource
-    from ctapipe.visualization import CameraDisplay
-
-    default_cleaning_levels = {
-        "CHEC": (2, 4, 2),
-        "LSTCam": (3.5, 7, 2),
-        "FlashCam": (3.5, 7, 2),
-        "NectarCam": (4, 8, 2),
-    }
-
-    source = EventSource(filename, max_events=1)
-    event = None
-    events = [copy.deepcopy(event) for event in source]
-    if len(events) > 1:
-        event = events[-1]
-    else:
-        try:
-            event = events[0]
-        except IndexError:
-            event = events
-    tel_id = sorted(event.r1.tel.keys())[0]
-
-    calib = CameraCalibrator(subarray=source.subarray)
-    calib(event)
-
-    geometry = source.subarray.tel[1].camera.geometry
-    image = event.dl1.tel[tel_id].image
-    cleaned = image.copy()
-
-    if return_cleaned:
-        if cleaning_args is None:
-            boundary, picture, min_neighbors = default_cleaning_levels[geometry.name]
-        else:
-            boundary, picture, min_neighbors = cleaning_args
-        mask = tailcuts_clean(
-            geometry,
-            image,
-            picture_thresh=picture,
-            boundary_thresh=boundary,
-            min_number_picture_neighbors=min_neighbors,
-        )
-        cleaned[~mask] = 0
-
-    fig, ax = plt.subplots(dpi=300)
-    title = f"CT{tel_id}, run {event.index.obs_id} event {event.index.event_id}"
-    disp = CameraDisplay(geometry, image=cleaned, norm="symlog", ax=ax)
-    disp.cmap = "RdBu_r"
-    disp.add_colorbar(fraction=0.02, pad=-0.1)
-    disp.set_limits_percent(100)
-    ax.set_title(title, pad=20)
-    ax.annotate(
-        f"tel type: {source.subarray.tel[1].type.name}\n"
-        f"optics: {source.subarray.tel[1].optics.name}\n"
-        f"camera: {source.subarray.tel[1].camera_name}\n"
-        f"distance: {distance.to(u.m)}",
-        xy=(0, 0),
-        xytext=(0.1, 1),
-        xycoords="axes fraction",
-        va="top",
-        size=7,
-    )
-    ax.annotate(
-        f"dl1 image,\ntotal $p.e._{{reco}}$: {np.round(np.sum(image))}\n",
-        xy=(0, 0),
-        xytext=(0.75, 1),
-        xycoords="axes fraction",
-        va="top",
-        ha="left",
-        size=7,
-    )
-    ax.set_axis_off()
-    fig.tight_layout()
-    return fig
-
-
 def save_figure(fig, output_file, figure_format=None, log_title="", dpi="figure"):
     """
     Save figure to output file(s).