gammasimtools 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/visualization/plot_tables.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 """Plot tabular data."""
 
+import logging
 from pathlib import Path
 
 import numpy as np
@@ -9,11 +10,13 @@ from astropy.table import Table
 import simtools.utils.general as gen
 from simtools.constants import SCHEMA_PATH
 from simtools.db import db_handler
-from simtools.io_operations import legacy_data_handler
+from simtools.io import ascii_handler, legacy_data_handler
 from simtools.visualization import visualize
 
+_logger = logging.getLogger(__name__)
 
-def plot(config, output_file, db_config=None):
+
+def plot(config, output_file, db_config=None, data_path=None):
     """
     Plot tabular data from data or from model parameter files.
 
@@ -25,8 +28,10 @@ def plot(config, output_file, db_config=None):
         Output file.
     db_config: dict, optional
         Database configuration dictionary for accessing the model parameter database.
+    data_path: Path or str, optional
+        Path to the data files (optional). Expect all files to be in the same directory.
     """
-    data = read_table_data(config, db_config)
+    data = read_table_data(config, db_config, data_path)
 
     fig = visualize.plot_1d(
         data,
@@ -37,7 +42,7 @@ def plot(config, output_file, db_config=None):
     return output_file
 
 
-def read_table_data(config, db_config):
+def read_table_data(config, db_config, data_path=None):
     """
     Read table data from file or parameter database.
 
@@ -45,6 +50,10 @@ def read_table_data(config, db_config):
     ----------
     config: dict
         Configuration dictionary for plotting.
+    db_config: dict
+        Database configuration dictionary for accessing the model parameter database.
+    data_path: Path or str, optional
+        Path to the data files (optional). Expect all files to be in the same directory.
 
     Returns
     -------
@@ -57,36 +66,55 @@ def read_table_data(config, db_config):
         if "parameter" in _config:
             table = _read_table_from_model_database(_config, db_config)
         elif "file_name" in _config:
+            file_name = (
+                _config["file_name"]
+                if data_path is None or _config.get("ignore_table_data_path", False)
+                else Path(data_path) / _config["file_name"]
+            )
+            _logger.info(f"Reading tabular data from {file_name}")
             if "legacy" in _config.get("type", ""):
-                table = legacy_data_handler.read_legacy_data_as_table(
-                    _config["file_name"], _config["type"]
-                )
+                table = legacy_data_handler.read_legacy_data_as_table(file_name, _config["type"])
             else:
-                table = Table.read(_config["file_name"], format="ascii.ecsv")
+                table = Table.read(file_name, format="ascii.ecsv")
         else:
             raise ValueError("No table data defined in configuration.")
 
-        if _config.get("normalize_y"):
-            table[_config["column_y"]] = (
-                table[_config["column_y"]] / table[_config["column_y"]].max()
-            )
-        if _config.get("select_values"):
-            table = _select_values_from_table(
-                table,
-                _config["select_values"]["column_name"],
-                _config["select_values"]["value"],
-            )
-        label = _config.get("label", f"{_config.get('column_x')} vs {_config.get('column_y')}")
-        data[label] = gen.get_structure_array_from_table(
+        data[_get_plotting_label(_config, data)] = _process_table_data(table, _config)
+
+    return data
+
+
+def _get_plotting_label(config, data):
+    """Get a label for plotting based on the configuration."""
+    label = config.get("label", f"{config.get('column_x')} vs {config.get('column_y')}")
+    if label in data:
+        index = 1
+        while f"{label} ({index})" in data:
+            index += 1
+        label = f"{label} ({index})"
+    return label
+
+
+def _process_table_data(table, _config):
+    """Process table data based on configuration."""
+    if _config.get("normalize_y"):
+        table[_config["column_y"]] = table[_config["column_y"]] / table[_config["column_y"]].max()
+    if _config.get("select_values"):
+        table = _select_values_from_table(
             table,
-            [
-                _config["column_x"],
-                _config["column_y"],
-                _config.get("column_x_err"),
-                _config.get("column_y_err"),
-            ],
+            _config["select_values"]["column_name"],
+            _config["select_values"]["value"],
         )
-    return data
+
+    return gen.get_structure_array_from_table(
+        table,
+        [
+            _config["column_x"],
+            _config["column_y"],
+            _config.get("column_x_err"),
+            _config.get("column_y_err"),
+        ],
+    )
 
 
 def _read_table_from_model_database(table_config, db_config):
@@ -119,8 +147,38 @@ def _select_values_from_table(table, column_name, value):
     return table[np.isclose(table[column_name], value)]
 
 
+def _filter_config_by_plot_type(config, plot_type):
+    """Filter a configuration based on plot type."""
+    if plot_type != "all" and config.get("type") != plot_type:
+        return False
+    return True
+
+
+def _validate_config_columns(config, valid_columns, logger):
+    """Validate that all required columns in a config exist and have valid data."""
+    for table_config in config.get("tables", []):
+        required_cols = [table_config.get("column_x"), table_config.get("column_y")]
+        if not all(col in valid_columns for col in required_cols if col):
+            missing_cols = [col for col in required_cols if col not in valid_columns]
+            logger.info(
+                f"Skipping plot config {config.get('type')}: "
+                f"Missing valid data in columns: {missing_cols}"
+            )
+            return False
+    return True
+
+
+def _get_valid_columns(table):
+    """Return columns that exist and have valid data (not all NaN)."""
+    valid_columns = []
+    for col in table.colnames:
+        if not all(np.isnan(table[col])):
+            valid_columns.append(col)
+    return valid_columns
+
+
 def generate_plot_configurations(
-    parameter, parameter_version, site, telescope, output_path, plot_type
+    parameter, parameter_version, site, telescope, output_path, plot_type, db_config
 ):
     """
     Generate plot configurations for a model parameter from schema files.
@@ -129,6 +187,18 @@ def generate_plot_configurations(
     ----------
     parameter: str
         Model parameter name.
+    parameter_version: str
+        Parameter version.
+    site: str
+        Site name.
+    telescope: str
+        Telescope name.
+    output_path: str or Path
+        Output path for the plots.
+    plot_type: str
+        Plot type or "all" for all plots.
+    db_config: dict
+        Database configuration.
 
     Returns
     -------
@@ -136,26 +206,52 @@ def generate_plot_configurations(
         Tuple containing a list of plot configurations and a list of output file names.
         Return None, if no plot configurations are found.
     """
+    logger = logging.getLogger(__name__)
+
+    # Get schema configuration
     schema = gen.change_dict_keys_case(
-        gen.collect_data_from_file(
+        ascii_handler.collect_data_from_file(
             file_name=SCHEMA_PATH / "model_parameters" / f"{parameter}.schema.yml"
         )
     )
     configs = schema.get("plot_configuration")
     if not configs:
         return None
-    if plot_type != "all":
-        configs = [config for config in configs if config.get("type") == plot_type]
-        if not configs:
-            raise ValueError(
-                f"No plot configuration found for type '{plot_type}' in parameter '{parameter}'."
-            )
 
+    # Get data table and determine valid columns
+    table = _read_table_from_model_database(
+        {
+            "parameter": parameter,
+            "site": site,
+            "telescope": telescope,
+            "parameter_version": parameter_version,
+        },
+        db_config=db_config,
+    )
+    valid_columns = _get_valid_columns(table)
+
+    # Filter configs based on plot type and column validity
+    valid_configs = []
+    for config in configs:
+        if not _filter_config_by_plot_type(config, plot_type):
+            continue
+
+        if _validate_config_columns(config, valid_columns, logger):
+            valid_configs.append(config)
+
+    if not valid_configs:
+        if plot_type != "all":
+            logger.warning("No valid plot config found.")
+        return None
+
+    # Generate output files
     output_files = []
-    for _config in configs:
+    for _config in valid_configs:
         for _table in _config.get("tables", []):
             _table["parameter_version"] = parameter_version
             _table["site"] = site
+            _table["telescope"] = telescope
+
         output_files.append(
             _generate_output_file_name(
                 parameter=parameter,
@@ -167,7 +263,7 @@ def generate_plot_configurations(
             )
         )
 
-    return configs, output_files
+    return valid_configs, output_files
 
 
 def _generate_output_file_name(