gammasimtools 0.16.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/simtel/simtel_io_event_writer.py

@@ -1,10 +1,11 @@
-"""Generate a reduced dataset from given simulation event list and save the output to file."""
+"""Generate a reduced dataset from sim_telarray output files using astropy tables."""
 
 import logging
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 
+import astropy.units as u
 import numpy as np
-import tables
+from astropy.table import Table
 from eventio import EventIOFile
 from eventio.simtel import (
     ArrayEvent,
@@ -15,107 +16,132 @@ from eventio.simtel import (
     TriggerInformation,
 )
 
+from simtools.corsika.primary_particle import PrimaryParticle
+from simtools.simtel.simtel_io_file_info import get_corsika_run_header
 from simtools.utils.geometry import calculate_circular_mean
 
-DEFAULT_FILTERS = tables.Filters(complevel=5, complib="zlib", shuffle=True, bitshuffle=False)
-
 
 @dataclass
-class ShowerEventData:
-    """Shower event data."""
-
-    simulated_energy: list = field(default_factory=list)
-    x_core: list = field(default_factory=list)
-    y_core: list = field(default_factory=list)
-    shower_azimuth: list = field(default_factory=list)
-    shower_altitude: list = field(default_factory=list)
-    shower_id: list = field(default_factory=list)
-    area_weight: list = field(default_factory=list)
-
-    x_core_shower: list = field(default_factory=list)
-    y_core_shower: list = field(default_factory=list)
-    core_distance_shower: list = field(default_factory=list)
-
-
-@dataclass
-class TriggeredEventData:
-    """Triggered event data."""
-
-    triggered_id: list = field(default_factory=list)
-    array_altitudes: list = field(default_factory=list)
-    array_azimuths: list = field(default_factory=list)
-    trigger_telescope_list_list: list = field(default_factory=list)
-    angular_distance: list = field(default_factory=list)
+class TableSchemas:
+    """Define schemas for output tables with units."""
+
+    shower_schema = {
+        "shower_id": (np.uint32, None),
+        "event_id": (np.uint32, None),
+        "file_id": (np.uint32, None),
+        "simulated_energy": (np.float64, u.TeV),
+        "x_core": (np.float64, u.m),
+        "y_core": (np.float64, u.m),
+        "shower_azimuth": (np.float64, u.rad),
+        "shower_altitude": (np.float64, u.rad),
+        "area_weight": (np.float64, None),
+    }
+
+    trigger_schema = {
+        "shower_id": (np.uint32, None),
+        "event_id": (np.uint32, None),
+        "file_id": (np.uint32, None),
+        "array_altitude": (np.float64, u.rad),
+        "array_azimuth": (np.float64, u.rad),
+        "telescope_list": (str, None),  # Store as comma-separated string
+    }
+
+    file_info_schema = {
+        "file_name": (str, None),
+        "file_id": (np.uint32, None),
+        "particle_id": (np.uint32, None),
+        "energy_min": (np.float64, u.TeV),
+        "energy_max": (np.float64, u.TeV),
+        "viewcone_min": (np.float64, u.deg),
+        "viewcone_max": (np.float64, u.deg),
+        "core_scatter_min": (np.float64, u.m),
+        "core_scatter_max": (np.float64, u.m),
+        "zenith": (np.float64, u.deg),
+        "azimuth": (np.float64, u.deg),
+        "nsb_level": (np.float64, None),
+    }
 
 
 class SimtelIOEventDataWriter:
     """
-    Generate a reduced dataset from given simulation event list and save the output to file.
+    Process sim_telarray events and write tables to file.
+
+    Extracts essential information from sim_telarray output files:
+
+    - Shower parameters (energy, core location, direction)
+    - Trigger patterns
+    - Telescope pointing
 
     Attributes
     ----------
     input_files : list
         List of input file paths to process.
-    output_file : str
-        Path to the output file.
     max_files : int, optional
         Maximum number of files to process.
     """
 
-    def __init__(self, input_files, output_file, max_files=100):
+    def __init__(self, input_files, max_files=100):
         """Initialize class."""
         self._logger = logging.getLogger(__name__)
         self.input_files = input_files
-        self.output_file = output_file
         try:
             self.max_files = max_files if max_files < len(input_files) else len(input_files)
         except TypeError as exc:
             raise TypeError("No input files provided.") from exc
-        self.shower = None
+
         self.n_use = None
-        self.shower_id_offset = 0
-        self.event_data = ShowerEventData()
-        self.triggered_data = TriggeredEventData()
-        self.file_names = []
+        self.shower_data = []
+        self.trigger_data = []
+        self.file_info = []
 
     def process_files(self):
-        """Process the input files and store them in an file."""
-        self.shower_id_offset = 0
-
-        for i, file in enumerate(self.input_files[: self.max_files], start=1):
-            self._logger.info(f"Processing file {i}/{self.max_files}: {file}")
-            self._process_file(file)
-            if i == 1 or len(self.event_data.simulated_energy) >= 1e7:
-                self._write_data(mode="w" if i == 1 else "a")
-                self.shower_id_offset += len(self.event_data.simulated_energy)
-                self._reset_data()
-
-        self._write_data(mode="a")
-
-    def get_event_data(self):
         """
-        Return shower and triggered event data.
+        Process input files and return tables.
 
         Returns
         -------
-        ShowerEventData, TriggeredEventData
-            Shower and triggered event data.
+        list
+            List of astropy tables containing processed data.
         """
-        return self.event_data, self.triggered_data
-
-    def _process_file(self, file):
+        for i, file in enumerate(self.input_files[: self.max_files]):
+            self._logger.info(f"Processing file {i + 1}/{self.max_files}: {file}")
+            self._process_file(i, file)
+
+        return self.create_tables()
+
+    def create_tables(self):
+        """Create astropy tables from collected data."""
+        tables = []
+        for data, schema, name in [
+            (self.shower_data, TableSchemas.shower_schema, "SHOWERS"),
+            (self.trigger_data, TableSchemas.trigger_schema, "TRIGGERS"),
+            (self.file_info, TableSchemas.file_info_schema, "FILE_INFO"),
+        ]:
+            table = Table(rows=data, names=schema.keys())
+            table.meta["EXTNAME"] = name
+            self._add_units_to_table(table, schema)
+            tables.append(table)
+        return tables
+
+    def _add_units_to_table(self, table, schema):
+        """Add units to a single table's columns."""
+        for col, (_, unit) in schema.items():
+            if unit is not None:
+                table[col].unit = unit
+
+    def _process_file(self, file_id, file):
         """Process a single file and update data lists."""
+        self._process_file_info(file_id, file)
         with EventIOFile(file) as f:
             for eventio_object in f:
                 if isinstance(eventio_object, MCRunHeader):
                     self._process_mc_run_header(eventio_object)
                 elif isinstance(eventio_object, MCShower):
-                    self._process_mc_shower(eventio_object)
+                    self._process_mc_shower(eventio_object, file_id)
                 elif isinstance(eventio_object, MCEvent):
                     self._process_mc_event(eventio_object)
                 elif isinstance(eventio_object, ArrayEvent):
-                    self._process_array_event(eventio_object)
-            self.file_names.append(str(file))
+                    self._process_array_event(eventio_object, file_id)
 
     def _process_mc_run_header(self, eventio_object):
         """Process MC run header and update data lists."""
@@ -123,36 +149,93 @@ class SimtelIOEventDataWriter:
         self.n_use = mc_head["n_use"]  # reuse factor n_use needed to extend the values below
         self._logger.info(f"Shower reuse factor: {self.n_use} (viewcone: {mc_head['viewcone']})")
 
-    def _process_mc_shower(self, eventio_object):
+    def _process_file_info(self, file_id, file):
+        """Process file information and append to file info list."""
+        run_info = get_corsika_run_header(file)
+        particle = PrimaryParticle(
+            particle_id_type="eventio_id", particle_id=run_info.get("primary_id", 1)
+        )
+        self.file_info.append(
+            {
+                "file_name": str(file),
+                "file_id": file_id,
+                "particle_id": particle.corsika7_id,
+                "energy_min": run_info["E_range"][0],
+                "energy_max": run_info["E_range"][1],
+                "viewcone_min": run_info["viewcone"][0],
+                "viewcone_max": run_info["viewcone"][1],
+                "core_scatter_min": run_info["core_range"][0],
+                "core_scatter_max": run_info["core_range"][1],
+                "zenith": 90.0 - np.degrees(run_info["direction"][1]),
+                "azimuth": np.degrees(run_info["direction"][0]),
+                "nsb_level": self._get_preliminary_nsb_level(str(file)),
+            }
+        )
+
+    def _process_mc_shower(self, eventio_object, file_id):
         """
         Process MC shower and update shower event list.
 
         Duplicated entries 'self.n_use' times to match the number simulated events with
         different core positions.
         """
-        self.shower = eventio_object.parse()
-
-        self.event_data.simulated_energy.extend([self.shower["energy"]] * self.n_use)
-        self.event_data.shower_azimuth.extend([self.shower["azimuth"]] * self.n_use)
-        self.event_data.shower_altitude.extend([self.shower["altitude"]] * self.n_use)
+        shower = eventio_object.parse()
+
+        self.shower_data.extend(
+            {
+                "shower_id": shower["shower"],
+                "event_id": None,  # filled in _process_mc_event
+                "file_id": file_id,
+                "simulated_energy": shower["energy"],
+                "x_core": None,  # filled in _process_mc_event
+                "y_core": None,  # filled in _process_mc_event
+                "shower_azimuth": shower["azimuth"],
+                "shower_altitude": shower["altitude"],
+                "area_weight": None,  # filled in _process_mc_event
+            }
+            for _ in range(self.n_use)
+        )
 
     def _process_mc_event(self, eventio_object):
-        """Process MC event and update shower event list."""
+        """
+        Process MC event and update shower event list.
+
+        Expected to be called n_use times after _process_shower.
+        """
         event = eventio_object.parse()
 
-        self.event_data.shower_id.append(event["shower_num"])
-        self.event_data.x_core.append(event["xcore"])
-        self.event_data.y_core.append(event["ycore"])
-        self.event_data.area_weight.append(event["aweight"])
+        shower_data_index = len(self.shower_data) - self.n_use + event["event_id"] % 100
 
-    def _process_array_event(self, eventio_object):
+        try:
+            if self.shower_data[shower_data_index]["shower_id"] != event["shower_num"]:
+                raise IndexError
+        except IndexError as exc:
+            raise IndexError(
+                f"Inconsistent shower and MC event data for shower id {event['shower_num']}"
+            ) from exc
+
+        self.shower_data[shower_data_index].update(
+            {
+                "event_id": event["event_id"],
+                "x_core": event["xcore"],
+                "y_core": event["ycore"],
+                "area_weight": event["aweight"],
+            }
+        )
+
+    def _process_array_event(self, eventio_object, file_id):
         """Process array event and update triggered event list."""
         tracking_positions = []
+        telescopes = []
 
-        for _, obj in enumerate(eventio_object):
+        for obj in eventio_object:
             if isinstance(obj, TriggerInformation):
-                self._process_trigger_information(obj)
-
+                trigger_info = obj.parse()
+                telescopes = (
+                    trigger_info["triggered_telescopes"]
+                    if len(trigger_info["triggered_telescopes"]) > 0
+                    else []
+                )
             if isinstance(obj, TrackingPosition):
                 tracking_position = obj.parse()
                 tracking_positions.append(
@@ -162,156 +245,53 @@ class SimtelIOEventDataWriter:
                     }
                 )
 
-        if tracking_positions:
-            self._process_tracking_positions(tracking_positions)
+        if len(telescopes) > 0 and tracking_positions:
+            self._fill_array_event(telescopes, tracking_positions, eventio_object.event_id, file_id)
 
-    def _process_tracking_positions(self, tracking_positions):
-        """
-        Process collected tracking positions and update triggered event list.
-
-        Use mean telescope tracking positions, averaged over all triggered telescopes.
-        """
+    def _fill_array_event(self, telescopes, tracking_positions, event_id, file_id):
+        """Add array event triggered events with tracking positions."""
         altitudes = [pos["altitude"] for pos in tracking_positions]
         azimuths = [pos["azimuth"] for pos in tracking_positions]
 
-        self.triggered_data.array_altitudes.append(np.mean(altitudes))
-        self.triggered_data.array_azimuths.append(calculate_circular_mean(azimuths))
-
-    def _process_trigger_information(self, trigger_info):
-        """Process trigger information and update triggered event list."""
-        trigger_info = trigger_info.parse()
-        telescopes = trigger_info["triggered_telescopes"]
-        if len(telescopes) > 0:
-            # add offset to obtain unique shower IDs among all files
-            self.triggered_data.triggered_id.append(self.shower["shower"] + self.shower_id_offset)
-            self.triggered_data.trigger_telescope_list_list.append(
-                np.array(telescopes, dtype=np.int16)
-            )
-
-    def _table_descriptions(self):
-        """HDF5 table descriptions for shower data, triggered data, and file names."""
-        shower_data_desc = {
-            "shower_id": tables.Int32Col(),
-            "simulated_energy": tables.Float32Col(),
-            "x_core": tables.Float32Col(),
-            "y_core": tables.Float32Col(),
-            "area_weight": tables.Float32Col(),
-            "shower_azimuth": tables.Float32Col(),
-            "shower_altitude": tables.Float32Col(),
-        }
-        triggered_data_desc = {
-            "triggered_id": tables.Int32Col(),
-            "array_altitudes": tables.Float32Col(),
-            "array_azimuths": tables.Float32Col(),
-            "telescope_list_index": tables.Int32Col(),  # Index into VLArray
-        }
-        file_names_desc = {
-            "file_names": tables.StringCol(256),
-        }
-        return shower_data_desc, triggered_data_desc, file_names_desc
-
-    def _tables(self, output_file, data_group, mode="a"):
-        """Create or get HDF5 tables."""
-        descriptions = self._table_descriptions()
-        table_names = ["reduced_data", "triggered_data", "file_names"]
-
-        table_dict = {}
-        for name, desc in zip(table_names, descriptions):
-            path = f"/data/{name}"
-            table_dict[name] = (
-                output_file.create_table(
-                    data_group, name, desc, name.replace("_", " ").title(), filters=DEFAULT_FILTERS
-                )
-                if mode == "w" or path not in output_file
-                else output_file.get_node(path)
-            )
-
-        return table_dict["reduced_data"], table_dict["triggered_data"], table_dict["file_names"]
-
-    def _write_event_data(self, reduced_table):
-        """Fill event data tables."""
-        if len(self.event_data.simulated_energy) == 0:
-            return
-        row = reduced_table.row
-        for i, energy in enumerate(self.event_data.simulated_energy):
-            row["shower_id"] = (
-                self.event_data.shower_id[i] if i < len(self.event_data.shower_id) else 0
-            )
-            row["simulated_energy"] = energy
-            row["x_core"] = self.event_data.x_core[i] if i < len(self.event_data.x_core) else 0
-            row["y_core"] = self.event_data.y_core[i] if i < len(self.event_data.y_core) else 0
-            row["area_weight"] = (
-                self.event_data.area_weight[i] if i < len(self.event_data.area_weight) else 0
-            )
-            row["shower_azimuth"] = (
-                self.event_data.shower_azimuth[i] if i < len(self.event_data.shower_azimuth) else 0
-            )
-            row["shower_altitude"] = (
-                self.event_data.shower_altitude[i]
-                if i < len(self.event_data.shower_altitude)
-                else 0
-            )
-            row.append()
-        reduced_table.flush()
-
-    def _writer_triggered_data(self, triggered_table, vlarray):
-        """Fill triggered event data tables."""
-        # Get or create VLArray for telescope lists
-        if len(self.triggered_data.triggered_id) == 0:
-            return
-        row = triggered_table.row
-        start_idx = vlarray.nrows
-        for i, triggered_id in enumerate(self.triggered_data.triggered_id):
-            row["triggered_id"] = triggered_id
-            row["array_altitudes"] = (
-                self.triggered_data.array_altitudes[i]
-                if i < len(self.triggered_data.array_altitudes)
-                else 0
-            )
-            row["array_azimuths"] = (
-                self.triggered_data.array_azimuths[i]
-                if i < len(self.triggered_data.array_azimuths)
-                else 0
-            )
-            row["telescope_list_index"] = start_idx + i  # Index into the VLArray
-            row.append()
-            vlarray.append(
-                self.triggered_data.trigger_telescope_list_list[i]
-                if i < len(self.triggered_data.trigger_telescope_list_list)
-                else []
-            )
-        triggered_table.flush()
-
-    def _write_data(self, mode="a"):
-        """Write data to HDF5 file."""
-        with tables.open_file(self.output_file, mode=mode) as f:
-            data_group = (
-                f.create_group("/", "data", "Data group")
-                if mode == "w" or "/data" not in f
-                else f.get_node("/data")
-            )
-
-            reduced_table, triggered_table, file_names_table = self._tables(f, data_group, mode)
-            self._write_event_data(reduced_table)
-
-            vlarray = (
-                f.create_vlarray(
-                    data_group,
-                    "trigger_telescope_list_list",
-                    tables.Int16Atom(),
-                    "List of triggered telescope IDs",
-                )
-                if mode == "w" or "/data/trigger_telescope_list_list" not in f
-                else f.get_node("/data/trigger_telescope_list_list")
-            )
-            self._writer_triggered_data(triggered_table, vlarray)
-
-            if self.file_names:
-                file_names_table.append([[name] for name in self.file_names])
-                file_names_table.flush()
-
-    def _reset_data(self):
-        """Reset data structures for batch processing."""
-        self.event_data = ShowerEventData()
-        self.triggered_data = TriggeredEventData()
-        self.file_names = []
+        self.trigger_data.append(
+            {
+                "shower_id": self.shower_data[-1]["shower_id"],
+                "event_id": event_id,
+                "file_id": file_id,
+                "array_altitude": float(np.mean(altitudes)),
+                "array_azimuth": float(calculate_circular_mean(azimuths)),
+                "telescope_list": ",".join(map(str, telescopes)),
+            }
+        )
+
+    def _get_preliminary_nsb_level(self, file):
+        """
+        Return preliminary NSB level from file name.
+
+        Hardwired values are used for "dark", "half", and "full" NSB levels
+        (actual values are made up for this example). Will be replaced with
+        reading of sim_telarray metadata entry for NSB level (to be implemented,
+        see issue #1572).
+
+        Parameters
+        ----------
+        file : str
+            File name to extract NSB level from.
+
+        Returns
+        -------
+        float
+            NSB level extracted from file name.
+        """
+        nsb_levels = {"dark": 1.0, "half": 2.0, "full": 5.0}
+
+        for key, value in nsb_levels.items():
+            try:
+                if key in file.lower():
+                    self._logger.warning(f"NSB level set to hardwired value of {value}")
+                    return value
+            except AttributeError as exc:
+                raise AttributeError("Invalid file name.") from exc
+
+        self._logger.warning("No NSB level found in file name, defaulting to 1.0")
+        return 1.0

simtools/simtel/simtel_io_file_info.py

@@ -2,7 +2,7 @@
 """Read file info and run headers from sim_telarray files."""
 
 from eventio import EventIOFile
-from eventio.simtel import MCRunHeader, RunHeader
+from eventio.simtel import MCRunHeader, MCShower, RunHeader
 
 
 def get_corsika_run_number(file):
@@ -25,10 +25,10 @@ def get_corsika_run_number(file):
 
 def get_corsika_run_header(file):
     """
-    Return the CORSIKA run header from a sim_telarray file.
+    Return the CORSIKA run header information from a sim_telarray file.
 
     Reads both RunHeader and MCRunHeader object from file and
-    returns a merged dictionary.
+    returns a merged dictionary. Adds primary id from the first event.
 
     Parameters
     ----------
@@ -42,6 +42,7 @@ def get_corsika_run_header(file):
     """
     run_header = None
     mc_run_header = None
+    primary_id = None
 
     with EventIOFile(file) as f:
         for o in f:
@@ -49,9 +50,13 @@ def get_corsika_run_header(file):
                 run_header = o.parse()
             elif isinstance(o, MCRunHeader) and mc_run_header is None:
                 mc_run_header = o.parse()
-            if run_header and mc_run_header:
+            elif isinstance(o, MCShower):  # get primary_id from first MCShower
+                primary_id = o.parse().get("primary_id")
+            if run_header and mc_run_header and primary_id is not None:
                 break
 
     run_header = run_header or {}
     mc_run_header = mc_run_header or {}
+    if primary_id is not None:
+        mc_run_header["primary_id"] = primary_id
     return run_header | mc_run_header or None

simtools/simtel/simtel_io_metadata.py

@@ -1,11 +1,14 @@
 #!/usr/bin/python3
 """Read metadata from sim_telarray files."""
 
+import logging
 from functools import cache
 
 from eventio import EventIOFile
 from eventio.simtel import HistoryMeta
 
+_logger = logging.getLogger(__name__)
+
 
 @cache
 def read_sim_telarray_metadata(file, encoding="utf8"):
@@ -28,10 +31,6 @@ def read_sim_telarray_metadata(file, encoding="utf8"):
     telescope_meta: dict
         Dictionary of telescope metadata, keyed by telescope ID.
     """
-
-    def decode(meta):
-        return {k.decode(encoding): v.decode(encoding) for k, v in meta.items()}
-
     global_meta = None
     telescope_meta = {}
 
@@ -45,7 +44,7 @@ def read_sim_telarray_metadata(file, encoding="utf8"):
                     break
                 continue
 
-            meta = decode(o.parse())
+            meta = _decode_dictionary(o.parse(), encoding=encoding)
             if o.header.id == -1:
                 global_meta = meta
             else:
@@ -62,6 +61,22 @@ def read_sim_telarray_metadata(file, encoding="utf8"):
     }
 
 
+def _decode_dictionary(meta, encoding="utf8"):
+    """Decode metadata dictionary."""
+
+    def safe_decode(byte_str, encoding, errors="ignore"):
+        return byte_str.decode(encoding, errors=errors)
+
+    try:
+        return {k.decode(encoding, errors="ignore"): v.decode(encoding) for k, v in meta.items()}
+    except UnicodeDecodeError as e:
+        _logger.warning(
+            f"Failed to decode metadata with encoding {encoding}: {e}. "
+            "Falling back to 'utf-8' with errors='ignore'."
+        )
+        return {safe_decode(k, encoding): safe_decode(v, encoding) for k, v in meta.items()}
+
+
 def get_sim_telarray_telescope_id(telescope_name, file):
     """
     Return the telescope ID for a given telescope name in a sim_telarray file.

simtools/simtel/simulator_array.py

@@ -71,7 +71,7 @@ class SimulatorArray(SimtelRunner):
         config_dir = self.corsika_config.array_model.get_config_directory()
         self._log_file = self.get_file_name(file_type="log", run_number=run_number)
         histogram_file = self.get_file_name(file_type="histogram", run_number=run_number)
-        output_file = self.get_file_name(file_type="output", run_number=run_number)
+        output_file = self.get_file_name(file_type="simtel_output", run_number=run_number)
         self.corsika_config.array_model.export_all_simtel_config_files()
 
         command = str(self._simtel_path.joinpath("sim_telarray/bin/sim_telarray"))
@@ -124,7 +124,7 @@ class SimulatorArray(SimtelRunner):
         InvalidOutputFileError
             If simtel output file does not exist.
         """
-        output_file = self.get_file_name(file_type="output", run_number=run_number)
+        output_file = self.get_file_name(file_type="simtel_output", run_number=run_number)
         if not output_file.exists():
             msg = f"sim_telarray output file {output_file} does not exist."
             self._logger.error(msg)