gammasimtools 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/db/db_model_upload.py

@@ -0,0 +1,139 @@
+"""Upload a simulation model (parameters and production tables) to the database."""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.utils import names
+
+logger = logging.getLogger(__name__)
+
+
+def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
+    """
+    Upload new model parameter from json files to db.
+
+    Parameters
+    ----------
+    file : list
+        Json file to be uploaded to the DB.
+    collection : str
+        The DB collection to which to add the file.
+    db : DatabaseHandler
+        Database handler object.
+    db_name : str
+        Name of the database to be created.
+    file_prefix : str
+        Path to location of all additional files to be uploaded.
+    """
+    par_dict = gen.collect_data_from_file(file_name=file)
+    logger.info(
+        f"Adding the following parameter to the DB: {par_dict['parameter']} "
+        f"version {par_dict['parameter_version']} "
+        f"(collection {collection} in database {db_name})"
+    )
+
+    db.add_new_parameter(
+        db_name=db_name,
+        par_dict=par_dict,
+        collection_name=collection,
+        file_prefix=file_prefix,
+    )
+
+
+def add_model_parameters_to_db(args_dict, db):
+    """
+    Read model parameters from a directory and upload them to the database.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db : DatabaseHandler
+        Database handler object.
+    """
+    input_path = Path(args_dict["input_path"])
+    logger.info(f"Reading model parameters from repository path {input_path}")
+    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
+    for element in array_elements:
+        collection = names.get_collection_name_from_array_element_name(element.name, False)
+        if collection == "Files":
+            logger.info("Files (tables) are uploaded with the corresponding model parameters")
+            continue
+        logger.info(f"Reading model parameters for {element.name} into collection {collection}")
+        files_to_insert = list(Path(element).rglob("*json"))
+        for file in files_to_insert:
+            add_values_from_json_to_db(
+                file=file,
+                collection=collection,
+                db=db,
+                db_name=args_dict["db_name"],
+                file_prefix=input_path / "Files",
+            )
+
+
+def add_production_tables_to_db(args_dict, db):
+    """
+    Read production tables from a directory and upload them to the database.
+
+    One dictionary per collection is prepared for each model version, containing
+    tables of all array elements, sites, and configuration parameters.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db : DatabaseHandler
+        Database handler object.
+    """
+    input_path = Path(args_dict["input_path"])
+    logger.info(f"Reading production tables from repository path {input_path}")
+
+    for model in filter(Path.is_dir, input_path.iterdir()):
+        logger.info(f"Reading production tables for model version {model.name}")
+        model_dict = {}
+        for file in sorted(model.rglob("*json")):
+            _read_production_table(model_dict, file, model.name)
+
+        for collection, data in model_dict.items():
+            if not data["parameters"]:
+                logger.info(f"No production table for {collection} in model version {model.name}")
+                continue
+            logger.info(f"Adding production table for {collection} to the database")
+            db.add_production_table(
+                db_name=args_dict["db_name"],
+                production_table=data,
+            )
+
+
+def _read_production_table(model_dict, file, model_name):
+    """Read a single production table from file."""
+    array_element = file.stem
+    collection = names.get_collection_name_from_array_element_name(array_element, False)
+    model_dict.setdefault(
+        collection,
+        {
+            "collection": collection,
+            "model_version": model_name,
+            "parameters": {},
+            "design_model": {},
+        },
+    )
+    parameter_dict = gen.collect_data_from_file(file_name=file)
+    logger.info(f"Reading production table for {array_element} (collection {collection})")
+    try:
+        if array_element in ("configuration_corsika", "configuration_sim_telarray"):
+            model_dict[collection]["parameters"] = parameter_dict["parameters"]
+        else:
+            model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][
+                array_element
+            ]
+    except KeyError as exc:
+        logger.error(f"KeyError: {exc}")
+        raise
+    try:
+        model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][
+            array_element
+        ]
+    except KeyError:
+        pass

simtools/io_operations/hdf5_handler.py

@@ -41,40 +41,26 @@ def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data)
     meta_data: dict
         Dictionary with the histogram metadata.
     """
-    # Complement metadata
-    if x_label is not None:
-        meta_data["x_bin_edges"] = sanitize_name(x_label)
+    validate_histogram(hist, y_bin_edges)
+
+    meta_data["x_bin_edges"] = x_bin_edges
     meta_data["x_bin_edges_unit"] = (
         x_bin_edges.unit if isinstance(x_bin_edges, u.Quantity) else u.dimensionless_unscaled
     )
-
     if y_bin_edges is not None:
-        if y_label is not None:
-            meta_data["y_bin_edges"] = sanitize_name(y_label)
-            names = [
-                f"{meta_data['y_bin_edges'].split('__')[0]}_{i}"
-                for i in range(len(y_bin_edges[:-1]))
-            ]
-        else:
-            names = [
-                f"{meta_data['Title'].split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
-            ]
+        meta_data["y_bin_edges"] = y_bin_edges
         meta_data["y_bin_edges_unit"] = (
             y_bin_edges.unit if isinstance(y_bin_edges, u.Quantity) else u.dimensionless_unscaled
         )
 
-        table = Table(
-            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
-            names=names,
-            meta=meta_data,
-        )
-
-    else:
+    if hist.ndim == 1:
         if x_label is not None:
-            meta_data["x_bin_edges"] = sanitize_name(x_label)
-            names = meta_data["x_bin_edges"]
-        else:
+            names = sanitize_name(x_label)
+        try:
             names = meta_data["Title"]
+        except KeyError:
+            _logger.warning("Title not found in metadata.")
+
         table = Table(
             [
                 x_bin_edges[:-1],
@@ -83,9 +69,53 @@ def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data)
             names=(names, sanitize_name("Values")),
             meta=meta_data,
         )
+    else:
+        if y_label is not None:
+            names = [
+                f"{sanitize_name(y_label).split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+        try:
+            names = [
+                f"{(meta_data['Title']).split('__')[0]}_{sanitize_name(y_label)}_{i}"
+                for i in range(len(y_bin_edges[:-1]))
+            ]
+        except KeyError:
+            _logger.warning("Title not found in metadata.")
+            names = [
+                f"{sanitize_name(y_label).split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+
+        table = Table(
+            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
+            names=names,
+            meta=meta_data,
+        )
+
     return table
 
 
+def validate_histogram(hist, y_bin_edges):
+    """Validate histogram dimensions and y_bin_edges consistency.
+
+    Parameters
+    ----------
+    hist (np.ndarray): The histogram array, expected to be 1D or 2D.
+    y_bin_edges (array-like or None): Bin edges for the second dimension (if applicable).
+
+    Raises
+    ------
+    ValueError: If histogram dimensions are invalid or inconsistent with y_bin_edges.
+    """
+    if hist.ndim not in (1, 2):
+        raise ValueError("Histogram must be either 1D or 2D.")
+
+    if hist.ndim == 1 and y_bin_edges is not None:
+        raise ValueError("y_bin_edges should be None for 1D histograms.")
+
+    if hist.ndim == 2 and y_bin_edges is None:
+        raise ValueError("y_bin_edges should not be None for 2D histograms.")
+
+
 def read_hdf5(hdf5_file_name):
     """
     Read a hdf5 output file.

simtools/io_operations/legacy_data_handler.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python3
+"""Reading of legacy data files (expect that this will be obsolete in future)."""
+
+import logging
+
+from astropy.table import Table
+
+__all__ = [
+    "read_legacy_data_as_table",
+    "read_legacy_lst_single_pe",
+]
+
+logger = logging.getLogger(__name__)
+
+
+def read_legacy_data_as_table(file_path, file_type):
+    """
+    Read legacy data file.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the legacy data file.
+    file_type: str
+        Type of legacy data file.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+
+    Raises
+    ------
+    ValueError
+        If unsupported legacy data file type.
+    """
+    logger.debug(f"Reading legacy data file of type {file_type} from {file_path}")
+
+    try:
+        return globals()[f"read_{file_type}"](file_path)
+    except KeyError as exc:
+        raise ValueError(f"Unsupported legacy data file type: {file_type}") from exc
+
+
+def read_legacy_lst_single_pe(file_path):
+    """
+    Read LST single pe file (in legacy data format).
+
+    File contains two columns: amplitude (in units of single p.e) and response.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the legacy data file.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+    """
+    return Table.read(file_path, format="ascii.csv", names=("amplitude", "response"))

simtools/job_execution/htcondor_script_generator.py

@@ -0,0 +1,133 @@
+"""HT Condor script generator for simulation production."""
+
+import logging
+from pathlib import Path
+
+import astropy.units as u
+
+_logger = logging.getLogger(__name__)
+
+
+def generate_submission_script(args_dict):
+    """
+    Generate the HT Condor submission script.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Arguments dictionary.
+    """
+    _logger.info("Generating HT Condor submission scripts ")
+
+    work_dir = Path(args_dict["output_path"])
+    log_dir = work_dir / "logs"
+    work_dir.mkdir(parents=True, exist_ok=True)
+    log_dir.mkdir(parents=True, exist_ok=True)
+    submit_file_name = "simulate_prod.submit"
+
+    with open(work_dir / f"{submit_file_name}.condor", "w", encoding="utf-8") as submit_file_handle:
+        submit_file_handle.write(
+            _get_submit_file(
+                f"{submit_file_name}.sh",
+                args_dict["apptainer_image"],
+                args_dict["priority"],
+                +args_dict["number_of_runs"],
+            )
+        )
+
+    with open(work_dir / f"{submit_file_name}.sh", "w", encoding="utf-8") as submit_script_handle:
+        submit_script_handle.write(_get_submit_script(args_dict))
+
+    Path(work_dir / f"{submit_file_name}.sh").chmod(0o755)
+
+
+def _get_submit_file(executable, apptainer_image, priority, n_jobs):
+    """
+    Return HT Condor submit file.
+
+    Database access variables are passed through the environment file.
+
+    Parameters
+    ----------
+    executable: str
+        Name of the executable script.
+    apptainer_image: str
+        Path to the Apptainer image.
+    priority: int
+        Priority of the job.
+    n_jobs: int
+        Number of jobs to queue.
+
+    Returns
+    -------
+    str
+        HT Condor submit file content.
+    """
+    return f"""universe = container
+container_image = {apptainer_image}
+transfer_container = false
+
+executable = {executable}
+error      = logs/err.$(cluster)_$(process)
+output     = logs/out.$(cluster)_$(process)
+log        = logs/log.$(cluster)_$(process)
+
+priority = {priority}
+arguments = "$(process) env.txt"
+
+queue {n_jobs}
+"""
+
+
+def _get_submit_script(args_dict):
+    """
+    Return HT Condor submit script.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Arguments dictionary.
+
+    Returns
+    -------
+    str
+        HT Condor submit script content.
+    """
+    azimuth_angle_string = f"{args_dict['azimuth_angle'].to(u.deg).value}"
+    zenith_angle_string = f"{args_dict['zenith_angle'].to(u.deg).value}"
+    energy_range = args_dict["energy_range"]
+    energy_range_string = (
+        f'"{energy_range[0].to(u.GeV).value} GeV {energy_range[1].to(u.GeV).value} GeV"'
+    )
+    core_scatter = args_dict["core_scatter"]
+    core_scatter_string = f'"{core_scatter[0]} {core_scatter[1].to(u.m).value} m"'
+
+    label = args_dict["label"] if args_dict["label"] else "simulate-prod"
+
+    return f"""#!/usr/bin/env bash
+
+# Process ID used to generate run number
+process_id="$1"
+# Load environment variables (for DB access)
+set -a; source "$2"
+
+simtools-simulate-prod \\
+    --simulation_software {args_dict["simulation_software"]} \\
+    --label {label} \\
+    --model_version {args_dict["model_version"]} \\
+    --site {args_dict["site"]} \\
+    --array_layout_name {args_dict["array_layout_name"]} \\
+    --primary {args_dict["primary"]} \\
+    --azimuth_angle {azimuth_angle_string} \\
+    --zenith_angle {zenith_angle_string} \\
+    --nshow {args_dict["nshow"]} \\
+    --energy_range {energy_range_string} \\
+    --core_scatter {core_scatter_string} \\
+    --run_number_start $((process_id + {args_dict["run_number_start"]})) \\
+    --number_of_runs 1 \\
+    --submit_engine \"local\" \\
+    --data_directory /tmp/simtools-data \\
+    --output_path /tmp/simtools-output \\
+    --log_level {args_dict["log_level"]} \\
+    --pack_for_grid_register simtools-output
+"""

simtools/job_execution/job_manager.py

@@ -1,7 +1,7 @@
 """Interface to workload managers like gridengine or HTCondor."""
 
 import logging
-import os
+import subprocess
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -65,11 +65,9 @@ class JobManager:
         ValueError
             if invalid submit engine.
         """
-        if value is None:
-            value = "local"
-        if value not in self.engines:
-            raise ValueError(f"Invalid submit command: {value}")
-        self._submit_engine = value
+        self._submit_engine = value or "local"
+        if self._submit_engine not in self.engines:
+            raise ValueError(f"Invalid submit command: {self._submit_engine}")
 
     def check_submission_system(self):
         """
@@ -77,14 +75,17 @@ class JobManager:
 
         Raises
         ------
-        MissingWorkloadManagerError
+        JobExecutionError
             if workflow manager is not found.
         """
-        if self.submit_engine is None or self.submit_engine == "local":
-            return
-
-        if gen.program_is_executable(self.engines[self.submit_engine]):
-            return
+        try:
+            if self.submit_engine in (None, "local") or gen.program_is_executable(
+                self.engines[self.submit_engine]
+            ):
+                return
+        except KeyError:
+            pass
+        raise JobExecutionError(f"Submit engine {self.submit_engine} not found")
 
     def submit(self, run_script=None, run_out_file=None, log_file=None):
         """
@@ -109,12 +110,14 @@ class JobManager:
         self._logger.info(f"Job error stream {self.run_out_file + '.err'}")
         self._logger.info(f"Job log stream {self.run_out_file + '.job'}")
 
-        if self.submit_engine == "gridengine":
-            self._submit_gridengine()
-        elif self.submit_engine == "htcondor":
-            self._submit_htcondor()
-        elif self.submit_engine == "local":
-            self._submit_local(log_file)
+        submit_result = 0
+        if self.submit_engine == "local":
+            submit_result = self._submit_local(log_file)
+        else:
+            submit_result = getattr(self, f"_submit_{self.submit_engine}")()
+
+        if submit_result != 0:
+            raise JobExecutionError(f"Job submission failed with return code {submit_result}")
 
     def _submit_local(self, log_file):
         """
@@ -125,50 +128,72 @@ class JobManager:
         log_file: str or Path
             The log file of the actual simulator (CORSIKA or sim_telarray).
             Provided in order to print the log excerpt in case of run time error.
+
+        Returns
+        -------
+        int
+            Return code of the executed script
         """
         self._logger.info("Running script locally")
 
-        shell_command = f"{self.run_script} > {self.run_out_file}.out 2> {self.run_out_file}.err"
-
-        if not self.test:
-            sys_output = os.system(shell_command)
-            if sys_output != 0:
-                msg = gen.get_log_excerpt(f"{self.run_out_file}.err")
-                self._logger.error(msg)
-                if log_file.exists() and gen.get_file_age(log_file) < 5:
-                    msg = gen.get_log_excerpt(log_file)
-                    self._logger.error(msg)
-                raise JobExecutionError("See excerpt from log file above\n")
-        else:
+        if self.test:
             self._logger.info("Testing (local)")
+            return 0
+
+        result = None
+        try:
+            with (
+                open(f"{self.run_out_file}.out", "w", encoding="utf-8") as stdout,
+                open(f"{self.run_out_file}.err", "w", encoding="utf-8") as stderr,
+            ):
+                result = subprocess.run(
+                    f"{self.run_script}",
+                    shell=True,
+                    check=True,
+                    text=True,
+                    stdout=stdout,
+                    stderr=stderr,
+                )
+        except subprocess.CalledProcessError as exc:
+            self._logger.error(gen.get_log_excerpt(f"{self.run_out_file}.err"))
+            if log_file.exists() and gen.get_file_age(log_file) < 5:
+                self._logger.error(gen.get_log_excerpt(log_file))
+            raise JobExecutionError("See excerpt from log file above\n") from exc
+
+        return result.returncode if result else 0
 
     def _submit_htcondor(self):
         """Submit a job described by a shell script to HTcondor."""
         _condor_file = self.run_script + ".condor"
+        lines = [
+            f"Executable = {self.run_script}",
+            f"Output = {self.run_out_file}.out",
+            f"Error = {self.run_out_file}.err",
+            f"Log = {self.run_out_file}.job",
+        ]
+        if self.submit_options:
+            lines.extend(option.lstrip() for option in self.submit_options.split(","))
+        lines.append("queue 1")
         try:
             with open(_condor_file, "w", encoding="utf-8") as file:
-                file.write(f"Executable = {self.run_script}\n")
-                file.write(f"Output = {self.run_out_file + '.out'}\n")
-                file.write(f"Error = {self.run_out_file + '.err'}\n")
-                file.write(f"Log = {self.run_out_file + '.job'}\n")
-                if self.submit_options:
-                    submit_option_list = self.submit_options.split(",")
-                    for option in submit_option_list:
-                        file.write(option.lstrip() + "\n")
-                file.write("queue 1\n")
+                file.write("\n".join(lines) + "\n")
         except FileNotFoundError as exc:
             self._logger.error(f"Failed creating condor submission file {_condor_file}")
             raise JobExecutionError from exc
 
-        self._execute(self.submit_engine, self.engines[self.submit_engine] + " " + _condor_file)
+        return self._execute(self.submit_engine, [self.engines[self.submit_engine], _condor_file])
 
     def _submit_gridengine(self):
         """Submit a job described by a shell script to gridengine."""
-        this_sub_cmd = self.engines[self.submit_engine]
-        this_sub_cmd = this_sub_cmd + " -o " + self.run_out_file + ".out"
-        this_sub_cmd = this_sub_cmd + " -e " + self.run_out_file + ".err"
-
-        self._execute(self.submit_engine, this_sub_cmd + " " + self.run_script)
+        this_sub_cmd = [
+            self.engines[self.submit_engine],
+            "-o",
+            self.run_out_file + ".out",
+            "-e",
+            self.run_out_file + ".err",
+            self.run_script,
+        ]
+        return self._execute(self.submit_engine, this_sub_cmd)
 
     def _execute(self, engine, shell_command):
         """
@@ -178,13 +203,15 @@ class JobManager:
         ----------
         engine : str
             Engine to use.
-        shell_command : str
-            Shell command to execute.
+        shell_command : list
+            List of shell command plus arguments.
         """
         self._logger.info(f"Submitting script to {engine}")
         self._logger.debug(shell_command)
+        result = None
         if not self.test:
-            os.system(shell_command)
+            result = subprocess.run(shell_command, shell=True, check=True)
         else:
-            self._logger.info(f"Testing ({engine})")
-            self._logger.info(shell_command)
+            self._logger.info(f"Testing ({engine}: {shell_command})")
+
+        return result.returncode if result else 0