gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/configuration/configurator.py

@@ -1,13 +1,15 @@
+import argparse
 import logging
 import os
 import sys
 import uuid
 
 import astropy.units as u
-import yaml
+from dotenv import load_dotenv
 
 import simtools.configuration.commandline_parser as argparser
-from simtools import io_handler
+from simtools.io_operations import io_handler
+from simtools.utils import general as gen
 
 __all__ = [
     "Configurator",
@@ -30,7 +32,7 @@ class Configurator:
     - configuration dict when calling the class
     - environmental variables
 
-    Assigns unique ACIVITY_ID to this configuration (uuid).
+    Assigns unique ACTIVITY_ID to this configuration (uuid).
 
     Configuration parameter names are converted always to lower case.
 
@@ -60,7 +62,11 @@ class Configurator:
         self.label = label
         self.config = {}
         self.parser = argparser.CommandLineParser(
-            prog=self.label, usage=usage, description=description, epilog=epilog
+            prog=self.label,
+            usage=usage,
+            description=description,
+            epilog=epilog,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         )
 
     def default_config(self, arg_list=None, add_db_config=False):
@@ -91,9 +97,11 @@ class Configurator:
 
     def initialize(
         self,
+        require_command_line=True,
         paths=True,
         output=False,
         telescope_model=False,
+        site_model=False,
         db_config=False,
         job_submission=False,
     ):
@@ -110,12 +118,16 @@ class Configurator:
 
         Parameters
         ----------
+        require_command_line: bool
+            Require at least one command line argument.
         paths: bool
             Add path configuration to list of args.
         output: bool
             Add output file configuration to list of args.
         telescope_model: bool
             Add telescope model configuration to list of args.
+        site_model: bool
+            Add site model configuration to list of args (not required if telescope_model is set)
         db_config: bool
             Add database configuration parameters to list of args.
         job_submission: bool
@@ -139,15 +151,13 @@ class Configurator:
             paths=paths,
             output=output,
             telescope_model=telescope_model,
+            site_model=site_model,
             db_config=db_config,
             job_submission=job_submission,
         )
 
-        self._fill_from_command_line()
-        try:
-            self._fill_from_config_file(self.config["config"])
-        except KeyError:
-            pass
+        self._fill_from_command_line(require_command_line=require_command_line)
+        self._fill_from_config_file(self.config.get("config"))
         self._fill_from_config_dict(self.config_class_init)
         self._fill_from_environmental_variables()
 
@@ -157,37 +167,76 @@ class Configurator:
             self.config["label"] = self.label
 
         self._initialize_io_handler()
-        self._initialize_output()
+        if output:
+            self._initialize_output()
         _db_dict = self._get_db_parameters()
 
         return self.config, _db_dict
 
-    def _fill_from_command_line(self, arg_list=None):
+    def _fill_from_command_line(self, arg_list=None, require_command_line=True):
         """
         Fill configuration parameters from command line arguments.
+        Triggers a print of the help if no command line arguments are given and
+        require_command_line is set.
+
+        Parameters
+        ----------
+        arg_list: list
+            List of arguments.
+        require_command_line: bool
+            Require at least one command line argument.
+
+        Raises
+        ------
+        InvalidConfigurationParameter
+              invalid configuration.
 
         """
 
         if arg_list is None:
             arg_list = sys.argv[1:]
 
+        if require_command_line and len(arg_list) == 0:
+            self._logger.debug("No command line arguments given, printing help.")
+            arg_list = ["--help"]
+
+        if "--config" in arg_list:
+            self._reset_required_arguments()
+
         self._fill_config(arg_list)
 
-    def _fill_from_config_dict(self, _input_dict):
+    def _reset_required_arguments(self):
+        """
+        Reset required parser arguments (i.e., arguments added with "required=True").
+        Includes also mutually exclusive groups.
+
+        Access protected attributes of parser (no public method available).
+
+        """
+
+        for group in self.parser._mutually_exclusive_groups:  # pylint: disable=protected-access
+            group.required = False
+        for action in self.parser._actions:  # pylint: disable=protected-access
+            action.required = False
+
+    def _fill_from_config_dict(self, input_dict, overwrite=False):
         """
         Fill configuration parameters from dictionary. Enforce that configuration parameter names\
          are lower case.
 
         Parameters
         ----------
-        _input_dict: dict
+        input_dict: dict
             dictionary with configuration parameters.
+        overwrite: bool
+            overwrite existing configuration parameters.
 
         """
         _tmp_config = {}
         try:
-            for key, value in _input_dict.items():
-                self._check_parameter_configuration_status(key, value)
+            for key, value in input_dict.items():
+                if not overwrite:
+                    self._check_parameter_configuration_status(key, value)
                 _tmp_config[key.lower()] = value
         except AttributeError:
             pass
@@ -242,17 +291,27 @@ class Configurator:
 
         try:
             self._logger.debug(f"Reading configuration from {config_file}")
-            with open(config_file, "r", encoding="utf-8") as stream:
-                _config_dict = yaml.safe_load(stream)
-            if "CTASIMPIPE" in _config_dict:
+            _config_dict = gen.collect_data_from_file_or_dict(
+                file_name=config_file, in_dict=None, allow_empty=True
+            )
+            # yaml parser adds \n in multiline strings, remove them
+            _config_dict = gen.remove_substring_recursively_from_dict(_config_dict, substring="\n")
+            if "CTA_SIMPIPE" in _config_dict:
                 try:
-                    self._fill_from_config_dict(_config_dict["CTASIMPIPE"]["CONFIGURATION"])
+                    self._fill_from_config_dict(
+                        input_dict=gen.change_dict_keys_case(
+                            _config_dict["CTA_SIMPIPE"]["CONFIGURATION"],
+                        ),
+                        overwrite=True,
+                    )
                 except KeyError:
-                    self._logger.info(f"No CTASIMPIPE:CONFIGURATION dict found in {config_file}.")
+                    self._logger.info(f"No CTA_SIMPIPE:CONFIGURATION dict found in {config_file}.")
             else:
-                self._fill_from_config_dict(_config_dict)
+                self._fill_from_config_dict(
+                    input_dict=gen.change_dict_keys_case(_config_dict), overwrite=True
+                )
         # TypeError is raised for config_file=None
-        except TypeError:
+        except (TypeError, AttributeError):
             pass
         except FileNotFoundError:
             self._logger.error(f"Configuration file not found: {config_file}")
@@ -260,18 +319,24 @@ class Configurator:
 
     def _fill_from_environmental_variables(self):
         """
-        Fill any unconfigured configuration parameters (i.e., parameter is None) \
-        from environmental variables.
+        Fill any configuration parameters which is not already configured (i.e., parameter is None)
+        from environmental variables or from file (default: ".env").
 
         """
 
         _env_dict = {}
         try:
-            for key, value in self.config.items():
-                if value is None:
-                    _env_dict[key] = os.environ.get(key.upper())
-        except AttributeError:
+            load_dotenv(self.config["env_file"])
+        except KeyError:
             pass
+        for key, value in self.config.items():
+            # environmental variables for simtools should always start with SIMTOOLS_
+            env_variable_to_read = f"SIMTOOLS_{key.upper()}"
+            if value is None:
+                env_value = os.environ.get(env_variable_to_read)
+                if env_value is not None:
+                    env_value = env_value.split("#")[0].strip().replace('"', "").replace("'", "")
+                _env_dict[key] = env_value
 
         self._fill_from_config_dict(_env_dict)
 
@@ -290,20 +355,20 @@ class Configurator:
 
     def _initialize_output(self):
         """
-        Initialize default output file names (in out output_file is not configured).
+        Initialize default output file names (in case output_file is not configured).
 
         """
         if self.config.get("output_file", None) is None:
             prefix = "TEST"
-            label = extention = ""
+            label = extension = ""
             if not self.config.get("test", False):
                 prefix = self.config["activity_id"]
                 if self.config.get("label", "") and len(self.config.get("label", "")) > 0:
                     label = f"-{self.config['label']}"
             if len(self.config.get("output_file_format", "")) > 0:
-                extention = f".{self.config['output_file_format']}"
+                extension = f".{self.config['output_file_format']}"
 
-            self.config["output_file"] = f"{prefix}{label}{extention}"
+            self.config["output_file"] = f"{prefix}{label}{extension}"
 
     @staticmethod
     def _arglist_from_config(input_var):
@@ -339,7 +404,7 @@ class Configurator:
                     _list_args += value
                 elif isinstance(value, u.Quantity):
                     _list_args.append("--" + key)
-                    _list_args.append(str(value.value))
+                    _list_args.append(str(value))
                 elif not isinstance(value, bool) and value is not None and len(str(value)) > 0:
                     _list_args.append("--" + key)
                     _list_args.append(str(value))

simtools/constants.py

@@ -0,0 +1,9 @@
+"""Project wide constants."""
+
+# Path to metadata jsonschema
+METADATA_JSON_SCHEMA = "schemas/metadata.metaschema.yml"
+# Path to data and modelparameter jsonschema
+DATA_JSON_SCHEMA = "schemas/data.metaschema.yml"
+
+# URL to the schema repository
+SCHEMA_URL = "https://raw.githubusercontent.com/gammasim/workflows/main/schemas/"

simtools/corsika/corsika_config.py

@@ -7,10 +7,10 @@ import numpy as np
 from astropy.io.misc import yaml
 
 import simtools.utils.general as gen
-from simtools import io_handler
-from simtools.layout.layout_array import LayoutArray
+from simtools.io_operations import io_handler
+from simtools.layout.array_layout import ArrayLayout
 from simtools.utils import names
-from simtools.utils.general import collect_data_from_yaml_or_dict
+from simtools.utils.general import collect_data_from_file_or_dict
 
 __all__ = [
     "CorsikaConfig",
@@ -90,17 +90,18 @@ class CorsikaConfig:
         self.label = label
         self.site = names.validate_site_name(site)
         self.primary = None
-        self._config_file_path = None
+        self.eslope = None
+        self.config_file_path = None
         self._output_generic_file_name = None
         self._simtel_source_path = simtel_source_path
 
         self.io_handler = io_handler.IOHandler()
 
-        # Grabbing layout name and building LayoutArray
-        self.layout_name = names.validate_layout_array_name(layout_name)
-        self.layout = LayoutArray.from_layout_array_name(
+        # Grabbing layout name and building ArrayLayout
+        self.layout_name = names.validate_array_layout_name(layout_name)
+        self.layout = ArrayLayout.from_array_layout_name(
             mongo_db_config=mongo_db_config,
-            layout_array_name=f"{self.site}-{self.layout_name}",
+            array_layout_name=f"{self.site}-{self.layout_name}",
             label=self.label,
         )
 
@@ -112,7 +113,7 @@ class CorsikaConfig:
 
         self._corsika_parameters = self.load_corsika_parameters_file(corsika_parameters_file)
 
-        corsika_config_data = collect_data_from_yaml_or_dict(
+        corsika_config_data = collect_data_from_file_or_dict(
             corsika_config_file, corsika_config_data
         )
         self.set_user_parameters(corsika_config_data)
@@ -246,6 +247,8 @@ class CorsikaConfig:
             value_args = [0 * par_info["unit"][0], value_args[0]]
         elif par_name == "PRMPAR":
             value_args = self._convert_primary_input_and_store_primary_name(value_args)
+        elif par_name == "ESLOPE":
+            self.eslope = value_args[0]
 
         if len(value_args) != par_info["len"]:
             msg = f"CORSIKA input entry with wrong len: {par_name}"
@@ -352,9 +355,9 @@ class CorsikaConfig:
             the output directly to sim_telarray or not.
         """
 
-        dir_type = "corsika_simtel" if use_multipipe else "corsika"
-        self._set_output_file_and_directory(dir_type)
-        self._logger.debug(f"Exporting CORSIKA input file to {self._config_file_path}")
+        sub_dir = "corsika_simtel" if use_multipipe else "corsika"
+        self._set_output_file_and_directory(sub_dir)
+        self._logger.debug(f"Exporting CORSIKA input file to {self.config_file_path}")
 
         def _get_text_single_line(pars):
             text = ""
@@ -374,7 +377,7 @@ class CorsikaConfig:
                     text += _get_text_single_line(new_pars)
             return text
 
-        with open(self._config_file_path, "w", encoding="utf-8") as file:
+        with open(self.config_file_path, "w", encoding="utf-8") as file:
             file.write("\n* [ RUN PARAMETERS ]\n")
             # Removing AZM entry first
             _user_pars_temp = copy.copy(self._user_parameters)
@@ -421,7 +424,7 @@ class CorsikaConfig:
 
             file.write("\n* [ OUTUPUT FILE ]\n")
             if use_multipipe:
-                run_cta_script = Path(self._config_file_path.parent).joinpath("run_cta_multipipe")
+                run_cta_script = Path(self.config_file_path.parent).joinpath("run_cta_multipipe")
                 file.write(f"TELFIL |{str(run_cta_script)}\n")
             else:
                 file.write(f"TELFIL {self._output_generic_file_name}\n")
@@ -474,20 +477,23 @@ class CorsikaConfig:
             )
         file_name = (
             f"{self.primary}_{self.site}_{self.layout_name}_"
-            f"za{int(self._user_parameters['THETAP'][0]):d}-"
-            f"{int(self._user_parameters['THETAP'][1]):d}deg"
+            f"za{int(self._user_parameters['THETAP'][0]):03}-"
+            f"azm{int(self._user_parameters['AZM'][0]):03}deg"
             f"{view_cone}{file_label}"
         )
 
         if file_type == "config_tmp":
             if run_number is not None:
-                return f"corsika_config_run{run_number}_{file_name}.txt"
+                return f"corsika_config_run{run_number:06}_{file_name}.txt"
             raise ValueError("Must provide a run number for a temporary CORSIKA config file")
         if file_type == "config":
             return f"corsika_config_{file_name}.input"
         if file_type == "output_generic":
+            # The XXXXXX will be replaced by the run number after the pfp step with sed
             file_name = (
-                "corsika_run${RUNNR}_${PRMNAME}_za${ZA}deg_azm${AZM}deg"
+                f"corsika_runXXXXXX_"
+                f"{self.primary}_za{int(self._user_parameters['THETAP'][0]):03}deg_"
+                f"azm{int(self._user_parameters['AZM'][0]):03}deg"
                 f"_{self.site}_{self.layout_name}{file_label}.zst"
             )
             return file_name
@@ -496,12 +502,12 @@ class CorsikaConfig:
 
         raise ValueError(f"The requested file type ({file_type}) is unknown")
 
-    def _set_output_file_and_directory(self, dir_type="corsika"):
+    def _set_output_file_and_directory(self, sub_dir="corsika"):
         config_file_name = self.get_file_name(file_type="config")
-        file_directory = self.io_handler.get_output_directory(label=self.label, dir_type=dir_type)
+        file_directory = self.io_handler.get_output_directory(label=self.label, sub_dir=sub_dir)
         self._logger.info(f"Creating directory {file_directory}, if needed.")
         file_directory.mkdir(parents=True, exist_ok=True)
-        self._config_file_path = file_directory.joinpath(config_file_name)
+        self.config_file_path = file_directory.joinpath(config_file_name)
 
         self._output_generic_file_name = self.get_file_name(file_type="output_generic")
 
@@ -532,4 +538,4 @@ class CorsikaConfig:
         """
         if not self._is_file_updated:
             self.export_input_file(use_multipipe)
-        return self._config_file_path
+        return self.config_file_path