CTADIRAC 2.2.83__tar.gz → 2.2.85__tar.gz

{ctadirac-2.2.83/src/CTADIRAC.egg-info → ctadirac-2.2.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 2.2.83
+Version: 2.2.85
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License
@@ -15,7 +15,7 @@ Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENCE
-Requires-Dist: DIRAC==8.0.71
+Requires-Dist: DIRAC==8.0.78
 Requires-Dist: ruamel.yaml
 Requires-Dist: shellescape
 Requires-Dist: matplotlib
@@ -48,11 +48,6 @@ See the dedicated [client installation documentation](docs/install_client.md).
 
 See the dedicated [server installation documentation](docs/install_CTADIRAC.md).
 
-## Deploying on Kubernetes
-[CTADIRAC Helm charts](https://gitlab.cta-observatory.org/cta-computing/dpps/workload/CTADIRAC-charts) (in development).
-
-[CTADIRAC fleet deployment](https://gitlab.cta-observatory.org/cta-computing/dpps/workload/ctadirac-deployment) on a Kubernetes cluster.
-
 # Registry
 
 * Get `CTADIRAC` on `PyPi`:

{ctadirac-2.2.83 → ctadirac-2.2.85}/README.md

@@ -17,11 +17,6 @@ See the dedicated [client installation documentation](docs/install_client.md).
 
 See the dedicated [server installation documentation](docs/install_CTADIRAC.md).
 
-## Deploying on Kubernetes
-[CTADIRAC Helm charts](https://gitlab.cta-observatory.org/cta-computing/dpps/workload/CTADIRAC-charts) (in development).
-
-[CTADIRAC fleet deployment](https://gitlab.cta-observatory.org/cta-computing/dpps/workload/ctadirac-deployment) on a Kubernetes cluster.
-
 # Registry
 
 * Get `CTADIRAC` on `PyPi`:

{ctadirac-2.2.83 → ctadirac-2.2.85}/docker/dirac-client/Dockerfile

@@ -34,6 +34,8 @@ WORKDIR ${DIRAC_DIR}
 # Install DIRAC client
 USER dirac
 
+RUN git config --global --add safe.directory /CTADIRAC/
+
 RUN curl -LO https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh \
     && bash DIRACOS-Linux-x86_64.sh -p ${DIRAC_DIR}/diracos \
     && rm -f DIRACOS-Linux-x86_64.sh \

{ctadirac-2.2.83 → ctadirac-2.2.85}/docs/install_client.md

@@ -33,6 +33,21 @@ source diracos/diracosrc
 pip install CTADIRAC
 ```
 
+:warning: DIRAC is currently in transition to [DiracX](https://github.com/DIRACGrid/diracx).
+The DIRAC client must follow a specific CTADIRAC versioning depending on the server you are using:
+  - for the [certification instance](https://majestix-vm3.zeuthen.desy.de/DIRAC), which is using DiracX and DIRAC v9.0
+    - the client must use `CTADIRAC >= 3.0.2`
+    - use the following configure command:
+      ```
+      dirac-configure -F -C dips://majestix-vm1.zeuthen.desy.de:9135/Configuration/Server --DiracxUrl=https://diracx-cert.cta-test.zeuthen.desy.de -ddd
+      ```
+  - for the [CTADIRAC legacy instance](https://ctadirac-01.cscs.cta-observatory.org/DIRAC/), which is still using DIRAC v8
+    - the client must use `CTADIRAC >= 2.2.83 < 3.0.0`.
+    - use the following configure command:
+      ```
+      dirac-configure -F -C dips://ctadirac-01.cscs.cta-observatory.org:9135/Configuration/Server -S CTA -ddd
+      ```
+
 ## Configuration
 
 Only the first time:

{ctadirac-2.2.83 → ctadirac-2.2.85}/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
 ]
 license = { text = "BSD 3-Clause License" }
 dependencies = [
-    "DIRAC==8.0.71",
+    "DIRAC==8.0.78",
     "ruamel.yaml",
     "shellescape",
     "matplotlib",

{ctadirac-2.2.83 → ctadirac-2.2.85}/sonar-project.properties

@@ -1,3 +1,5 @@
+sonar.projectKey=cta-computing_dpps_workload_CTADIRAC_ce362036-3b75-4dac-9466-56ac6c965545
+sonar.host.url=https://sonar-ctao.zeuthen.desy.de
 sonar.qualitygate.wait=true
 sonar.language=python
 sonar.python.version=3
@@ -5,5 +7,3 @@ sonar.python.coverage.reportPaths=coverage.xml
 sonar.coverage.exclusions=tests/**, src/CTADIRAC/Core/scripts/**, src/CTADIRAC/ProductionSystem/scripts/**, src/CTADIRAC/DataManagementSystem/**, src/CTADIRAC/version.py, src/CTADIRAC/_dev/**
 sonar.cpd.exclusions=tests/**, src/CTADIRAC/Core/scripts/**, src/CTADIRAC/ProductionSystem/scripts/**, src/CTADIRAC/DataManagementSystem/**, src/CTADIRAC/version.py, src/CTADIRAC/_dev/**
 sonar.exclusions=dpps-**
-sonar.projectKey=cta-computing_dpps_CTADIRAC_AYcypMiTBJOBl6qHRhPT
-sonar.host.url=https://sonar-cta-dpps.zeuthen.desy.de

{ctadirac-2.2.83 → ctadirac-2.2.85}/src/CTADIRAC/Interfaces/API/CTAJob.py

@@ -52,6 +52,7 @@ class MetadataDict(OrderedDict):
         "type",
         "version",
         "systematics",
+        "view_cone",
     ]
 
     def __setitem__(self, key, value) -> None:
@@ -119,6 +120,8 @@ class CTAJob(Job):
         self.output_metadata["outputType"] = self.output_type
         self.output_metadata["configuration_id"] = self.configuration_id
         self.output_metadata["MCCampaign"] = self.MCCampaign
+        if metadata.get("view_cone"):
+            self.output_metadata["view_cone"] = metadata["view_cone"]
         if metadata.get("systematic_uncertainty_to_test"):
             metadata_systematics = self.systematic_uncertainty_to_test.rstrip(
                 "/"

{ctadirac-2.2.83 → ctadirac-2.2.85}/src/CTADIRAC/Interfaces/API/CWLJob.py

@@ -7,7 +7,6 @@ from cwl_utils.parser import save
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Interfaces.API.Job import Job
 from ruamel.yaml import YAML
-
 from CTADIRAC.Interfaces.Utilities.CWLTranslator import CWLTranslator
 
 
@@ -32,11 +31,10 @@ class CWLJob(Job, CWLTranslator):
     ) -> None:
         Job.__init__(self)
         CWLTranslator.__init__(self, cwl_workflow, cwl_inputs)
-
+        self.logfile_name = "cwl_job"
         self.cvmfs_base_path = cvmfs_base_path
         self.apptainer_options = apptainer_options if apptainer_options else []
         self._output_se = output_se
-
         self.translate(self.cvmfs_base_path, self.apptainer_options)
 
     def submit(self):
@@ -48,14 +46,12 @@ class CWLJob(Job, CWLTranslator):
         """
         dirac = Dirac()
         yaml = YAML()
-
         # Create the modified Dirac compliant CWL workflow and inputs files to submit
         with tempfile.NamedTemporaryFile(
             suffix=f"_{self.cwl_workflow_path.name}"
         ) as temp_workflow:
             yaml.dump(save(self.transformed_cwl), temp_workflow)
             temp_workflow.flush()
-
             with tempfile.NamedTemporaryFile(
                 suffix=f"_{self.cwl_inputs_path.name}"
             ) as temp_inputs:
@@ -76,9 +72,10 @@ class CWLJob(Job, CWLTranslator):
                 arguments_str = (
                     f"{Path(temp_workflow.name).name} {Path(temp_inputs.name).name}"
                 )
-
                 self.setExecutable(
-                    "cwltool", arguments=arguments_str, logFile=f"{self.name}.log"
+                    "cwltool",
+                    arguments=arguments_str,
+                    logFile=f"{self.logfile_name}.log",
                 )
                 res = dirac.submitJob(self)
         return res

{ctadirac-2.2.83 → ctadirac-2.2.85}/src/CTADIRAC/Interfaces/API/MCPipeJob.py

@@ -56,23 +56,22 @@ class MCPipeJob(CTAJob):
             self.degraded_mirror = ""
 
     def set_div_ang(self, div_ang=None) -> None:
-        div_default = [
-            "0.0022",
-            "0.0043",
-            "0.008",
-            "0.01135",
-            "0.01453",
-        ]  # Default divergent angles
+        allowed_div_sets = [
+            ["0.0022", "0.0043", "0.008", "0.01135", "0.01453"],
+            ["0.0121", "0.046"],
+        ]
         if div_ang is not None:
-            if str(div_ang).replace(", ", ",").split(sep=",") != div_default:
-                div_ang_str: str = ", ".join(str(x) for x in div_default)
-                DIRAC.gLogger.error(
-                    f"Unknown div_ang option: {div_ang}. Option for simulation step is: {div_ang_str}"
-                )
-                DIRAC.exit(-1)
-            else:
-                self.div_ang = div_default
-                self.output_file_metadata["div_ang"] = div_default
+            div_ang_list = [x.strip() for x in str(div_ang).split(",")]
+            for allowed in allowed_div_sets:
+                if div_ang_list == allowed:
+                    self.div_ang = allowed
+                    self.output_file_metadata["div_ang"] = allowed
+                    return
+            allowed_str = " or ".join([", ".join(s) for s in allowed_div_sets])
+            DIRAC.gLogger.error(
+                f"Unknown div_ang option: {div_ang}. Options for simulation step are: {allowed_str}"
+            )
+            DIRAC.exit(-1)
 
     def set_magic(self, with_magic=False) -> None:
         """Set to simulate with MAGIC

{ctadirac-2.2.83 → ctadirac-2.2.85}/src/CTADIRAC/Interfaces/API/SimPipeJob.py

@@ -31,8 +31,11 @@ class SimPipeJob(CTAJob):
         self._azimuth_angle = None
         self.software_category = "simulations"
         self.particle = None
+        self.view_cone = None
         self.output_directory = None
         self.simpipe_config_options = None
+        self.save_reduced_event_lists = False
+        self.output_reduced_event_lists_type = "reduced_event_lists"
 
     def set_simpipe_config(self, value) -> None:
         config = self._load_config_file(value)
@@ -62,6 +65,12 @@ class SimPipeJob(CTAJob):
                 self, "output_directory", value
             ),
             "primary": lambda value: setattr(self, "particle", value.lower()),
+            "save_reduced_event_lists": lambda value: setattr(
+                self, "save_reduced_event_lists", bool(value)
+            ),
+            "view_cone": lambda value: setattr(
+                self, "view_cone", str(value).replace(" ", "_")
+            ),
         }
 
         # Add generic handlers for attributes that map directly
@@ -181,47 +190,72 @@ class SimPipeJob(CTAJob):
         step["Value"]["name"] = "Step_Env"
         step["Value"]["descr_short"] = "Dump environment"
 
-    def upload_and_register_data_file(
+    def upload_and_register_file(
         self,
         meta_data_json,
         file_meta_data_json,
-        data_output_pattern,
+        file_pattern,
         log_str,
+        data_type: str = "data",
     ) -> None:
-        output_data_type = self.output_data_type
-        step = self.setExecutable(
-            "cta-prod-managedata",
-            arguments=(
-                f"'{meta_data_json}' '{file_meta_data_json}' {self.base_path} "
-                f"'{data_output_pattern}' {self.package} "
-                f"{self.program_category} '{self.catalogs}' "
-                f"{output_data_type}"
-            ),
-            logFile=f"DataManagement_{log_str}_Log.txt",
-        )
+        """
+        Upload and register files to SE and register them in DFC.
+
+        Args:
+            meta_data_json: JSON string containing metadata
+            file_meta_data_json: JSON string containing file metadata
+            file_pattern: Pattern for files to upload
+            log_str: String identifier for logging
+            data_type: Type of data ('data', 'log', 'reduced_event_lists')
+        """
+        # Define configuration for different data types
+        type_config = {
+            "data": {
+                "output_type_attr": "output_data_type",
+                "management_type": "DataManagement",
+                "step_name": "Step_DataManagement",
+                "description": "Save data files to SE and register them in DFC",
+            },
+            "log": {
+                "output_type_attr": "output_log_type",
+                "management_type": "LogManagement",
+                "step_name": "Step_LogManagement",
+                "description": "Save log files to SE and register them in DFC",
+            },
+            "reduced_event_lists": {
+                "output_type_attr": "output_reduced_event_lists_type",
+                "management_type": "ReducedEventListsManagement",
+                "step_name": "Step_ReducedEventListsManagement",
+                "description": (
+                    "Save reduced event lists to SE and register them in DFC"
+                ),
+            },
+        }
 
-        step["Value"]["name"] = "Step_DataManagement"
-        step["Value"]["descr_short"] = "Save data files to SE and register them in DFC"
+        if data_type not in type_config:
+            raise ValueError(
+                f"Unknown data type: {data_type}. "
+                f"Supported types: {list(type_config.keys())}"
+            )
+
+        config = type_config[data_type]
+        output_data_type = getattr(self, config["output_type_attr"])
+        management_type = config["management_type"]
+        step_name = config["step_name"]
+        description = config["description"]
 
-    def upload_and_register_log(
-        self,
-        meta_data_json,
-        file_meta_data_json,
-        log_file_pattern,
-        log_str,
-    ) -> None:
-        output_data_type = self.output_log_type
         step = self.setExecutable(
             "cta-prod-managedata",
             arguments=(
                 f"'{meta_data_json}' '{file_meta_data_json}' {self.base_path} "
-                f"'{log_file_pattern}' {self.package} {self.program_category} "
+                f"'{file_pattern}' {self.package} {self.program_category} "
                 f"'{self.catalogs}' {output_data_type}"
             ),
-            logFile=f"LogManagement_{log_str}_Log.txt",
+            logFile=f"{management_type}_{log_str}_Log.txt",
         )
-        step["Value"]["name"] = "Step_LogManagement"
-        step["Value"]["descr_short"] = "Save log to SE and register them in DFC"
+
+        step["Value"]["name"] = step_name
+        step["Value"]["descr_short"] = description
 
     def set_metadata_and_register_data(self, propagate_run_number=True) -> None:
         meta_data_json: str = json.dumps(self.output_metadata)
@@ -242,13 +276,30 @@ class SimPipeJob(CTAJob):
             log_str = f"{model_version}"
 
             # Upload and register data files for the current model_version
-            self.upload_and_register_data_file(
+            self.upload_and_register_file(
                 meta_data_json,
                 file_meta_data_json,
                 data_output_pattern,
                 log_str,
+                data_type="data",
             )
 
+            # Upload reduced event lists files if available
+            if self.save_reduced_event_lists:
+                reduced_data_output_pattern = (
+                    f"{self.output_directory}/*_{model_version}_*"
+                    ".reduced_event_data.hdf5"
+                )
+
+                # Upload and register reduced event lists files
+                self.upload_and_register_file(
+                    meta_data_json,
+                    file_meta_data_json,
+                    reduced_data_output_pattern,
+                    f"{log_str}_reduced_event_lists",
+                    data_type="reduced_event_lists",
+                )
+
             # Use model_version in log file patterns
             log_file_pattern = (
                 f"{self.output_directory}/*_{model_version}_*.log_hist.tar.gz"
@@ -259,11 +310,12 @@ class SimPipeJob(CTAJob):
                 )
 
             # Upload and register log files for the current model_version
-            self.upload_and_register_log(
+            self.upload_and_register_file(
                 meta_data_json,
                 file_meta_data_json,
                 log_file_pattern,
                 log_str,
+                data_type="log",
             )
 
     def run_dedicated_software(self) -> None: