airflow-unicore-integration 0.2.3__tar.gz → 0.2.5__tar.gz

{airflow_unicore_integration-0.2.3/src/airflow_unicore_integration.egg-info → airflow_unicore_integration-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.2.3
+Version: 0.2.5
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause

{airflow_unicore_integration-0.2.3 → airflow_unicore_integration-0.2.5}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "airflow-unicore-integration"
-version = "0.2.3"
+version = "0.2.5"
 authors = [
   { name="Christian Böttcher", email="c.boettcher@fz-juelich.de" },
 ]

{airflow_unicore_integration-0.2.3 → airflow_unicore_integration-0.2.5}/src/airflow_unicore_integration/executors/unicore_executor.py

@@ -20,8 +20,10 @@ from airflow.models.taskinstance import TaskInstance
 from airflow.models.taskinstancekey import TaskInstanceKey
 from airflow.utils.state import TaskInstanceState
 from pyunicore import client
+from pyunicore.credentials import AuthenticationFailedException
 from pyunicore.credentials import Credential
 from pyunicore.credentials import create_credential
+from requests.exceptions import RequestException
 
 from ..util.job import JobDescriptionGenerator
 from ..util.job import NaiveJobDescriptionGenerator
@@ -152,7 +154,24 @@ class UnicoreExecutor(BaseExecutor):
             raise TypeError(f"Don't know how to queue workload of type {type(workload).__name__}")
 
         # submit job to unicore and add to active_jobs dict for task state management
-        job = self._submit_job(workload)
+        try:
+            job = self._submit_job(workload)
+        except AuthenticationFailedException as auth_exception:
+            self.fail(workload.ti.key)
+            self.log.error(
+                "Invalid default credentials for UNICORE. Failing all UNICORE tasks until this has been resolved."
+            )
+            self.log.error(auth_exception)
+            return
+        except RequestException as re:
+            # Any kind of requests error, from DNS issues, to timeout Issues, to invalid UNICORE URL issues
+            self.fail(workload.ti.key)
+            self.log.error(
+                "Some Error appeared during the connection attempt to UNICORE. Failing this task Instance."
+            )
+            self.log.error(re)
+            return
+
         self.active_jobs[workload.ti.key] = job
 
     def end(self, heartbeat_interval=10) -> None:

{airflow_unicore_integration-0.2.3 → airflow_unicore_integration-0.2.5}/src/airflow_unicore_integration/util/job.py

@@ -27,6 +27,7 @@ class JobDescriptionGenerator:
     EXECUTOR_CONFIG_POST_COMMANDS = "postcommands"  # gets added to the unicore job descirption
     EXECUTOR_CONFIG_JOB_TYPE = "job_type"
     EXECUTOR_CONFIG_LOGIN_NODE = "login_node"
+    EXECUTOR_CONFIG_JOB_DESCRIPTION_PARAMS = "custom_job_description_additions"
     EXECUTOR_CONFIG_UNICORE_CONN_KEY = (
         "unicore_connection_id"  # alternative connection id for the Unicore connection to use
     )
@@ -63,6 +64,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         user_added_post_commands: list[str] = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_POST_COMMANDS, [])  # type: ignore
         user_defined_job_type: str = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_JOB_TYPE, None)  # type: ignore
         user_defined_login_node: str = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_LOGIN_NODE, None)  # type: ignore
+        user_added_job_description: Dict[str, Any] = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_JOB_DESCRIPTION_PARAMS, {})  # type: ignore
         # get local dag path from cmd and fix dag path in arguments
         dag_rel_path = str(workload.dag_rel_path)
         if dag_rel_path.startswith("DAG_FOLDER"):
@@ -79,7 +81,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         if user_defined_job_type:
             job_descr_dict["Job type"] = user_defined_job_type
         if user_defined_login_node:
-            job_descr_dict["Login Node"] = user_defined_login_node
+            job_descr_dict["Login node"] = user_defined_login_node
 
         # check which python virtualenv to use
         if user_defined_python_env:
@@ -194,4 +196,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         if user_added_resources:
             job_descr_dict["Resources"] = user_added_resources
 
+        # overwrite with values from user added field
+        job_descr_dict.update(user_added_job_description)
+
         return job_descr_dict

{airflow_unicore_integration-0.2.3 → airflow_unicore_integration-0.2.5/src/airflow_unicore_integration.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.2.3
+Version: 0.2.5
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause