airflow-unicore-integration 0.2.4__tar.gz → 0.3.0__tar.gz

{airflow_unicore_integration-0.2.4/src/airflow_unicore_integration.egg-info → airflow_unicore_integration-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.2.4
+Version: 0.3.0
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause

{airflow_unicore_integration-0.2.4 → airflow_unicore_integration-0.3.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "airflow-unicore-integration"
-version = "0.2.4"
+version = "0.3.0"
 authors = [
   { name="Christian Böttcher", email="c.boettcher@fz-juelich.de" },
 ]

{airflow_unicore_integration-0.2.4 → airflow_unicore_integration-0.3.0}/src/airflow_unicore_integration/executors/unicore_executor.py

@@ -20,8 +20,10 @@ from airflow.models.taskinstance import TaskInstance
 from airflow.models.taskinstancekey import TaskInstanceKey
 from airflow.utils.state import TaskInstanceState
 from pyunicore import client
+from pyunicore.credentials import AuthenticationFailedException
 from pyunicore.credentials import Credential
 from pyunicore.credentials import create_credential
+from requests.exceptions import RequestException
 
 from ..util.job import JobDescriptionGenerator
 from ..util.job import NaiveJobDescriptionGenerator
@@ -152,7 +154,24 @@ class UnicoreExecutor(BaseExecutor):
             raise TypeError(f"Don't know how to queue workload of type {type(workload).__name__}")
 
         # submit job to unicore and add to active_jobs dict for task state management
-        job = self._submit_job(workload)
+        try:
+            job = self._submit_job(workload)
+        except AuthenticationFailedException as auth_exception:
+            self.fail(workload.ti.key)
+            self.log.error(
+                "Invalid credentials for UNICORE. Failing all UNICORE tasks until this has been resolved."
+            )
+            self.log.error(auth_exception)
+            return
+        except RequestException as re:
+            # Any kind of requests error, from DNS issues, to timeout Issues, to invalid UNICORE URL issues
+            self.fail(workload.ti.key)
+            self.log.error(
+                "Some Error appeared during the connection attempt to UNICORE. Failing this task Instance."
+            )
+            self.log.error(re)
+            return
+
         self.active_jobs[workload.ti.key] = job
 
     def end(self, heartbeat_interval=10) -> None:

{airflow_unicore_integration-0.2.4 → airflow_unicore_integration-0.3.0}/src/airflow_unicore_integration/operators/container.py

@@ -1,6 +1,3 @@
-from typing import Dict
-from typing import List
-
 from airflow_unicore_integration.operators.unicore_operators import (
     UnicoreGenericOperator,
 )
@@ -10,13 +7,13 @@ class UnicoreContainerOperator(UnicoreGenericOperator):
     def __init__(
         self, name: str, docker_image_url: str, command: str, options: str | None = None, **kwargs
     ):
-        params: Dict[str, str | List[str]] = {"COMMAND": command, "IMAGE_URL": docker_image_url}
-        if options is not None:
-            params["OPTIONS"] = options
         super().__init__(
             name=name,
             application_name="CONTAINER",
             application_version="1.0",
-            parameters=params,
             **kwargs,
         )
+        self.params["COMMAND"] = command
+        self.params["IMAGE_URL"] = docker_image_url
+        if options is not None:
+            self.params["OPTIONS"] = options

{airflow_unicore_integration-0.2.4 → airflow_unicore_integration-0.3.0}/src/airflow_unicore_integration/operators/unicore_operators.py

@@ -45,7 +45,7 @@ class UnicoreGenericOperator(BaseOperator):
         user_post_command: str | None = None,
         run_user_post_command_on_login_node: bool | None = None,
         user_post_command_ignore_non_zero_exit_code: bool | None = None,
-        resources: Dict[str, str] | None = None,
+        uc_resources: Dict[str, str] | None = None,
         project: str | None = None,
         imports: List[Dict[str, str | Sequence[str]]] | None = None,
         exports: List[Dict[str, str | List[str]]] | None = None,
@@ -97,7 +97,7 @@ class UnicoreGenericOperator(BaseOperator):
         self.user_post_command_ignore_non_zero_exit_code = (
             user_post_command_ignore_non_zero_exit_code
         )
-        self.resources = resources
+        self.resources = uc_resources
         self.project = project
         self.imports = imports
         self.exports = exports

{airflow_unicore_integration-0.2.4 → airflow_unicore_integration-0.3.0}/src/airflow_unicore_integration/util/job.py

@@ -27,6 +27,7 @@ class JobDescriptionGenerator:
     EXECUTOR_CONFIG_POST_COMMANDS = "postcommands"  # gets added to the unicore job descirption
     EXECUTOR_CONFIG_JOB_TYPE = "job_type"
     EXECUTOR_CONFIG_LOGIN_NODE = "login_node"
+    EXECUTOR_CONFIG_JOB_DESCRIPTION_PARAMS = "custom_job_description_additions"
     EXECUTOR_CONFIG_UNICORE_CONN_KEY = (
         "unicore_connection_id"  # alternative connection id for the Unicore connection to use
     )
@@ -63,6 +64,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         user_added_post_commands: list[str] = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_POST_COMMANDS, [])  # type: ignore
         user_defined_job_type: str = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_JOB_TYPE, None)  # type: ignore
         user_defined_login_node: str = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_LOGIN_NODE, None)  # type: ignore
+        user_added_job_description: Dict[str, Any] = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_JOB_DESCRIPTION_PARAMS, {})  # type: ignore
         # get local dag path from cmd and fix dag path in arguments
         dag_rel_path = str(workload.dag_rel_path)
         if dag_rel_path.startswith("DAG_FOLDER"):
@@ -194,4 +196,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         if user_added_resources:
             job_descr_dict["Resources"] = user_added_resources
 
+        # overwrite with values from user added field
+        job_descr_dict.update(user_added_job_description)
+
         return job_descr_dict

{airflow_unicore_integration-0.2.4 → airflow_unicore_integration-0.3.0/src/airflow_unicore_integration.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.2.4
+Version: 0.3.0
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause