airflow-unicore-integration 0.1.13__py3-none-any.whl → 0.2.1__py3-none-any.whl

airflow_unicore_integration/executors/unicore_executor.py

@@ -22,6 +22,7 @@ from airflow.utils.state import TaskInstanceState
 from pyunicore import client
 from pyunicore.credentials import create_credential
 
+from ..hooks.unicore_hooks import UnicoreHook
 from ..util.job import JobDescriptionGenerator
 from ..util.job import NaiveJobDescriptionGenerator
 
@@ -78,11 +79,18 @@ class UnicoreExecutor(BaseExecutor):
         overwrite_unicore_credential = executor_config.get(  # type: ignore
             UnicoreExecutor.EXECUTOR_CONFIG_UNICORE_CREDENTIAL_KEY, None
         )  # task can provide a different credential to use, else default from connection is used
+        overwrite_conn_id = executor_config.get(  # type: ignore
+            UnicoreExecutor.EXECUTOR_CONFIG_UNICORE_CONN_KEY, None
+        )
         token = conf.get("unicore.executor", "AUTH_TOKEN", fallback="")
         base_url = conf.get(
             "unicore.executor", "DEFAULT_URL", fallback="http://localhost:8080/DEMO-SITE/rest/core"
         )
         credential = create_credential(token=token)
+        if overwrite_conn_id is not None:
+            hook = UnicoreHook(overwrite_conn_id)
+            base_url = hook.get_base_url()
+            credential = hook.get_credential()
         if overwrite_unicore_site is not None:
             base_url = overwrite_unicore_site
         if overwrite_unicore_credential is not None:

airflow_unicore_integration/hooks/unicore_hooks.py

@@ -65,10 +65,23 @@ class UnicoreHook(BaseHook):
             credential = overwrite_credential
         if not base_url:
             raise TypeError()
-        logger.info(f"Using credential {credential} for SITE {base_url}.")
         conn = client.Client(credential, base_url)
         return conn
 
+    def get_credential(self) -> credentials.Credential:
+        """Return the credential part of the connection as a Credential object."""
+        params = self.get_connection(self.uc_conn_id)
+        credential = credentials.UsernamePassword(params.login, params.password)
+        auth_token = params.extra_dejson.get("auth_token", None)
+        if auth_token is not None:
+            credential = credentials.create_credential(token=auth_token)
+        return credential
+
+    def get_base_url(self) -> str:
+        """Return the base url of the connection."""
+        params = self.get_connection(self.uc_conn_id)
+        return params.host
+
     def test_connection(self) -> tuple[bool, str]:
         """Test the connection by sending an access_info request"""
         conn = self.get_conn()

{airflow_unicore_integration-0.1.13.dist-info → airflow_unicore_integration-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.1.13
+Version: 0.2.1
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause
@@ -24,7 +24,7 @@ Unicore Airflow Integration
 
 |Generic badge|
 
-.. |Generic badge| image:: https://github.com/UNICORE-EU/airflow-unicore-integration/actions/workflows/publish-to-pypi.yml/badge.svg 
+.. |Generic badge| image:: https://github.com/UNICORE-EU/airflow-unicore-integration/actions/workflows/publish-to-pypi.yml/badge.svg
    :target: https://github.com/UNICORE-EU/airflow-unicore-integration/actions/workflows/publish-to-pypi.yml
 
 This project integrates `UNICORE <https://github.com/UNICORE-EU>`_ and `Apache Airflow <https://airflow.apache.org/>`_.
@@ -34,6 +34,31 @@ Airflow is a platform to programmatically author, schedule and monitor workflows
 In the current state, this projects provides a set of airflow `operators <https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/operators.html>`_, which can be used as part of airflow workflows to submit jobs to Unicore.
 The UnicoreExecutor only offers experimental support for airflow 3 so far. Further support is currently being worked on.
 
+-------------------------
+Using the UnicoreExecutor
+-------------------------
+
+To use the UnicoreExecutor, this library needs to be installed in your airflow environment and then some configuration work needs to be done.
+
+How to configure these settigns is up to your deployment, as it uses the standard airflow configuration mechanism.
+In case of a helm deployemnt via the official helm chart, you will need to use environment variables, as all unicore related options are not present in the chart and will cause schema-validation to fail.
+
+All options fall under the [unicore.executor] section in airflow.cfg, or have the ``AIRFLOW__UNICORE_EXECUTOR__`` prefix as an environment variable.
+
+========================= ============================================ ===========================================================================================
+Option name               default                                      description
+========================= ============================================ ===========================================================================================
+EXECUTION_API_SERVER_URL  <The default from the airflow config>        The url to reach the airflow API server from the execution environment (e.g. compute nodes)
+AUTH_TOKEN                mandatory                                    The unicore auth token to use for job submission
+DEFAULT_URL               http://localhost:8080/DEMO-SITE/rest/core    The default unicore site to submit jobs to
+DEFAULT_ENV               mandatory                                    The default activation script for a functional airflow environment on the execution machine
+TMP_DIR                   /tmp                                         A temporary directory to store data such as GitDagBundles
+========================= ============================================ ===========================================================================================
+
+The default env is loaded via ``. default_env.sh``, and must enable an environment, where python is available in a suitable version, and the ``apache-airflow-task-sdk`` and ``apache-airflow-providers-git`` packages are available. All other dependencies depend on the dags to be run, but must already be included in the environment.
+
+A simple solution for this may be the "activate" script for a python venv. If the target systems requires additional commands to enable python (e.g. ``module load``), these may be added to the top of the activate script.
+
 ---------------------------
 Using the Unicore Operators
 ---------------------------

{airflow_unicore_integration-0.1.13.dist-info → airflow_unicore_integration-0.2.1.dist-info}/RECORD

@@ -1,18 +1,18 @@
 airflow_unicore_integration/__init__.py,sha256=Qy1mlyxe2Y-PPSn0LgIW0sT6BxFuwW4_LGsPBf-Wm4s,549
 airflow_unicore_integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/executors/run_task_via_supervisor.py,sha256=3ErgPf-Oy3B4Di5yNXhhPkaojIJykvCxMZ9MlKSYPI8,2756
-airflow_unicore_integration/executors/unicore_executor.py,sha256=E1nOskWSBmC-ReLRvA8E3bY-G0lpxP403tazlBNhgFQ,5919
+airflow_unicore_integration/executors/unicore_executor.py,sha256=KRkwUVOvVTBTtxlkuL2D9UsSKjavtk8mTRXsMZJT_nI,6289
 airflow_unicore_integration/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airflow_unicore_integration/hooks/unicore_hooks.py,sha256=NAR5YmnPVzSsqpNVpRHt1sS3_HkEP_e8qoCv7daUZAA,2579
+airflow_unicore_integration/hooks/unicore_hooks.py,sha256=NEJ6sgQCD9AyZO9TA3vOR8RdijqaIX_FqY-Abz74Yys,3148
 airflow_unicore_integration/operators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/operators/container.py,sha256=D-2La59XZazXvBDVn-_wFIZ0IrCYeGy18miK8S5Usmc,693
 airflow_unicore_integration/operators/unicore_operators.py,sha256=PrcB-riakRUt8DHR9Hkk0ccOe-xk6NjJmNAmDpNgpN4,16868
 airflow_unicore_integration/policies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/util/job.py,sha256=vYyaNzGw2EQWQlDTGO6q4T1gZTHoB9-RIb5KYYd6pAc,10556
 airflow_unicore_integration/util/launch_script_content.py,sha256=42_aFpaCMmvFmmUxQDGcudkleX1YSK_yYWE8T41NOy0,2915
-airflow_unicore_integration-0.1.13.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
-airflow_unicore_integration-0.1.13.dist-info/METADATA,sha256=KXZXAPP9dv8EN2UheZDJ9TeXiIb5jL-t6AggYOJFDPU,11189
-airflow_unicore_integration-0.1.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-airflow_unicore_integration-0.1.13.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
-airflow_unicore_integration-0.1.13.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
-airflow_unicore_integration-0.1.13.dist-info/RECORD,,
+airflow_unicore_integration-0.2.1.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
+airflow_unicore_integration-0.2.1.dist-info/METADATA,sha256=tCUfwjb256THL9La8bfC5KU9BUPqaqjUoflDcG5eyUk,13696
+airflow_unicore_integration-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+airflow_unicore_integration-0.2.1.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
+airflow_unicore_integration-0.2.1.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
+airflow_unicore_integration-0.2.1.dist-info/RECORD,,