airflow-unicore-integration 0.1.1__tar.gz → 0.1.3__tar.gz

{airflow_unicore_integration-0.1.1/src/airflow_unicore_integration.egg-info → airflow_unicore_integration-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.1.1
+Version: 0.1.3
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause

{airflow_unicore_integration-0.1.1 → airflow_unicore_integration-0.1.3}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "airflow-unicore-integration"
-version = "0.1.1"
+version = "0.1.3"
 authors = [
   { name="Christian Böttcher", email="c.boettcher@fz-juelich.de" },
 ]

{airflow_unicore_integration-0.1.1 → airflow_unicore_integration-0.1.3}/src/airflow_unicore_integration/util/job.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any
 from typing import Dict
 
@@ -52,7 +53,9 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         # local_dag_path = conf.get("core", "DAGS_FOLDER") + "/" + dag_rel_path
         base_url = conf.get("api", "base_url", fallback="/")
         default_execution_api_server = f"{base_url.rstrip('/')}/execution/"
-        server = conf.get("core", "execution_api_server_url", fallback=default_execution_api_server)
+        server = conf.get(
+            "unicore.executor", "execution_api_server_url", fallback=default_execution_api_server
+        )
 
         # check which python virtualenv to use
         if user_defined_python_env:
@@ -65,7 +68,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         # dag_import = {"To": dag_rel_path, "Data": dag_content}
         worker_script_import = {
             "To": "run_task_via_supervisor.py",
-            "From": "https://gist.githubusercontent.com/cboettcher/3f1101a1d1b67e7944d17c02ecd69930/raw/6da9ec16ba598ddda9cf288900498fab5e226788/run_task_via_supervisor.py",
+            "From": "https://gist.githubusercontent.com/cboettcher/3f1101a1d1b67e7944d17c02ecd69930/raw/1d90bf38199d8c0adf47a79c8840c3e3ddf57462/run_task_via_supervisor.py",
         }
         # start filling the actual job description
         job_descr_dict["Name"] = f"{key.dag_id} - {key.task_id} - {key.run_id} - {key.try_number}"
@@ -81,11 +84,18 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
             "AIRFLOW__CORE__DAGS_FOLDER": "./",
             "AIRFLOW__LOGGING__LOGGING_LEVEL": "DEBUG",
             "AIRFLOW__CORE__EXECUTOR": "LocalExecutor,airflow_unicore_integration.executors.unicore_executor.UnicoreExecutor",
+            "AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST": os.environ.get(
+                "AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST", ""
+            ),
         }
+        # insert connection details that are provided via env vars to get bundles
+        for env_key in os.environ.keys():
+            if env_key.startswith("AIRFLOW_CONN_"):
+                job_descr_dict["Environment"][env_key] = os.environ[env_key]
         user_added_pre_commands.append(f"source {python_env}/bin/activate")
         job_descr_dict["User precommand"] = ";".join(user_added_pre_commands)
         job_descr_dict["RunUserPrecommandOnLoginNode"] = (
-            "false"  # precommand is activating the python env, this can also be done on compute node right before running the job
+            "false"  # precommand includes activating the python env, this should be done on compute node right before running the job
         )
         job_descr_dict["Imports"] = [worker_script_import]
         # add user defined options to description

{airflow_unicore_integration-0.1.1 → airflow_unicore_integration-0.1.3/src/airflow_unicore_integration.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.1.1
+Version: 0.1.3
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause