airflow-unicore-integration 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

airflow_unicore_integration/util/job.py

@@ -63,7 +63,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         else:
             python_env = conf.get("unicore.executor", "DEFAULT_ENV")
         # prepare dag file to be uploaded via unicore
-        # dag_file = open(local_dag_path)
+        # dag_file = open("/tmp/test")
         # dag_content = dag_file.readlines()
         # dag_import = {"To": dag_rel_path, "Data": dag_content}
         worker_script_import = {
@@ -84,20 +84,28 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
             "AIRFLOW__CORE__DAGS_FOLDER": "./",
             "AIRFLOW__LOGGING__LOGGING_LEVEL": "DEBUG",
             "AIRFLOW__CORE__EXECUTOR": "LocalExecutor,airflow_unicore_integration.executors.unicore_executor.UnicoreExecutor",
-            "AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST": os.environ.get(
-                "AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST", ""
-            ),
         }
+
+        # build filecontent string for importing in the job | this is needed to avoid confusing nested quotes and trying to escape them properly when using unicore env vars directly
+        env_file_content: list[str] = [
+            f"export AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='{os.environ.get("AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST", "")}'"
+        ]
+
         # insert connection details that are provided via env vars to get bundles
         for env_key in os.environ.keys():
             if env_key.startswith("AIRFLOW_CONN_"):
-                job_descr_dict["Environment"][env_key] = os.environ[env_key]
-        user_added_pre_commands.append(f"source {python_env}/bin/activate")
+                env_file_content.append(f"export {env_key}='{os.environ[env_key]}'")
+
+        airflow_env_import = {"To": "airflow_config.env", "Data": env_file_content}
+
+        user_added_pre_commands.append(
+            f"source airflow_config.env && source {python_env}/bin/activate"
+        )
         job_descr_dict["User precommand"] = ";".join(user_added_pre_commands)
         job_descr_dict["RunUserPrecommandOnLoginNode"] = (
             "false"  # precommand includes activating the python env, this should be done on compute node right before running the job
         )
-        job_descr_dict["Imports"] = [worker_script_import]
+        job_descr_dict["Imports"] = [worker_script_import, airflow_env_import]
         # add user defined options to description
         if user_added_env:
             job_descr_dict["Environment"].update(user_added_env)

{airflow_unicore_integration-0.1.3.dist-info → airflow_unicore_integration-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.1.3
+Version: 0.1.5
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause

{airflow_unicore_integration-0.1.3.dist-info → airflow_unicore_integration-0.1.5.dist-info}/RECORD

@@ -7,10 +7,10 @@ airflow_unicore_integration/hooks/unicore_hooks.py,sha256=JjcjogWtN1xveagpkraQuY
 airflow_unicore_integration/operators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/operators/unicore_operators.py,sha256=xYX1t_QzjpqHZDoEu6jJNNXVmBEnjIlApFWvgYoYoB0,16790
 airflow_unicore_integration/policies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airflow_unicore_integration/util/job.py,sha256=HNWwr7kENnnmX0gO65jm1yGHAYuTFZJ8jg-FKKiZ1ro,6327
-airflow_unicore_integration-0.1.3.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
-airflow_unicore_integration-0.1.3.dist-info/METADATA,sha256=AE34sl5fO5XTUxv-b71OvctxROXUK22Ip8hOC9J4_4o,11188
-airflow_unicore_integration-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-airflow_unicore_integration-0.1.3.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
-airflow_unicore_integration-0.1.3.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
-airflow_unicore_integration-0.1.3.dist-info/RECORD,,
+airflow_unicore_integration/util/job.py,sha256=BSP-XuWL5-TRGACX__MavIcmPAA52gYLdwWL236cVz8,6703
+airflow_unicore_integration-0.1.5.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
+airflow_unicore_integration-0.1.5.dist-info/METADATA,sha256=ZtH2sUzOSHxsDmVSIgb-yJ61i7bi9Zc2olzdPhaeMHE,11188
+airflow_unicore_integration-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+airflow_unicore_integration-0.1.5.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
+airflow_unicore_integration-0.1.5.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
+airflow_unicore_integration-0.1.5.dist-info/RECORD,,