airflow-unicore-integration 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. {airflow_unicore_integration-0.1.2/src/airflow_unicore_integration.egg-info → airflow_unicore_integration-0.1.4}/PKG-INFO +1 -1
  2. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/pyproject.toml +1 -1
  3. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/util/job.py +12 -1
  4. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4/src/airflow_unicore_integration.egg-info}/PKG-INFO +1 -1
  5. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/LICENSE +0 -0
  6. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/README.rst +0 -0
  7. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/setup.cfg +0 -0
  8. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/__init__.py +0 -0
  9. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/executors/__init__.py +0 -0
  10. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/executors/run_task_via_supervisor.py +0 -0
  11. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/executors/unicore_executor.py +0 -0
  12. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/hooks/__init__.py +0 -0
  13. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/hooks/unicore_hooks.py +0 -0
  14. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/operators/__init__.py +0 -0
  15. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/operators/unicore_operators.py +0 -0
  16. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/policies/__init__.py +0 -0
  17. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration.egg-info/SOURCES.txt +0 -0
  18. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration.egg-info/dependency_links.txt +0 -0
  19. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration.egg-info/entry_points.txt +0 -0
  20. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration.egg-info/requires.txt +0 -0
  21. {airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration.egg-info/top_level.txt +0 -0
{airflow_unicore_integration-0.1.2/src/airflow_unicore_integration.egg-info → airflow_unicore_integration-0.1.4}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: airflow-unicore-integration
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: Running Unicore Jobs from airflow DAGs.
5
5
  Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
6
6
  License-Expression: BSD-3-Clause
{airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/pyproject.toml RENAMED
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
7
7
 
8
8
  [project]
9
9
  name = "airflow-unicore-integration"
10
- version = "0.1.2"
10
+ version = "0.1.4"
11
11
  authors = [
12
12
  { name="Christian Böttcher", email="c.boettcher@fz-juelich.de" },
13
13
  ]
{airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4}/src/airflow_unicore_integration/util/job.py RENAMED
@@ -1,3 +1,5 @@
1
+ import json
2
+ import os
1
3
  from typing import Any
2
4
  from typing import Dict
3
5
 
@@ -83,11 +85,20 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
83
85
  "AIRFLOW__CORE__DAGS_FOLDER": "./",
84
86
  "AIRFLOW__LOGGING__LOGGING_LEVEL": "DEBUG",
85
87
  "AIRFLOW__CORE__EXECUTOR": "LocalExecutor,airflow_unicore_integration.executors.unicore_executor.UnicoreExecutor",
88
+ "AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST": json.dumps(
89
+ os.environ.get("AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST", "")
90
+ ).replace("\\n", ""),
86
91
  }
92
+ # insert connection details that are provided via env vars to get bundles
93
+ for env_key in os.environ.keys():
94
+ if env_key.startswith("AIRFLOW_CONN_"):
95
+ job_descr_dict["Environment"][env_key] = json.dumps(os.environ[env_key]).replace(
96
+ "\\n", ""
97
+ )
87
98
  user_added_pre_commands.append(f"source {python_env}/bin/activate")
88
99
  job_descr_dict["User precommand"] = ";".join(user_added_pre_commands)
89
100
  job_descr_dict["RunUserPrecommandOnLoginNode"] = (
90
- "false" # precommand is activating the python env, this can also be done on compute node right before running the job
101
+ "false" # precommand includes activating the python env, this should be done on compute node right before running the job
91
102
  )
92
103
  job_descr_dict["Imports"] = [worker_script_import]
93
104
  # add user defined options to description
{airflow_unicore_integration-0.1.2 → airflow_unicore_integration-0.1.4/src/airflow_unicore_integration.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: airflow-unicore-integration
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: Running Unicore Jobs from airflow DAGs.
5
5
  Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
6
6
  License-Expression: BSD-3-Clause