airflow-unicore-integration 0.1.10__py3-none-any.whl → 0.1.12__py3-none-any.whl

airflow_unicore_integration/util/job.py

@@ -25,6 +25,7 @@ class JobDescriptionGenerator:
     EXECUTOR_CONFIG_PROJECT = "Project"  # gets added to the unicore job description
     EXECUTOR_CONFIG_PRE_COMMANDS = "precommands"  # gets added to the unicore job description
     EXECUTOR_CONFIG_POST_COMMANDS = "postcommands"  # gets added to the unicore job descirption
+    EXECUTOR_CONFIG_JOB_TYPE = "job_type"
     EXECUTOR_CONFIG_UNICORE_CONN_KEY = (
         "unicore_connection_id"  # alternative connection id for the Unicore connection to use
     )
@@ -59,6 +60,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         user_added_pre_commands: list[str] = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_PRE_COMMANDS, [])  # type: ignore
         user_defined_python_env: str = workload.ti.executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_PYTHON_ENV_KEY, None)  # type: ignore
         user_added_post_commands: list[str] = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_POST_COMMANDS, [])  # type: ignore
+        user_defined_job_type: str = executor_config.get(JobDescriptionGenerator.EXECUTOR_CONFIG_JOB_TYPE, None)  # type: ignore
         # get local dag path from cmd and fix dag path in arguments
         dag_rel_path = str(workload.dag_rel_path)
         if dag_rel_path.startswith("DAG_FOLDER"):
@@ -71,6 +73,10 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
         )
         logger.debug(f"Server is {server}")
 
+        # set job type
+        if user_defined_job_type:
+            job_descr_dict["Job type"] = user_defined_job_type
+
         # check which python virtualenv to use
         if user_defined_python_env:
             python_env = user_defined_python_env
@@ -105,7 +111,6 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
             # "AIRFLOW__CORE__DAGS_FOLDER": "./",
             "AIRFLOW__LOGGING__LOGGING_LEVEL": "DEBUG",
             "AIRFLOW__CORE__EXECUTOR": "LocalExecutor,airflow_unicore_integration.executors.unicore_executor.UnicoreExecutor",
-            "AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_STORAGE_PATH": f"{tmp_dir}/{workload.ti.id}/dagbundle",
         }
 
         # build filecontent string for importing in the job | this is needed to avoid confusing nested quotes and trying to escape them properly when using unicore env vars directly
@@ -136,9 +141,14 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
             ):
                 git_hook = GitHook(conn_id_to_transmit)
                 git_remote_url = git_hook.repo_url
-                git_local_url = f"{tmp_dir}/{workload.ti.id}/dagmirror"
-                # add precommand to clone repo on ligon node
-                git_precommand = f". {python_env} && mkdir -p {tmp_dir}/{workload.ti.id}/dagmirror && mkdir -p {tmp_dir}/{workload.ti.id}/dagbundle && git clone {git_remote_url} {git_local_url}"
+                git_dir_prefix = f"{tmp_dir}/{workload.ti.dag_id}/{workload.ti.task_id}/{workload.ti.run_id}/{workload.ti.try_number}"
+                git_local_url = f"{git_dir_prefix}/dagmirror"
+                dag_bundle_path = f"{git_dir_prefix}/dagbundle"
+                # add precommand to clone repo on login node
+                git_precommand = f". {python_env} && mkdir -p {git_local_url} && mkdir -p {dag_bundle_path} && git clone {git_remote_url} {git_local_url}"
+                job_descr_dict["Environment"][
+                    "AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_STORAGE_PATH"
+                ] = f"{dag_bundle_path}"
                 logger.info(f"git precommand is {git_precommand}")
                 user_added_pre_commands.append(git_precommand)
                 # add connection to local clone to env of job
@@ -150,7 +160,7 @@ class NaiveJobDescriptionGenerator(JobDescriptionGenerator):
                 )
                 logger.info(f"connection is '{airflow_conn_string}'")
                 # add cleanup of local git repo to job description
-                git_cleanup_command = f"rm -r {tmp_dir}/{workload.ti.id}"
+                git_cleanup_command = f"rm -r {git_dir_prefix}"
                 logger.info(f"git cleanup is {git_cleanup_command}")
                 user_added_post_commands.append(git_cleanup_command)
 

{airflow_unicore_integration-0.1.10.dist-info → airflow_unicore_integration-0.1.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.1.10
+Version: 0.1.12
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause

{airflow_unicore_integration-0.1.10.dist-info → airflow_unicore_integration-0.1.12.dist-info}/RECORD

@@ -7,11 +7,11 @@ airflow_unicore_integration/hooks/unicore_hooks.py,sha256=JjcjogWtN1xveagpkraQuY
 airflow_unicore_integration/operators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/operators/unicore_operators.py,sha256=xYX1t_QzjpqHZDoEu6jJNNXVmBEnjIlApFWvgYoYoB0,16790
 airflow_unicore_integration/policies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airflow_unicore_integration/util/job.py,sha256=sOCizQa748VFkfa2eIXBnDDe0SxUSBgKRuZsX0gcZtE,10071
+airflow_unicore_integration/util/job.py,sha256=vYyaNzGw2EQWQlDTGO6q4T1gZTHoB9-RIb5KYYd6pAc,10556
 airflow_unicore_integration/util/launch_script_content.py,sha256=42_aFpaCMmvFmmUxQDGcudkleX1YSK_yYWE8T41NOy0,2915
-airflow_unicore_integration-0.1.10.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
-airflow_unicore_integration-0.1.10.dist-info/METADATA,sha256=LaldAzHtkWk8Ex0OuGZ6RNc_rg_0PQhHhPmANcsaW8k,11189
-airflow_unicore_integration-0.1.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-airflow_unicore_integration-0.1.10.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
-airflow_unicore_integration-0.1.10.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
-airflow_unicore_integration-0.1.10.dist-info/RECORD,,
+airflow_unicore_integration-0.1.12.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
+airflow_unicore_integration-0.1.12.dist-info/METADATA,sha256=aK_J520LHA3xOf4vqqpo71zcJ6WDrVTiMf120Ph00R8,11189
+airflow_unicore_integration-0.1.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+airflow_unicore_integration-0.1.12.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
+airflow_unicore_integration-0.1.12.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
+airflow_unicore_integration-0.1.12.dist-info/RECORD,,