airflow-unicore-integration 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

airflow_unicore_integration/__init__.py

@@ -9,4 +9,7 @@ def get_provider_info():
                 "hook-class-name": "airflow_unicore_integration.hooks.unicore_hooks.UnicoreHook",
             }
         ],
+        "executors": [
+            "airflow_unicore_integration.executors.unicore_executor.UnicoreExecutor",
+        ],
     }

airflow_unicore_integration/executors/unicore_executor.py

@@ -19,8 +19,8 @@ from airflow.executors.workloads import All
 from airflow.executors.workloads import ExecuteTask
 from airflow.models.taskinstancekey import TaskInstanceKey
 from airflow.utils.state import TaskInstanceState
-
-from airflow_unicore_integration.hooks import unicore_hooks
+from pyunicore import client
+from pyunicore import credentials
 
 from ..util.job import JobDescriptionGenerator
 from ..util.job import NaiveJobDescriptionGenerator
@@ -39,9 +39,14 @@ STATE_MAPPINGS: Dict[uc_client.JobStatus, TaskInstanceState] = {
 
 class UnicoreExecutor(BaseExecutor):
 
+    EXECUTOR_CONFIG_UNICORE_CONN_KEY = (
+        "unicore_connection_id"  # alternative connection id for the Unicore connection to use
+    )
+    EXECUTOR_CONFIG_UNICORE_SITE_KEY = "unicore_site"  # alternative Unicore site to run at, only required if different than connection default
+    EXECUTOR_CONFIG_UNICORE_CREDENTIAL_KEY = "unicore_credential"  # alternative unicore credential to use for the job, only required if different than connection default
+
     def start(self):
         self.active_jobs: Dict[TaskInstanceKey, uc_client.Job] = {}
-        self.uc_conn = unicore_hooks.UnicoreHook().get_conn()
         # TODO get job description generator class and init params from config
         self.job_descr_generator: JobDescriptionGenerator = NaiveJobDescriptionGenerator()
 
@@ -67,25 +72,26 @@ class UnicoreExecutor(BaseExecutor):
         return []
 
     def _get_unicore_client(self, executor_config: dict | None = {}):
-        # TODO fix this only temporary solution
-        return self.uc_conn
-        # END TODO fix this
-        # include client desires from executor_config
-        unicore_conn_id = executor_config.get(  # type: ignore
-            UnicoreExecutor.EXECUTOR_CONFIG_UNICORE_CONN_KEY,
-            conf.get("unicore.executor", "UNICORE_CONN_ID"),
-        )  # task can provide a different unicore connection to use, else airflow-wide default is used
-        self.log.info(f"Using base unicore connection with id '{unicore_conn_id}'")
-        hook = unicore_hooks.UnicoreHook(uc_conn_id=unicore_conn_id)
-        unicore_site = executor_config.get(  # type: ignore
+        overwrite_unicore_site = executor_config.get(  # type: ignore
             UnicoreExecutor.EXECUTOR_CONFIG_UNICORE_SITE_KEY, None
         )  # task can provide a different site to run at, else default from connetion is used
-        unicore_credential = executor_config.get(  # type: ignore
+        overwrite_unicore_credential = executor_config.get(  # type: ignore
             UnicoreExecutor.EXECUTOR_CONFIG_UNICORE_CREDENTIAL_KEY, None
         )  # task can provide a different credential to use, else default from connection is used
-        return hook.get_conn(
-            overwrite_base_url=unicore_site, overwrite_credential=unicore_credential
+        user = conf.get("unicore.executor", "DEFAULT_USER", fallback="demouser")
+        password = conf.get("unicore.executor", "DEFAULT_PASS", fallback="test123")
+        base_url = conf.get(
+            "unicore.executor", "DEFAULT_URL", fallback="http://localhost:8080/DEMO-SITE/rest/core"
         )
+        credential = credentials.UsernamePassword(user, password)
+        if overwrite_unicore_site is not None:
+            base_url = overwrite_unicore_site
+        if overwrite_unicore_credential is not None:
+            credential = overwrite_unicore_credential
+        if not base_url:
+            raise TypeError()
+        conn = client.Client(credential, base_url)
+        return conn
 
     def _submit_job(self, workload: ExecuteTask):
         uc_client = self._get_unicore_client(executor_config=workload.ti.executor_config)

{airflow_unicore_integration-0.1.5.dist-info → airflow_unicore_integration-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-unicore-integration
-Version: 0.1.5
+Version: 0.1.7
 Summary: Running Unicore Jobs from airflow DAGs.
 Author-email: Christian Böttcher <c.boettcher@fz-juelich.de>
 License-Expression: BSD-3-Clause

{airflow_unicore_integration-0.1.5.dist-info → airflow_unicore_integration-0.1.7.dist-info}/RECORD

@@ -1,16 +1,16 @@
-airflow_unicore_integration/__init__.py,sha256=sibcOlQKBCUKfTbZlAl0L7r1d2nxf0nGAQys612vzeA,429
+airflow_unicore_integration/__init__.py,sha256=Qy1mlyxe2Y-PPSn0LgIW0sT6BxFuwW4_LGsPBf-Wm4s,549
 airflow_unicore_integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/executors/run_task_via_supervisor.py,sha256=3ErgPf-Oy3B4Di5yNXhhPkaojIJykvCxMZ9MlKSYPI8,2756
-airflow_unicore_integration/executors/unicore_executor.py,sha256=f2PKj66tZlWb39xF5YacEBoZ2-eZ_JR9pnKQwNHAybc,5619
+airflow_unicore_integration/executors/unicore_executor.py,sha256=QNfF5sW4fRou89Hjo7j7SbfsXBo4fW3F4hkgbThrY6c,6008
 airflow_unicore_integration/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/hooks/unicore_hooks.py,sha256=JjcjogWtN1xveagpkraQuYOdXjkp2lSnEdQc0waqhU4,1662
 airflow_unicore_integration/operators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/operators/unicore_operators.py,sha256=xYX1t_QzjpqHZDoEu6jJNNXVmBEnjIlApFWvgYoYoB0,16790
 airflow_unicore_integration/policies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airflow_unicore_integration/util/job.py,sha256=BSP-XuWL5-TRGACX__MavIcmPAA52gYLdwWL236cVz8,6703
-airflow_unicore_integration-0.1.5.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
-airflow_unicore_integration-0.1.5.dist-info/METADATA,sha256=ZtH2sUzOSHxsDmVSIgb-yJ61i7bi9Zc2olzdPhaeMHE,11188
-airflow_unicore_integration-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-airflow_unicore_integration-0.1.5.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
-airflow_unicore_integration-0.1.5.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
-airflow_unicore_integration-0.1.5.dist-info/RECORD,,
+airflow_unicore_integration-0.1.7.dist-info/licenses/LICENSE,sha256=hZ5ouAedeNr8ClHrQE-RLsgMsARcmv3kSZz7tE2BTJE,1526
+airflow_unicore_integration-0.1.7.dist-info/METADATA,sha256=abUzAW9IfF_GXeiC1DdtUxxKxjAIfj9vJ8WzfBjQPu8,11188
+airflow_unicore_integration-0.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+airflow_unicore_integration-0.1.7.dist-info/entry_points.txt,sha256=PzEfCLYLSawjiYR-HNBzw8-YGfJxs1nPBULevgBQjoY,147
+airflow_unicore_integration-0.1.7.dist-info/top_level.txt,sha256=j45X-uIuOk3oL78iwlpHakMWtUkg__B7zUlJLwmZx6w,28
+airflow_unicore_integration-0.1.7.dist-info/RECORD,,