pinexq-client 0.9.2.20251028.52__py3-none-any.whl → 0.10.4rc1__py3-none-any.whl

pinexq_client/job_management/tool/job.py

@@ -1,5 +1,9 @@
 import json as json_
 import warnings
+import queue
+import datetime
+from datetime import datetime, timedelta
+
 from typing import Any, Self, List
 
 import httpx
@@ -7,6 +11,7 @@ from httpx import URL
 from pydantic import BaseModel, ConfigDict
 
 from pinexq_client.core import Link, MediaTypes, ClientException, ApiException
+from pinexq_client.core.api_event_manager import ApiEventManagerSingleton
 from pinexq_client.core.polling import wait_until, PollingException
 from pinexq_client.job_management.enterjma import enter_jma
 from pinexq_client.job_management.hcos import WorkDataLink, ProcessingStepLink, InputDataSlotHco, OutputDataSlotHco
@@ -92,7 +97,7 @@ class Job:
         .select_processing(processing_step='job_processing')
         .configure_parameters(**job_parameters)
         .start()
-        .wait_for_state(JobStates.completed)
+        .wait_for_completion()
         .delete()
     )
     """
@@ -127,7 +132,7 @@ class Job:
             The newly created job as `Job` object
         """
         job_link = self._jobs_root.create_job_action.execute(
-            CreateJobParameters(name=name)
+            CreateJobParameters(Name=name)
         )
         self._get_by_link(job_link)
         return self
@@ -182,7 +187,7 @@ class Job:
         self._raise_if_no_hco()
         parent_job_url = self.job_hco.self_link.get_url()
         sub_job_link = self._jobs_root.create_subjob_action.execute(
-            CreateSubJobParameters(name=name, parent_job_url=str(parent_job_url))
+            CreateSubJobParameters(Name=name, ParentJobUrl=str(parent_job_url))
         )
         sub_job = Job(self._client)
         sub_job._get_by_link(sub_job_link)
@@ -256,10 +261,10 @@ class Job:
                 raise TypeError('Instance passed to "function_name" is not of type "str"')
             # ToDo: provide more parameters to query a processing step
             query_param = ProcessingStepQueryParameters(
-                filter=ProcessingStepFilterParameter(
-                    function_name=function_name,
-                    function_name_match_type=FunctionNameMatchTypes.match_exact,
-                    version=function_version
+                Filter=ProcessingStepFilterParameter(
+                    FunctionName=function_name,
+                    FunctionNameMatchType=FunctionNameMatchTypes.match_exact,
+                    Version=function_version
                 )
             )
             query_result = self._processing_step_root.query_action.execute(query_param)
@@ -271,7 +276,7 @@ class Job:
             processing_url = query_result.processing_steps[0].self_link.get_url()
 
         self.job_hco.select_processing_action.execute(
-            SelectProcessingParameters(processing_step_url=str(processing_url))
+            SelectProcessingParameters(ProcessingStepUrl=str(processing_url))
         )
 
         self.refresh()
@@ -322,13 +327,64 @@ class Job:
         result = self.job_hco.result
         return json_.loads(result) if result else None
 
-    def wait_for_state(self, state: JobStates, timeout_ms: int = 5000, polling_interval_ms: int = 1000) -> Self:
+    def wait_for_state_sse(self, state: JobStates, timeout_s: float | None = None, fallback_polling_interval_s: float = 300) -> Self:
+        self._raise_if_no_hco()
+
+        # early exit
+        if self.job_hco.state == state:
+            return self
+        if self.job_hco.state == JobStates.error:
+            error_reason = self.job_hco.error_description
+            raise PollingException(f"Job failed'. Error:{error_reason}")
+
+        if timeout_s is None:
+            function_timeout_on = datetime.max
+        else:
+            # end this wait hard
+            function_timeout_on = datetime.now() + timedelta(seconds=timeout_s)
+
+        job_changed_signal = queue.Queue()
+        manager = ApiEventManagerSingleton()
+        job_url_str = str(self.job_hco.self_link.get_url())
+        manager.subscribe_waiter(self._client, job_url_str, job_changed_signal)
+
+        try:
+            self.get_state()
+            job_done = self.job_hco.state == state
+            while not job_done:
+                time_till_function_timeout = function_timeout_on - datetime.now()
+                if time_till_function_timeout.total_seconds() <= 0.0:
+                    raise PollingException(f"Timeout waiting for Job state. Current state: {self.job_hco.state}")
+
+                next_wait_timeout_s = min(float(time_till_function_timeout.seconds), fallback_polling_interval_s)
+
+                try:
+                    job_changed_signal.get(timeout=next_wait_timeout_s)
+                except queue.Empty:
+                    # nothing we do just a poll and loop again
+                    pass
+
+                # read all messages since we only want to poll new state once if there are multiple messages
+                while not job_changed_signal.empty():
+                    job_changed_signal.get(timeout=next_wait_timeout_s)
+
+                self.get_state()
+                job_done = self.job_hco.state == state
+                if self.job_hco.state == JobStates.error:
+                    error_reason = self.job_hco.error_description
+                    raise PollingException(f"Job failed'. Error:{error_reason}")
+        finally:
+            manager.unsubscribe_waiter(self._client, job_url_str, job_changed_signal)
+
+        return self
+
+    def wait_for_state(self, state: JobStates, timeout_s: float | None = None, polling_interval_s: float = 1) -> Self:
         """Wait for this job to reach a state. If the job enters error state an exception is risen
 
         Args:
             state: The state to wait for. After the job enters this state this function returns.
-            timeout_ms: Time span in milliseconds to wait for reaching the state before raising an exception.
-            polling_interval_ms: will determine how fast the API is polled for updates.
+            timeout_s: Time span in seconds to wait for reaching the state before raising an exception.
+            polling_interval_s: will determine how fast the API is polled for updates.
             Note that low values will produce unnecessary load.
 
         Returns:
@@ -340,10 +396,10 @@ class Job:
         try:
             wait_until(
                 condition=lambda: self.get_state() == state,
-                timeout_ms=timeout_ms,
+                timeout_ms= int(timeout_s * 1000) if timeout_s is not None else None,
                 timeout_message="Waiting for job completion",
                 error_condition=lambda: self.job_hco.state == JobStates.error,
-                polling_interval_ms=polling_interval_ms
+                polling_interval_ms= int(polling_interval_s * 1000)
             )
         except TimeoutError as timeout:
             raise TimeoutError(
@@ -358,18 +414,17 @@ class Job:
 
         return self
 
-    def wait_for_completion(self, timeout_ms: int = 60000, polling_interval_ms: int = 500) -> Self:
+    def wait_for_completion(self, timeout_s: float | None = None) -> Self:
         """Wait for this job to reach the state 'completed'.
 
         Args:
-            timeout_ms: Timeout to wait for the job to reach the next state.
-            polling_interval_ms: will determine how fast the API is polled for updates.
+            timeout_s: Timeout to wait for the job to reach the next state.
             Note that low values will produce unnecessary load.
 
         Returns:
             This `Job` object
         """
-        return self.wait_for_state(JobStates.completed, timeout_ms, polling_interval_ms)
+        return self.wait_for_state_sse(JobStates.completed, timeout_s)
 
     def assign_input_dataslot(
             self,
@@ -404,7 +459,7 @@ class Job:
         dataslot = self.job_hco.input_dataslots[index]
         dataslot.select_workdata_action.execute(
             parameters=SelectWorkDataForDataSlotParameters(
-                work_data_url=str(work_data.get_url())
+                WorkDataUrl=str(work_data.get_url())
             )
         )
         self.refresh()
@@ -445,7 +500,7 @@ class Job:
         dataslot = self.job_hco.input_dataslots[index]
         dataslot.select_workdata_collection_action.execute(
             parameters=SelectWorkDataCollectionForDataSlotParameters(
-                work_data_urls=[str(workdata_link.get_url()) for workdata_link in work_datas]
+                WorkDataUrls=[str(workdata_link.get_url()) for workdata_link in work_datas]
             )
         )
         self.refresh()
@@ -483,15 +538,15 @@ class Job:
     ) -> JobQueryResultHco:
         self._raise_if_no_hco()
         filter_param = JobFilterParameter(
-            is_sub_job=True,
-            parent_job_url=str(self.job_hco.self_link.get_url()),
-            state=state,
-            name=name,
+            IsSubJob=True,
+            ParentJobUrl=str(self.job_hco.self_link.get_url()),
+            State=state,
+            Name=name,
             show_deleted=show_deleted,
-            processing_step_url=processing_step_url,
+            ProcessingStepUrl=processing_step_url,
         )
 
-        query_param = JobQueryParameters(sort_by=sort_by, filter=filter_param)
+        query_param = JobQueryParameters(SortBy=sort_by, Filter=filter_param)
         job_query_result = self._jobs_root.job_query_action.execute(query_param)
         return job_query_result
 
@@ -511,32 +566,32 @@ class Job:
         query_result = self._get_sub_jobs(state=state)
         return query_result.total_entities
 
-    def wait_for_sub_jobs_complete(self, timeout_ms: int = 60000, polling_interval_ms: int = 1000) -> Self:
+    def wait_for_sub_jobs_complete(self, timeout_s: float = 60, polling_interval_s: float = 1) -> Self:
         """Wait for all sub-jobs to reach the state 'completed'.
 
         This function will block execution until the state is reached or raise an exception
         if the operation timed out or a sub-job returned an error. Only started jobs will be watched.
 
         Args:
-            timeout_ms: Timeout to wait for the sub-jobs to reach the next state.
+            timeout_s: Timeout to wait for the sub-jobs to reach the next state.
 
         Returns:
             This `Job` object
-            :param timeout_ms: Wil determine how long to wait for success
-            :param polling_interval_ms: will determine how fast the API is polled for updates.
+            :param timeout_s: Wil determine how long to wait for success
+            :param polling_interval_s: will determine how fast the API is polled for updates.
             Note that low values will produce unnecessary load.
         """
         wait_until(
             condition=lambda: self.sub_jobs_in_state(JobStates.pending) == 0,
-            timeout_ms=timeout_ms,
-            timeout_message=f"Timeout while waiting for sub-jobs to complete! [timeout: {timeout_ms}ms]",
-            polling_interval_ms=polling_interval_ms
+            timeout_ms= int(timeout_s * 1000),
+            timeout_message=f"Timeout while waiting for sub-jobs to complete! [timeout: {timeout_s}s]",
+            polling_interval_ms= int(polling_interval_s * 1000)
         )
         wait_until(
             condition=lambda: self.sub_jobs_in_state(JobStates.processing) == 0,
-            timeout_ms=timeout_ms,
-            timeout_message=f"Timeout while waiting for sub-jobs to complete! [timeout: {timeout_ms}ms]",
-            polling_interval_ms=polling_interval_ms
+            timeout_ms= int(timeout_s * 1000),
+            timeout_message=f"Timeout while waiting for sub-jobs to complete! [timeout: {timeout_s}ms]",
+            polling_interval_ms= int(polling_interval_s * 1000)
         )
 
         error_count = self.sub_jobs_in_state(JobStates.error)
@@ -574,29 +629,47 @@ class Job:
             delete_subjobs_with_data: bool = True
     ) -> Self:
         """Delete this job after deleting output workdata and subjobs (recursive call) depending on the flag.
-        Afterward, also deletes input workdata depending on the flag. This is a best effort operation,
-        if an operation can not be executed a warning will be printed but the process continues.
+                Afterward, also deletes input workdata depending on the flag. This is a best effort operation,
+                if an operation can not be executed a warning will be printed but the process continues.
 
-        Args:
-            delete_output_workdata: boolean flag to specify if output WorkData should be attempted for deletion. Default: True
-            delete_input_workdata: boolean flag to specify if input WorkData should be attempted for deletion. Default: False
-            delete_subjobs_with_data: boolean flag tp specify if Sub jobs should be attempted for deletion. Default: True
+                Args:
+                    delete_output_workdata: boolean flag to specify if output WorkData should be attempted for deletion. Default: True
+                    delete_input_workdata: boolean flag to specify if input WorkData should be attempted for deletion. Default: False
+                    delete_subjobs_with_data: boolean flag tp specify if Sub jobs should be attempted for deletion. Default: True
 
-        Returns:
-            This `Job` object
+                Returns:
+                    This `Job` object
         """
+        self._delete_with_associated_internal(
+            delete_output_workdata=delete_output_workdata,
+            delete_input_workdata=delete_input_workdata,
+            delete_subjobs_with_data=delete_subjobs_with_data,
+            recursion_depth = 0)
+
+    def _delete_with_associated_internal(
+            self,
+            *,
+            delete_output_workdata: bool = True,
+            delete_input_workdata: bool = False,
+            delete_subjobs_with_data: bool = True,
+            recursion_depth: int = 0
+    ) -> Self:
         self._raise_if_no_hco()
 
         # delete subjobs
-        if delete_subjobs_with_data is True:
+        if delete_subjobs_with_data:
+            if recursion_depth > 20:
+                raise Exception("Recursion limit of subjob deletion exceeded.")
+
             for subjob in self._get_sub_jobs().iter_flat():
                 try:
                     # recursion
                     subjob_wrapper = Job.from_hco(subjob)
-                    subjob_wrapper.delete_with_associated(
+                    subjob_wrapper._delete_with_associated_internal(
                         delete_output_workdata=delete_output_workdata,
                         delete_input_workdata=delete_input_workdata,
-                        delete_subjobs_with_data=delete_subjobs_with_data)
+                        delete_subjobs_with_data=delete_subjobs_with_data,
+                        recursion_depth = recursion_depth + 1)
                     if subjob.self_link.exists():
                         warnings.warn(f"Could not delete subjob: {subjob.self_link.get_url()}")
                 except (ClientException, ApiException) as e:
@@ -605,7 +678,7 @@ class Job:
         self.refresh()
 
         # delete output workdatas
-        if delete_output_workdata is True:
+        if delete_output_workdata:
             for slot in self.job_hco.output_dataslots:
                 for wd in slot.assigned_workdatas:
                     try:
@@ -633,7 +706,7 @@ class Job:
             warnings.warn(f"Could not delete job: {self.self_link().get_url()}\n{e}")
 
         # finally delete input workdatas
-        if delete_input_workdata is True:
+        if delete_input_workdata:
             for slot in self.job_hco.input_dataslots:
                 for wd in slot.selected_workdatas:
                     try:
@@ -692,7 +765,7 @@ class Job:
         """
         self._raise_if_no_hco()
         self.job_hco.edit_tags_action.execute(
-            SetJobTagsParameters(tags=tags)
+            SetJobTagsParameters(Tags=tags)
         )
         self.refresh()
         return self
@@ -730,14 +803,23 @@ class Job:
             processing_step_instance: ProcessingStep | None = None,
             start: bool = True,
             parameters: str | None = None,
-            allow_output_data_slots: bool | None = None,
+            allow_output_data_deletion: bool | None = None,
             input_data_slots: List[InputDataSlotParameterFlexible] | None = None,
     ) -> Self:
         """
         Creates a new job and configures it rapidly with RapidJobSetupParameters.
 
         Args:
-            parameters: The parameters to configure the job with.
+            name: Name of the job to be created
+            parent_job_url: URL of the parent job as JobLink. Only one of parent_job_url or parent_job_instance must be provided.
+            parent_job_instance: Parent job as Job instance. Only one of parent_job_url or parent_job_instance must be provided.
+            tags: Tags to assign to the job
+            processing_step_url: URL of the processing step as ProcessingStepLink. Only one of processing_step_url or processing_step_instance must be provided.
+            processing_step_instance: Processing step as ProcessingStep instance. Only one of processing_step_url or processing_step_instance must be provided.
+            start: Flag indicating whether to start the job after creation
+            parameters: Input parameters to the job
+            allow_output_data_deletion: Flag indicating whether to allow output data deletion
+            input_data_slots: List of InputDataSlotParameterFlexible to assign work data to input data slots
 
         Returns:
             The newly created job as `Job` object
@@ -785,7 +867,7 @@ class Job:
             Tags=tags,
             Start=start,
             Parameters=parameters,
-            AllowOutputDataDeletion=allow_output_data_slots,
+            AllowOutputDataDeletion=allow_output_data_deletion,
             InputDataSlots=input_data_slots
         )