pinexq-client 0.9.2.20251028.52__py3-none-any.whl → 0.10.4rc1__py3-none-any.whl

pinexq_client/core/api_event_manager.py

@@ -0,0 +1,141 @@
+import queue
+import threading
+import time
+from datetime import datetime, timedelta
+from pydantic import BaseModel
+
+import httpx
+from httpx_sse import connect_sse
+
+from dataclasses import dataclass
+
+from pinexq_client.job_management import EntryPointHco, enter_jma
+
+
+@dataclass
+class WorkerInfo:
+    worker: threading.Thread
+    close_after: datetime | None = None
+
+class ApiEvent(BaseModel):
+    EventType: str
+    ResourceType: str
+    ResourceLink: str
+
+
+class ApiEventManagerSingleton:
+    _instance = None
+    listening_clients: dict[httpx.Client, dict[str, list[queue.Queue]]]
+    workers: dict[httpx.Client, WorkerInfo]
+    sse_connection_cooldown_s: int
+    connection_create_lock: threading.Lock
+    reconnect_time_s: int
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        # Only initialize once
+        if not hasattr(self, '_initialized'):
+            self._initialized = True
+            self.listening_clients = {}
+            self.workers = {}
+            self.connection_create_lock = threading.Lock()
+            self.sse_connection_cooldown_s = 30 # avoid opening and closing fast
+            self.reconnect_time_s = 5
+
+    def subscribe_waiter(self, client: httpx.Client, resource_link:str, wait_queue: queue.Queue):
+        open_sse_connections = False
+
+        if not client in self.listening_clients:
+            self.listening_clients[client] = {}
+            open_sse_connections = True
+
+        if resource_link not in self.listening_clients[client]:
+            self.listening_clients[client][resource_link] = []
+
+        self.listening_clients[client][resource_link].append(wait_queue)
+
+        with self.connection_create_lock:
+            if open_sse_connections:
+                self.open_sse_connection(client)
+            self.workers[client].close_after = None  # we use the connection now, do not close
+
+    def unsubscribe_waiter(self, client: httpx.Client, resource_link: str, wait_queue: queue.Queue):
+        if not client in self.listening_clients:
+            return
+
+        if resource_link not in self.listening_clients[client]:
+            return
+
+        # event should only be in here once
+        self.listening_clients[client][resource_link].remove(wait_queue)
+
+        # clean up if no events left
+        if len(self.listening_clients[client][resource_link]) == 0:
+            del self.listening_clients[client][resource_link]
+
+            # if noone is subscribed anymore set shutdown time
+            if not self.listening_clients[client]:
+                self.workers[client].close_after = datetime.now() + timedelta(seconds=self.sse_connection_cooldown_s)
+
+
+    def open_sse_connection(self, client: httpx.Client):
+        sse_endpoint = self.get_sse_endpoint_from_api(client)
+
+        # start worker for client
+        worker_thread = threading.Thread(target=self.consume_sse_events, args=(client, sse_endpoint))
+        self.workers[client] = WorkerInfo(worker=worker_thread)
+        worker_thread.start()
+
+    def consume_sse_events(self, client: httpx.Client, sse_endpoint: str):
+        while True:
+            try:
+                if client.is_closed:
+                    print("Client was closed, will close sse connection too")
+                    self.close_connections(client)
+                    return
+
+                with connect_sse(client, "GET", sse_endpoint) as event_source:
+                    for sse in event_source.iter_sse():
+                        # close the thread if no one is using it anymore and cooldown has passed
+                        close_after = self.workers[client].close_after
+                        if close_after and datetime.now() > close_after:
+                            self.close_connections(client)
+                            return
+
+                        if len(sse.data) <= 0: # keep alive has no data
+                            continue
+
+                        api_event = ApiEvent.model_validate_json(sse.data)
+
+                        # for now, we only process job events
+                        if api_event.ResourceType != "Job":
+                            continue
+
+                        job_link = api_event.ResourceLink
+                        subscribed = self.listening_clients[client]
+                        if job_link in subscribed:
+                            for wait_queue in subscribed[job_link]:
+                                wait_queue.put(job_link)
+            except httpx.ReadError:
+                print(f"SSE Connection lost. Reconnecting in {self.reconnect_time_s} seconds...")
+                time.sleep(self.reconnect_time_s)
+            except httpx.ConnectError as e:
+                print(f"Failed to connect to SSE endpoint: {e}. Retrying in {self.reconnect_time_s} seconds...")
+                time.sleep(self.reconnect_time_s)
+            except Exception as ex:
+                raise
+
+    def close_connections(self, client: httpx.Client):
+        del self.listening_clients[client]
+        del self.workers[client]  # worker not needed any more
+
+    @staticmethod
+    def get_sse_endpoint_from_api(client)-> str:
+        entrypoint: EntryPointHco = enter_jma(client)
+        endpoint =  str(entrypoint.info_link.navigate().api_events_endpoint.get_url())
+        return endpoint
+

pinexq_client/core/hco/hco_base.py

@@ -1,3 +1,4 @@
+import inspect
 from dataclasses import dataclass
 from typing import List, TypeVar, Generic, Callable, Any
 
@@ -51,6 +52,7 @@ class Hco(ClientContainer, Generic[THcoEntity]):
         if candidate_self_link is not None:
             return hash(URL(candidate_self_link.href).path)
         else:
+            # might be an issue if self link is not present from the beginning but is added later.
             return super().__hash__()
 
     def __eq__(self, other):
@@ -75,7 +77,8 @@ class Hco(ClientContainer, Generic[THcoEntity]):
         with `Property()` the variable gets set from a property in self._entity.properties
         with the same name.
         """
-        for var_name, var_type in self.__annotations__.items():
+        annotations = inspect.get_annotations(self.__class__)
+        for var_name, var_type in annotations.items():
             # Get the initialized object. The annotation contain only the annotated type.
             hco_property = getattr(self, var_name, None)
             # Skip everything that is not initialized as `Property()`

pinexq_client/core/polling.py

@@ -9,14 +9,14 @@ class PollingException(Exception):
 def wait_until(
         condition: Callable,
         polling_interval_ms: int = 200,
-        timeout_ms: int = 5000,
+        timeout_ms: int | None = 5000,
         timeout_message: str | None = None,
         error_condition: Callable | None = None,
         error_condition_message: str | None = None,
 
 ) -> None:
     start = time.time()
-    timeout = start + timeout_ms / 1000
+    timeout = (start + timeout_ms / 1000) if timeout_ms is not None else None
     while True:
         now = time.time()
         next_due = now + polling_interval_ms / 1000
@@ -30,7 +30,7 @@ def wait_until(
                 raise PollingException(
                     f"{f': {error_condition_message}' if error_condition_message else 'Error condition meet while waiting'}")
 
-        if (timeout > 0) and (now > timeout):
+        if timeout is not None and (timeout > 0) and (now > timeout):
             raise TimeoutError(
                 f"{f': {timeout_message}' if timeout_message else f'Timeout while waiting. Waited: {timeout_ms}ms'}")
 

pinexq_client/job_management/__init__.py

@@ -22,4 +22,4 @@ from .model import JobSortPropertiesSortParameter, ProcessingView
 from .tool import Job, JobGroup, ProcessingStep, WorkData
 
 # Protocol version the JMA is using
-__jma_version__ = [8, 5, 0]
+__jma_version__ = [8, 8, 0]

pinexq_client/job_management/enterjma.py

@@ -1,6 +1,9 @@
 from typing import TypeVar, Type
 
 import httpx
+from httpx import Client, _config
+# from hishel.httpx import SyncCacheClient
+from httpx_caching import CachingClient
 
 import warnings
 
@@ -17,6 +20,42 @@ THco = TypeVar("THco", bound=Hco)
 def _version_match_major_minor(ver1: list[int], ver2: list[int]) -> bool:
     return all([v1 == v2 for v1, v2 in zip(ver1[:2], ver2[:2])])
 
+def create_pinexq_client(
+        pinexq_api_endpoint: str,
+        api_key:str,
+        timeout: _config.TimeoutTypes = _config.DEFAULT_TIMEOUT_CONFIG,
+        use_client_cache: bool = True) -> httpx.Client:
+    """
+    Will create a httpx client, optional with caching, to be used with the API objects.
+
+    Args:
+        pinexq_api_endpoint: The endpoint for the pinexq API.
+        api_key: The API key for the pinexq API.
+        timeout: The timeout passed to the http client for operations.
+        use_client_cache: Whether to use a httpx client with caching.
+
+    """
+
+    headers = { 'x-api-key': api_key }
+    if use_client_cache:
+        # for now, we use the persistent cache, which is also shared between instances
+        # use  if you need each client to have a own cache storage=InMemoryStorage()
+        # broken, will cache SSE stream
+        #return SyncCacheClient(
+        #    base_url=pinexq_api_endpoint,
+        #    headers=headers,
+        #    timeout=timeout)
+        client = Client(
+            base_url=pinexq_api_endpoint,
+            headers=headers,
+            timeout=timeout)
+        return CachingClient(client)
+    else:
+        return Client(
+            base_url=pinexq_api_endpoint,
+            headers=headers,
+            timeout=timeout)
+
 
 def enter_jma(
     client: httpx.Client,
@@ -24,6 +63,16 @@ def enter_jma(
     entrypoint_entity_type: Type[Entity] = EntryPointEntity,
     entrypoint: str = "api/EntryPoint",
 ) -> EntryPointHco:
+    """
+    Gets the entrypoint object for the pinexq API. Will use the configured base url from the client.
+    Will check the current API version against the version of the client to ensure compatibility.
+
+    Args:
+        client: The configured pinexq client.
+        entrypoint_hco_type: The type to represent the api resource of the  entrypoint object [optional].
+        entrypoint_entity_type: The type of the entrypoint object containing typed properties [optional].
+        entrypoint: The path segment leading to the api entrypoint resource [optional].
+    """
     entry_point_hco = enter_api(client, entrypoint_hco_type, entrypoint_entity_type, entrypoint)
 
     info = entry_point_hco.info_link.navigate()

pinexq_client/job_management/hcos/entrypoint_hco.py

@@ -15,44 +15,44 @@ from pinexq_client.job_management.model.sirenentities import EntryPointEntity
 
 
 class EntryPointLink(LinkHco):
-	def navigate(self) -> 'EntryPointHco':
-		return EntryPointHco.from_entity(self._navigate_internal(EntryPointEntity), self._client)
+    def navigate(self) -> 'EntryPointHco':
+        return EntryPointHco.from_entity(self._navigate_internal(EntryPointEntity), self._client)
 
 
 class EntrypointRelations(StrEnum):
-	JOBS_ROOT = "JobsRoot"
-	WORKDATA_ROOT = "WorkDataRoot"
-	PROCESSINGSTEPS_ROOT = "ProcessingStepsRoot"
-	INFO = "Info"
-	ADMIN = "Admin"
+    JOBS_ROOT = "JobsRoot"
+    WORKDATA_ROOT = "WorkDataRoot"
+    PROCESSINGSTEPS_ROOT = "ProcessingStepsRoot"
+    API_EVENTS_ROOT = "ApiEvents"
+    INFO = "Info"
+    ADMIN = "Admin"
 
 
 class EntryPointHco(Hco[EntryPointEntity]):
-	self_link: EntryPointLink
-	job_root_link: JobsRootLink
-	work_data_root_link: WorkDataRootLink
-	processing_step_root_link: ProcessingStepsRootLink
-	info_link: InfoLink
-
-	admin_link: LinkHco | UnavailableLink
-
-	@classmethod
-	def from_entity(cls, entity: EntryPointEntity, client: httpx.Client) -> Self:
-		instance = cls(client, entity)
-		Hco.check_classes(instance._entity.class_, ["EntryPoint"])
-
-		instance.self_link = EntryPointLink.from_entity(
-			instance._client, instance._entity, Relations.SELF)
-		instance.info_link = InfoLink.from_entity(
-			instance._client, instance._entity, EntrypointRelations.INFO)
-		instance.job_root_link = JobsRootLink.from_entity(
-			instance._client, instance._entity, EntrypointRelations.JOBS_ROOT)
-		instance.work_data_root_link = WorkDataRootLink.from_entity(
-			instance._client, instance._entity, EntrypointRelations.WORKDATA_ROOT)
-		instance.processing_step_root_link = ProcessingStepsRootLink.from_entity(
-			instance._client, instance._entity, EntrypointRelations.PROCESSINGSTEPS_ROOT)
-
-		instance.admin_link = LinkHco.from_entity_optional(
-			instance._client, instance._entity, EntrypointRelations.ADMIN)
-
-		return instance
+    self_link: EntryPointLink
+    job_root_link: JobsRootLink
+    work_data_root_link: WorkDataRootLink
+    processing_step_root_link: ProcessingStepsRootLink
+    info_link: InfoLink
+    admin_link: LinkHco | UnavailableLink
+
+    @classmethod
+    def from_entity(cls, entity: EntryPointEntity, client: httpx.Client) -> Self:
+        instance = cls(client, entity)
+        Hco.check_classes(instance._entity.class_, ["EntryPoint"])
+
+        instance.self_link = EntryPointLink.from_entity(
+            instance._client, instance._entity, Relations.SELF)
+        instance.info_link = InfoLink.from_entity(
+            instance._client, instance._entity, EntrypointRelations.INFO)
+        instance.job_root_link = JobsRootLink.from_entity(
+            instance._client, instance._entity, EntrypointRelations.JOBS_ROOT)
+        instance.work_data_root_link = WorkDataRootLink.from_entity(
+            instance._client, instance._entity, EntrypointRelations.WORKDATA_ROOT)
+        instance.processing_step_root_link = ProcessingStepsRootLink.from_entity(
+            instance._client, instance._entity, EntrypointRelations.PROCESSINGSTEPS_ROOT)
+
+        instance.admin_link = LinkHco.from_entity_optional(
+            instance._client, instance._entity, EntrypointRelations.ADMIN)
+
+        return instance

pinexq_client/job_management/hcos/info_hco.py

@@ -13,6 +13,14 @@ class InfoLink(LinkHco):
     def navigate(self) -> "InfoHco":
         return InfoHco.from_entity(self._navigate_internal(InfoEntity), self._client)
 
+class ApiEventsEndpointLink(LinkHco):
+    pass
+
+class DeploymentRegistryEndpointLink(LinkHco):
+    pass
+
+class RemoteEndpointLink(LinkHco):
+    pass
 
 class InfoHco(Hco[InfoEntity]):
     api_version: str = Property()
@@ -22,6 +30,9 @@ class InfoHco(Hco[InfoEntity]):
     used_storage_in_bytes: int = Property()
 
     self_link: InfoLink
+    api_events_endpoint: ApiEventsEndpointLink
+    deployment_registry_endpoint: DeploymentRegistryEndpointLink
+    remote_endpoint: RemoteEndpointLink
 
     @classmethod
     def from_entity(cls, entity: InfoEntity, client: httpx.Client) -> Self:
@@ -33,6 +44,10 @@ class InfoHco(Hco[InfoEntity]):
             instance._client, instance._entity, Relations.SELF
         )
 
+        instance.api_events_endpoint = ApiEventsEndpointLink.from_entity(instance._client, instance._entity, Relations.API_EVENTS_ENDPOINT)
+        instance.deployment_registry_endpoint = DeploymentRegistryEndpointLink.from_entity_optional(instance._client, instance._entity, Relations.DEPLOYMENT_REGISTRY_ENDPOINT)
+        instance.remote_endpoint = RemoteEndpointLink.from_entity_optional(instance._client, instance._entity, Relations.REMOTE_ENDPOINT)
+
         instance._extract_current_user()
 
         return instance

pinexq_client/job_management/hcos/input_dataslot_hco.py

@@ -45,11 +45,13 @@ class InputDataSlotClearDataAction(ActionHco):
 
 
 class InputDataSlotHco(Hco[InputDataSlotEntity]):
-	is_configured: bool | None = Property()
+
 	title: str | None = Property()
 	description: str | None = Property()
+	name: str | None = Property()
 	media_type: str | None = Property()
 	selected_workdatas: list[WorkDataHco]
+	is_configured: bool | None = Property()
 
 	select_workdata_action: InputDataSlotSelectWorkDataAction | UnavailableAction
 	select_workdata_collection_action: InputDataSlotSelectWorkDataCollectionAction | UnavailableAction

pinexq_client/job_management/hcos/output_dataslot_hco.py

@@ -20,6 +20,7 @@ class OutputDataSlotLink(LinkHco):
 class OutputDataSlotHco(Hco[OutputDataSlotEntity]):
     title: str | None = Property()
     description: str | None = Property()
+    name: str | None = Property()
     media_type: str | None = Property()
     assigned_workdatas: list[WorkDataHco]
 

pinexq_client/job_management/hcos/processing_step_hco.py

@@ -14,9 +14,11 @@ from pinexq_client.core.hco.unavailable import UnavailableAction
 from pinexq_client.core.hco.upload_action_hco import UploadAction, UploadParameters
 from pinexq_client.job_management.known_relations import Relations
 from pinexq_client.job_management.model import CopyPsFromUserToOrgActionParameters, CopyPsFromOrgToUserActionParameters, \
-    DeprecatePsActionParameters
-from pinexq_client.job_management.model.open_api_generated import DataSpecificationHto, \
-    SetProcessingStepTagsParameters, EditProcessingStepParameters
+    DeprecatePsActionParameters, AssignCodeHashParameters
+from pinexq_client.job_management.model.open_api_generated import (DataSpecificationHto,
+                                                                   SetProcessingStepTagsParameters,
+                                                                   EditProcessingStepParameters,
+                                                                   ConfigureDeploymentParameters, DeploymentStates, ProcessingStepDeploymentHto)
 from pinexq_client.job_management.model.sirenentities import ProcessingStepEntity
 
 
@@ -81,6 +83,32 @@ class RestoreAction(ActionHco):
     def execute(self):
         self._execute_internal()
 
+
+class ConfigureExternalDeploymentAction(ActionHco):
+    def execute(self):
+        self._execute_internal()
+
+
+class RemoveDeploymentAction(ActionHco):
+    def execute(self):
+        self._execute_internal()
+
+
+class SuspendDeploymentAction(ActionHco):
+    def execute(self):
+        self._execute_internal()
+
+
+class ResumeDeploymentAction(ActionHco):
+    def execute(self):
+        self._execute_internal()
+
+
+class ClearCodeHashAction(ActionHco):
+    def execute(self):
+        self._execute_internal()
+
+
 class UploadConfigurationAction(UploadAction):
     def execute(self, parameters: UploadParameters):
         upload_json(self._client, self._action, parameters.json_, parameters.filename)
@@ -117,6 +145,24 @@ class ProcessingStepDeprecateAction(ActionWithParametersHco[DeprecatePsActionPar
                                             DeprecatePsActionParameters())
 
 
+class ConfigureDeploymentAction(ActionWithParametersHco[ConfigureDeploymentParameters]):
+    def execute(self, parameters: ConfigureDeploymentParameters):
+        self._execute(parameters)
+
+    def default_parameters(self) -> ConfigureDeploymentParameters:
+        return self._get_default_parameters(ConfigureDeploymentParameters,
+                                            ConfigureDeploymentParameters())
+
+
+class AssignCodeHashAction(ActionWithParametersHco[AssignCodeHashParameters]):
+    def execute(self, parameters: AssignCodeHashParameters):
+        self._execute(parameters)
+
+    def default_parameters(self) -> AssignCodeHashParameters:
+        return self._get_default_parameters(AssignCodeHashParameters,
+                                            AssignCodeHashParameters())
+
+
 class ProcessingStepHco(Hco[ProcessingStepEntity]):
     title: str = Property()
     version: str | None = Property()
@@ -134,8 +180,12 @@ class ProcessingStepHco(Hco[ProcessingStepEntity]):
     return_schema: str | None = Property()
     error_schema: str | None = Property()
     hidden: bool | None = Property()
-    deprecated_on: datetime | None = Property()
+
+    deprecated_at: datetime | None = Property()
     reason_for_deprecation: str | None = Property()
+    is_deprecated: bool | None = Property()
+    deployment_state: DeploymentStates = Property()
+    deployment: ProcessingStepDeploymentHto = Property()
 
     input_data_slot_specification: List[DataSpecificationHto] | None = Property()
     output_data_slot_specification: List[DataSpecificationHto] | None = Property()
@@ -151,6 +201,13 @@ class ProcessingStepHco(Hco[ProcessingStepEntity]):
     delete_action: DeleteAction | UnavailableAction
     deprecate_ps_action: ProcessingStepDeprecateAction | UnavailableAction
     restore_ps_action: RestoreAction | UnavailableAction
+    assign_code_hash_action: AssignCodeHashAction | UnavailableAction
+    configure_deployment_action: ConfigureDeploymentAction | UnavailableAction
+    configure_external_deployment_action: ConfigureExternalDeploymentAction | UnavailableAction
+    remove_deployment_action: RemoveDeploymentAction | UnavailableAction
+    suspend_deployment_action: SuspendDeploymentAction | UnavailableAction
+    resume_deployment_action: ResumeDeploymentAction | UnavailableAction
+    clear_code_hash_action: ClearCodeHashAction | UnavailableAction
 
     self_link: ProcessingStepLink
     download_link: DownloadLinkHco
@@ -189,5 +246,19 @@ class ProcessingStepHco(Hco[ProcessingStepEntity]):
             client, instance._entity, "Deprecate")
         instance.restore_ps_action = RestoreAction.from_entity_optional(
             client, instance._entity, "Restore")
+        instance.assign_code_hash_action = AssignCodeHashAction.from_entity_optional(
+            client, instance._entity, "AssignCodeHash")
+        instance.configure_deployment_action = ConfigureDeploymentAction.from_entity_optional(
+            client, instance._entity, "ConfigureDeployment")
+        instance.configure_external_deployment_action = ConfigureExternalDeploymentAction.from_entity_optional(
+            client, instance._entity, "ConfigureExternalDeployment")
+        instance.remove_deployment_action = RemoveDeploymentAction.from_entity_optional(
+            client, instance._entity, "RemoveDeployment")
+        instance.suspend_deployment_action = SuspendDeploymentAction.from_entity_optional(
+            client, instance._entity, "SuspendDeployment")
+        instance.resume_deployment_action = ResumeDeploymentAction.from_entity_optional(
+            client, instance._entity, "ResumeDeployment")
+        instance.clear_code_hash_action = ClearCodeHashAction.from_entity_optional(
+            client, instance._entity, "ClearCodeHash")
 
         return instance

pinexq_client/job_management/known_relations.py

@@ -32,3 +32,10 @@ class Relations(StrEnum):
 
     # info
     CURRENT_USER = "CurrentUser"
+
+    # Api Events
+    API_EVENTS_ENDPOINT = "ApiEventsEndpoint"
+
+    # Deployment
+    DEPLOYMENT_REGISTRY_ENDPOINT = "Registry"
+    REMOTE_ENDPOINT = "Remote"