dagster-k8s 0.24.6__py3-none-any.whl → 0.24.8__py3-none-any.whl

dagster_k8s/client.py

@@ -525,21 +525,25 @@ class DagsterKubernetesClient:
 
     def wait_for_pod(
         self,
-        pod_name,
-        namespace,
-        wait_for_state=WaitForPodState.Ready,
-        wait_timeout=DEFAULT_WAIT_TIMEOUT,
-        wait_time_between_attempts=DEFAULT_WAIT_BETWEEN_ATTEMPTS,
-        start_time=None,
+        pod_name: str,
+        namespace: str,
+        wait_for_state: WaitForPodState = WaitForPodState.Ready,
+        pod_launch_timeout: float = DEFAULT_WAIT_TIMEOUT,
+        wait_timeout: float = DEFAULT_WAIT_TIMEOUT,
+        wait_time_between_attempts: float = DEFAULT_WAIT_BETWEEN_ATTEMPTS,
+        start_time: Any = None,
         ignore_containers: Optional[Set] = None,
-    ):
+    ) -> None:
         """Wait for a pod to launch and be running, or wait for termination (useful for job pods).
 
         Args:
+        ----
             pod_name (str): Name of the pod to wait for.
             namespace (str): Namespace in which the pod is located.
             wait_for_state (WaitForPodState, optional): Whether to wait for pod readiness or
                 termination. Defaults to waiting for readiness.
+            pod_launch_timeout (numeric, optional): Timeout after which to give up and raise exception
+                if the pod never appears. Defaults to DEFAULT_WAIT_TIMEOUT. Set to 0 to disable.
             wait_timeout (numeric, optional): Timeout after which to give up and raise exception.
                 Defaults to DEFAULT_WAIT_TIMEOUT. Set to 0 to disable.
             wait_time_between_attempts (numeric, optional): Wait time between polling attempts. Defaults
@@ -549,36 +553,40 @@ class DagsterKubernetesClient:
                 when waiting for the pod to be ready/terminate.
 
         Raises:
+        ------
             DagsterK8sError: Raised when wait_timeout is exceeded or an error is encountered
+
         """
         check.str_param(pod_name, "pod_name")
         check.str_param(namespace, "namespace")
         check.inst_param(wait_for_state, "wait_for_state", WaitForPodState)
         check.numeric_param(wait_timeout, "wait_timeout")
+        check.numeric_param(pod_launch_timeout, "pod_launch_timeout")
         check.numeric_param(wait_time_between_attempts, "wait_time_between_attempts")
 
-        self.logger('Waiting for pod "%s"' % pod_name)
+        self.logger(f'Waiting for pod "{pod_name}"')
 
         start = start_time or self.timer()
 
         # A set of container names that have exited.
         exited_containers = set()
+        ready_containers = set()
         ignore_containers = ignore_containers or set()
         error_logs = []
 
         while True:
             pods = self.core_api.list_namespaced_pod(
-                namespace=namespace, field_selector="metadata.name=%s" % pod_name
+                namespace=namespace, field_selector=f"metadata.name={pod_name}"
             ).items
             pod = pods[0] if pods else None
 
-            if wait_timeout and self.timer() - start > wait_timeout:
+            if pod_launch_timeout and self.timer() - start > pod_launch_timeout:
                 raise DagsterK8sError(
-                    "Timed out while waiting for pod to become ready with pod info: %s" % str(pod)
+                    f"Timed out while waiting for pod to become ready with pod info: {pod!s}"
                 )
 
             if pod is None:
-                self.logger('Waiting for pod "%s" to launch...' % pod_name)
+                self.logger(f'Waiting for pod "{pod_name}" to launch...')
                 self.sleeper(wait_time_between_attempts)
                 continue
 
@@ -588,11 +596,25 @@ class DagsterKubernetesClient:
                 )
                 self.sleeper(wait_time_between_attempts)
                 continue
+            break
 
+        while True:
+            pods = self.core_api.list_namespaced_pod(
+                namespace=namespace, field_selector=f"metadata.name={pod_name}"
+            ).items
+            pod = pods[0] if pods else None
+            if pod is None:
+                raise DagsterK8sError(f'Pod "{pod_name}" was unexpectedly killed')
+
+            if wait_timeout and self.timer() - start > wait_timeout:
+                raise DagsterK8sError(
+                    f"Timed out while waiting for pod to get to status {wait_for_state.value} with pod info: {pod!s}"
+                )
             # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#containerstatus-v1-core
             all_statuses = []
             all_statuses.extend(pod.status.init_container_statuses or [])
             all_statuses.extend(pod.status.container_statuses or [])
+            initcontainers = set(s.name for s in (pod.status.init_container_statuses or []))
 
             # Filter out ignored containers
             all_statuses = [s for s in all_statuses if s.name not in ignore_containers]
@@ -603,7 +625,9 @@ class DagsterKubernetesClient:
             #
             # In case we are waiting for the pod to be ready, we will exit after
             # the first container in this list is ready.
-            container_status = next(s for s in all_statuses if s.name not in exited_containers)
+            container_status = next(
+                s for s in all_statuses if s.name not in exited_containers | ready_containers
+            )
 
             # State checks below, see:
             # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#containerstate-v1-core
@@ -614,12 +638,15 @@ class DagsterKubernetesClient:
                     # ready is boolean field of container status
                     ready = container_status.ready
                     if not ready:
-                        self.logger('Waiting for pod "%s" to become ready...' % pod_name)
+                        self.logger(f'Waiting for pod "{pod_name}" to become ready...')
                         self.sleeper(wait_time_between_attempts)
                         continue
                     else:
-                        self.logger('Pod "%s" is ready, done waiting' % pod_name)
-                        break
+                        ready_containers.add(container_status.name)
+                        if initcontainers.issubset(exited_containers | ready_containers):
+                            self.logger(f'Pod "{pod_name}" is ready, done waiting')
+                            break
+
                 else:
                     check.invariant(
                         wait_for_state == WaitForPodState.Terminated, "New invalid WaitForPodState"
@@ -630,13 +657,13 @@ class DagsterKubernetesClient:
             elif state.waiting is not None:
                 # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#containerstatewaiting-v1-core
                 if state.waiting.reason == KubernetesWaitingReasons.PodInitializing:
-                    self.logger('Waiting for pod "%s" to initialize...' % pod_name)
+                    self.logger(f'Waiting for pod "{pod_name}" to initialize...')
                     self.sleeper(wait_time_between_attempts)
                     continue
                 if state.waiting.reason == KubernetesWaitingReasons.CreateContainerConfigError:
                     self.logger(
-                        'Pod "%s" is waiting due to a CreateContainerConfigError with message "%s"'
-                        " - trying again to see if it recovers" % (pod_name, state.waiting.message)
+                        f'Pod "{pod_name}" is waiting due to a CreateContainerConfigError with message "{state.waiting.message}"'
+                        " - trying again to see if it recovers"
                     )
                     self.sleeper(wait_time_between_attempts)
                     continue
@@ -662,7 +689,7 @@ class DagsterKubernetesClient:
                         f' Message="{state.waiting.message}"\n{debug_info}'
                     )
                 else:
-                    raise DagsterK8sError("Unknown issue: %s" % state.waiting)
+                    raise DagsterK8sError(f"Unknown issue: {state.waiting}")
 
             # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#containerstateterminated-v1-core
             elif state.terminated is not None:
@@ -679,6 +706,10 @@ class DagsterKubernetesClient:
 
                     self.logger(msg)
                     error_logs.append(msg)
+                elif container_name in initcontainers:
+                    self.logger(
+                        f"Init container {container_name} in {pod_name} has exited successfully"
+                    )
                 else:
                     self.logger(f"Container {container_name} in {pod_name} has exited successfully")
 

dagster_k8s/version.py

@@ -1 +1 @@
-__version__ = "0.24.6"
+__version__ = "0.24.8"

{dagster_k8s-0.24.6.dist-info → dagster_k8s-0.24.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.24.6
+Version: 0.24.8
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs
@@ -15,7 +15,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.8.6
+Requires-Dist: dagster ==1.8.8
 Requires-Dist: kubernetes
 Requires-Dist: google-auth !=2.23.1
 

{dagster_k8s-0.24.6.dist-info → dagster_k8s-0.24.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dagster_k8s/__init__.py,sha256=7LyrMxxhXKdGQYClq7OJwMiVJW0KYPP-8lTDrWLwyzU,750
-dagster_k8s/client.py,sha256=BpQ987_Xd0UtadEFEjgFvKnB3xIxODvxZKU3Scjvx4o,36326
+dagster_k8s/client.py,sha256=9stIMp196v-vaVct8kBFp5pveo0lAup78y9Gi7g9VMM,37873
 dagster_k8s/container_context.py,sha256=gyFS0FgdupEUHIIsfYzS-TKHzXOd2mx-yX7onw2XuiY,21353
 dagster_k8s/executor.py,sha256=IZMWEtzXXd37INbU4gxkMU46cX4qoZw4MdgszxngZuQ,13934
 dagster_k8s/job.py,sha256=2tCIYlGUIt3ZydbZ9WRyPgr7YWz8t_2vaD20u7h5efw,41512
@@ -9,11 +9,11 @@ dagster_k8s/pipes.py,sha256=lsf657zbvxAH71zjEOIc-Oxo2oRvN-mp2XBjbahPUyU,17195
 dagster_k8s/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
 dagster_k8s/test.py,sha256=cNtcbzxytiZtd01wY5ip7KPi01y0BUQuQhohoIfAFUM,684
 dagster_k8s/utils.py,sha256=c1bHqh5f1p5RZ0JCT6WEbPPjDvbgUp3pl4nYZRaaI4s,786
-dagster_k8s/version.py,sha256=1LERjg6TTlEZCKHKtH82Ox1j8xI8WMA94hEjeTPs1Io,23
+dagster_k8s/version.py,sha256=O9QkGk46Yh2lp8IBRcbEaStijlXe7yZyxRbauvT7w1Q,23
 dagster_k8s/ops/__init__.py,sha256=ur-9GrE_DRfnsFCpYan03qOY9cWbjagC8KHZFZuiCmc,113
 dagster_k8s/ops/k8s_job_op.py,sha256=cIrIn30Ew1MQDVJOf4A4LsERG60m64Vp9J4q90enebw,20533
-dagster_k8s-0.24.6.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_k8s-0.24.6.dist-info/METADATA,sha256=0s5JP2JukvC7jj-MLXWvqADRTtyk-GJXteGefot-NJM,778
-dagster_k8s-0.24.6.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_k8s-0.24.6.dist-info/top_level.txt,sha256=wFPjskoWPlk2hOLugYCaoZhSiZdUcbCA1QZe9I4dals,12
-dagster_k8s-0.24.6.dist-info/RECORD,,
+dagster_k8s-0.24.8.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_k8s-0.24.8.dist-info/METADATA,sha256=WcRObvjp4C5A7TJSHS4ovVTp-q9Hjo0IQO_mGM0evIA,778
+dagster_k8s-0.24.8.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+dagster_k8s-0.24.8.dist-info/top_level.txt,sha256=wFPjskoWPlk2hOLugYCaoZhSiZdUcbCA1QZe9I4dals,12
+dagster_k8s-0.24.8.dist-info/RECORD,,