dagster-k8s 0.26.11rc1__py3-none-any.whl → 0.26.13__py3-none-any.whl

dagster_k8s/client.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import sys
 import time
 from enum import Enum
@@ -7,6 +8,7 @@ from typing import Any, Callable, Optional, TypeVar
 import kubernetes.client
 import kubernetes.client.rest
 import six
+import urllib3.exceptions
 from dagster import (
     DagsterInstance,
     _check as check,
@@ -22,12 +24,14 @@ try:
 except ImportError:
     K8S_EVENTS_API_PRESENT = False
 
+logger = logging.getLogger(__name__)
 
 T = TypeVar("T")
 
 DEFAULT_WAIT_TIMEOUT = 86400.0  # 1 day
 DEFAULT_WAIT_BETWEEN_ATTEMPTS = 10.0  # 10 seconds
 DEFAULT_JOB_POD_COUNT = 1  # expect job:pod to be 1:1 by default
+DEFAULT_JOB_CREATION_TIMEOUT = 10.0  # 10 seconds
 
 
 class WaitForPodState(Enum):
@@ -222,6 +226,20 @@ def k8s_api_retry_creation_mutation(
                     k8s_api_exception=e,
                     original_exc_info=sys.exc_info(),
                 ) from e
+        except urllib3.exceptions.HTTPError as e:
+            # Temporary for recovery detection
+            logger.error(
+                f"k8s_api_retry_creation_mutation: {e.__module__}.{e.__class__.__name__}: {e!s}"
+            )
+            if remaining_attempts > 0:
+                time.sleep(timeout)
+            else:
+                raise DagsterK8sAPIRetryLimitExceeded(
+                    msg_fn(),
+                    k8s_api_exception=e,
+                    max_retries=max_retries,
+                    original_exc_info=sys.exc_info(),
+                ) from e
     check.failed("Unreachable.")
 
 
@@ -740,13 +758,16 @@ class DagsterKubernetesClient:
             elif state.terminated is not None:
                 container_name = container_status.name
                 if state.terminated.exit_code != 0:
+                    tail_lines = int(
+                        os.getenv("DAGSTER_K8S_WAIT_FOR_POD_FAILURE_LOG_LINE_COUNT", "100")
+                    )
                     raw_logs = self.retrieve_pod_logs(
-                        pod_name, namespace, container_name=container_name
+                        pod_name, namespace, container_name=container_name, tail_lines=tail_lines
                     )
                     message = state.terminated.message
                     msg = (
-                        f'Container "{container_name}" failed with message: "{message}" '
-                        f'and pod logs: "{raw_logs}"'
+                        f'Container "{container_name}" failed with message: "{message}". '
+                        f'Last {tail_lines} log lines: "{raw_logs}"'
                     )
 
                     self.logger(msg)
@@ -1011,7 +1032,9 @@ class DagsterKubernetesClient:
         wait_time_between_attempts: float = DEFAULT_WAIT_BETWEEN_ATTEMPTS,
     ) -> None:
         k8s_api_retry_creation_mutation(
-            lambda: self.batch_api.create_namespaced_job(body=body, namespace=namespace),
+            lambda: self.batch_api.create_namespaced_job(
+                body=body, namespace=namespace, _request_timeout=DEFAULT_JOB_CREATION_TIMEOUT
+            ),
             max_retries=3,
             timeout=wait_time_between_attempts,
         )

dagster_k8s/launcher.py

@@ -445,8 +445,8 @@ class K8sRunLauncher(RunLauncher, ConfigurableClass):
                 WorkerStatus.FAILED, "Run has not completed but K8s job has no active pods"
             )
 
-        if status.failed:
-            return CheckRunHealthResult(WorkerStatus.FAILED, "K8s job failed")
         if status.succeeded:
             return CheckRunHealthResult(WorkerStatus.SUCCESS)
+        if status.failed and not status.active:
+            return CheckRunHealthResult(WorkerStatus.FAILED, "K8s job failed")
         return CheckRunHealthResult(WorkerStatus.RUNNING)

dagster_k8s/pipes.py

@@ -526,7 +526,7 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
                     pod_name=pod_name,
                     enable_multi_container_logs=enable_multi_container_logs,
                 ):
-                    # We need to wait for the pod to start up so that the log streaming is successful afterwards.
+                    # wait until the pod is fully terminated (or raise an exception if it failed)
                     client.wait_for_pod(
                         pod_name,
                         namespace,

dagster_k8s/version.py

@@ -1 +1 @@
-__version__ = "0.26.11rc1"
+__version__ = "0.26.13"

{dagster_k8s-0.26.11rc1.dist-info → dagster_k8s-0.26.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-k8s
-Version: 0.26.11rc1
+Version: 0.26.13
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs
@@ -14,7 +14,15 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.10.11rc1
-Requires-Dist: kubernetes <33
-Requires-Dist: google-auth !=2.23.1
-
+Requires-Dist: dagster==1.10.13
+Requires-Dist: kubernetes<33
+Requires-Dist: google-auth!=2.23.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{dagster_k8s-0.26.11rc1.dist-info → dagster_k8s-0.26.13.dist-info}/RECORD

@@ -1,20 +1,20 @@
 dagster_k8s/__init__.py,sha256=1Vi9HXEW9fatVuNzWTr6Oq-JEHmyDrVkonZuELsQ1C0,751
-dagster_k8s/client.py,sha256=jkTy4uAbFU8KkbpPDMUScNo8mNDZ6q9upe48R6fexQo,39933
+dagster_k8s/client.py,sha256=-Ns2saJri4Xxu-OpQ_T93e2K6aVnZvU19tj0iaMmTg8,40901
 dagster_k8s/container_context.py,sha256=399bdqmV3Ce6lNbfuxnhdyJSCy9keRRPJfVjDGn2U4I,22642
 dagster_k8s/executor.py,sha256=yOvA5KuGi9O5C3KGz6IP8HVPwt2CSD9NU7ITcWxe53M,14872
 dagster_k8s/job.py,sha256=v7RXYLZHzy2sTC7HYdz_INqbLWyZxUz9QIMwORArjJg,43065
 dagster_k8s/kubernetes_version.py,sha256=T7MDA5pOOVE_woRUkKWxtiRj4RCLAVfjSn6ZXUr6sAA,38
-dagster_k8s/launcher.py,sha256=8dPYFkB6W6GS5q7Ab9Mf4Oqyu3riX-HBLYyf7TKV7HQ,16669
+dagster_k8s/launcher.py,sha256=LlbXu3WYemmt_H7kOmbbKPNLnsQHszipQz_JmTMXdEA,16691
 dagster_k8s/models.py,sha256=REKzAY103pZw0pQFqvHaM6lL02hgbi0rxmm71qds4W8,6338
-dagster_k8s/pipes.py,sha256=KmCANVZIj6LkJ7b0jtCmQ043bI6CvjUQbZMe9xufs1c,34047
+dagster_k8s/pipes.py,sha256=kb1So6erUgjTDkicALpU1T5mG9xa9n8aeI4lPUVeU2o,34031
 dagster_k8s/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
 dagster_k8s/test.py,sha256=cNtcbzxytiZtd01wY5ip7KPi01y0BUQuQhohoIfAFUM,684
 dagster_k8s/utils.py,sha256=c1bHqh5f1p5RZ0JCT6WEbPPjDvbgUp3pl4nYZRaaI4s,786
-dagster_k8s/version.py,sha256=B5_KTP-FyikpTJttPb3-R24m53usTHHQpjdZOhW-ls0,27
+dagster_k8s/version.py,sha256=EDf5CcPxrnkRq2ViErwz2GB5gPrihrLuUdZnXlor9gM,24
 dagster_k8s/ops/__init__.py,sha256=ur-9GrE_DRfnsFCpYan03qOY9cWbjagC8KHZFZuiCmc,113
 dagster_k8s/ops/k8s_job_op.py,sha256=CQHtTowGGLAHH5EUGNfqZxDT1MYMqslAJ9u11SzCf-I,22036
-dagster_k8s-0.26.11rc1.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_k8s-0.26.11rc1.dist-info/METADATA,sha256=S3CbJu6azcMyNTf3GgFbnrtrLpjbJ3Dnnv-Qc8j6ZC8,741
-dagster_k8s-0.26.11rc1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_k8s-0.26.11rc1.dist-info/top_level.txt,sha256=wFPjskoWPlk2hOLugYCaoZhSiZdUcbCA1QZe9I4dals,12
-dagster_k8s-0.26.11rc1.dist-info/RECORD,,
+dagster_k8s-0.26.13.dist-info/licenses/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_k8s-0.26.13.dist-info/METADATA,sha256=pXbhkX0a9rboKfXe63XzpMA7gDwQ0giPW3kDffMbVQc,912
+dagster_k8s-0.26.13.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
+dagster_k8s-0.26.13.dist-info/top_level.txt,sha256=wFPjskoWPlk2hOLugYCaoZhSiZdUcbCA1QZe9I4dals,12
+dagster_k8s-0.26.13.dist-info/RECORD,,

{dagster_k8s-0.26.11rc1.dist-info → dagster_k8s-0.26.13.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: setuptools (80.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any