dagster-k8s 0.26.12__tar.gz → 0.26.14__tar.gz

{dagster-k8s-0.26.12/dagster_k8s.egg-info → dagster_k8s-0.26.14}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-k8s
-Version: 0.26.12
+Version: 0.26.14
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs
@@ -14,3 +14,15 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
+Requires-Dist: dagster==1.10.14
+Requires-Dist: kubernetes<33
+Requires-Dist: google-auth!=2.23.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{dagster-k8s-0.26.12 → dagster_k8s-0.26.14}/dagster_k8s/client.py

@@ -8,6 +8,7 @@ from typing import Any, Callable, Optional, TypeVar
 import kubernetes.client
 import kubernetes.client.rest
 import six
+import urllib3.exceptions
 from dagster import (
     DagsterInstance,
     _check as check,
@@ -23,12 +24,14 @@ try:
 except ImportError:
     K8S_EVENTS_API_PRESENT = False
 
+logger = logging.getLogger(__name__)
 
 T = TypeVar("T")
 
 DEFAULT_WAIT_TIMEOUT = 86400.0  # 1 day
 DEFAULT_WAIT_BETWEEN_ATTEMPTS = 10.0  # 10 seconds
 DEFAULT_JOB_POD_COUNT = 1  # expect job:pod to be 1:1 by default
+DEFAULT_JOB_CREATION_TIMEOUT = 10.0  # 10 seconds
 
 
 class WaitForPodState(Enum):
@@ -223,6 +226,20 @@ def k8s_api_retry_creation_mutation(
                     k8s_api_exception=e,
                     original_exc_info=sys.exc_info(),
                 ) from e
+        except urllib3.exceptions.HTTPError as e:
+            # Temporary for recovery detection
+            logger.error(
+                f"k8s_api_retry_creation_mutation: {e.__module__}.{e.__class__.__name__}: {e!s}"
+            )
+            if remaining_attempts > 0:
+                time.sleep(timeout)
+            else:
+                raise DagsterK8sAPIRetryLimitExceeded(
+                    msg_fn(),
+                    k8s_api_exception=e,
+                    max_retries=max_retries,
+                    original_exc_info=sys.exc_info(),
+                ) from e
     check.failed("Unreachable.")
 
 
@@ -1015,7 +1032,9 @@ class DagsterKubernetesClient:
         wait_time_between_attempts: float = DEFAULT_WAIT_BETWEEN_ATTEMPTS,
     ) -> None:
         k8s_api_retry_creation_mutation(
-            lambda: self.batch_api.create_namespaced_job(body=body, namespace=namespace),
+            lambda: self.batch_api.create_namespaced_job(
+                body=body, namespace=namespace, _request_timeout=DEFAULT_JOB_CREATION_TIMEOUT
+            ),
             max_retries=3,
             timeout=wait_time_between_attempts,
         )

dagster_k8s-0.26.14/dagster_k8s/component.py

@@ -0,0 +1,57 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from functools import cached_property
+from typing import Any, Optional, Union
+
+from dagster import AssetExecutionContext, AssetsDefinition, Definitions, multi_asset
+from dagster.components import Component, ComponentLoadContext, Resolvable, ResolvedAssetSpec
+
+from dagster_k8s.pipes import PipesK8sClient, build_pod_body
+
+
+@dataclass
+class PipesK8sComponent(Component, Resolvable):
+    """Component that creates assets backed by kubernetes pod execution via Dagster Pipes."""
+
+    name: str
+    assets: Sequence[ResolvedAssetSpec]
+    image: Optional[str] = None
+    command: Optional[Union[str, Sequence[str]]] = None
+    namespace: Optional[str] = None
+    env: Optional[Mapping[str, str]] = None
+    base_pod_meta: Optional[Mapping[str, Any]] = None
+    base_pod_spec: Optional[Mapping[str, Any]] = None
+
+    def __post_init__(self):
+        # validate that we can build a pod for the given args
+        # i.e. image or base_pod_self.image
+        build_pod_body(
+            pod_name=self.name,
+            image=self.image,
+            command=self.command,
+            env_vars=self.env or {},
+            base_pod_meta=self.base_pod_meta,
+            base_pod_spec=self.base_pod_spec,
+        )
+
+    @cached_property
+    def client(self):
+        return PipesK8sClient()
+
+    def build_defs(self, context: ComponentLoadContext):
+        return Definitions(assets=[self.build_asset()])
+
+    def build_asset(self) -> AssetsDefinition:
+        @multi_asset(name=self.name, specs=self.assets)
+        def _asset(context: AssetExecutionContext):
+            return self.client.run(
+                context=context,
+                image=self.image,
+                command=self.command,
+                namespace=self.namespace,
+                env=self.env,
+                base_pod_meta=self.base_pod_meta,
+                base_pod_spec=self.base_pod_spec,
+            ).get_results()
+
+        return _asset

{dagster-k8s-0.26.12 → dagster_k8s-0.26.14}/dagster_k8s/launcher.py

@@ -445,8 +445,8 @@ class K8sRunLauncher(RunLauncher, ConfigurableClass):
                 WorkerStatus.FAILED, "Run has not completed but K8s job has no active pods"
             )
 
-        if status.failed:
-            return CheckRunHealthResult(WorkerStatus.FAILED, "K8s job failed")
         if status.succeeded:
             return CheckRunHealthResult(WorkerStatus.SUCCESS)
+        if status.failed and not status.active:
+            return CheckRunHealthResult(WorkerStatus.FAILED, "K8s job failed")
         return CheckRunHealthResult(WorkerStatus.RUNNING)

dagster_k8s-0.26.14/dagster_k8s/version.py

@@ -0,0 +1 @@
+__version__ = "0.26.14"

{dagster-k8s-0.26.12 → dagster_k8s-0.26.14/dagster_k8s.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-k8s
-Version: 0.26.12
+Version: 0.26.14
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs
@@ -14,3 +14,15 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
+Requires-Dist: dagster==1.10.14
+Requires-Dist: kubernetes<33
+Requires-Dist: google-auth!=2.23.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{dagster-k8s-0.26.12 → dagster_k8s-0.26.14}/dagster_k8s.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ setup.cfg
 setup.py
 dagster_k8s/__init__.py
 dagster_k8s/client.py
+dagster_k8s/component.py
 dagster_k8s/container_context.py
 dagster_k8s/executor.py
 dagster_k8s/job.py

{dagster-k8s-0.26.12 → dagster_k8s-0.26.14}/dagster_k8s.egg-info/requires.txt

@@ -1,3 +1,3 @@
-dagster==1.10.12
+dagster==1.10.14
 kubernetes<33
 google-auth!=2.23.1

{dagster-k8s-0.26.12 → dagster_k8s-0.26.14}/setup.py

@@ -42,7 +42,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.13",
     install_requires=[
-        "dagster==1.10.12",
+        "dagster==1.10.14",
         f"kubernetes<{KUBERNETES_VERSION_UPPER_BOUND}",
         # exclude a google-auth release that added an overly restrictive urllib3 pin that confuses dependency resolvers
         "google-auth!=2.23.1",

dagster-k8s-0.26.12/dagster_k8s/version.py

@@ -1 +0,0 @@
-__version__ = "0.26.12"