dagster-k8s 0.26.2__tar.gz → 0.28.12__tar.gz

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Dagster Labs, Inc".
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{dagster-k8s-0.26.2/dagster_k8s.egg-info → dagster_k8s-0.28.12}/PKG-INFO

@@ -1,16 +1,29 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-k8s
-Version: 0.26.2
+Version: 0.28.12
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.13
+Requires-Python: >=3.10,<3.15
 License-File: LICENSE
+Requires-Dist: dagster==1.12.12
+Requires-Dist: kubernetes<36
+Requires-Dist: google-auth!=2.23.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_k8s-0.28.12/README.md

@@ -0,0 +1,4 @@
+# dagster-k8s
+
+The docs for `dagster-k8s` can be found
+[here](https://docs.dagster.io/integrations/libraries/k8s/dagster-k8s).

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/__init__.py

@@ -1,4 +1,4 @@
-from dagster._core.libraries import DagsterLibraryRegistry
+from dagster_shared.libraries import DagsterLibraryRegistry
 
 from dagster_k8s.executor import k8s_job_executor as k8s_job_executor
 from dagster_k8s.job import (

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/client.py

@@ -1,12 +1,15 @@
 import logging
+import os
 import sys
 import time
+from collections.abc import Callable
 from enum import Enum
-from typing import Any, Callable, Optional, TypeVar
+from typing import Any, Optional, TypeVar
 
 import kubernetes.client
 import kubernetes.client.rest
 import six
+import urllib3.exceptions
 from dagster import (
     DagsterInstance,
     _check as check,
@@ -22,12 +25,14 @@ try:
 except ImportError:
     K8S_EVENTS_API_PRESENT = False
 
+logger = logging.getLogger(__name__)
 
 T = TypeVar("T")
 
 DEFAULT_WAIT_TIMEOUT = 86400.0  # 1 day
 DEFAULT_WAIT_BETWEEN_ATTEMPTS = 10.0  # 10 seconds
 DEFAULT_JOB_POD_COUNT = 1  # expect job:pod to be 1:1 by default
+DEFAULT_JOB_CREATION_TIMEOUT = 10.0  # 10 seconds
 
 
 class WaitForPodState(Enum):
@@ -222,6 +227,20 @@ def k8s_api_retry_creation_mutation(
                     k8s_api_exception=e,
                     original_exc_info=sys.exc_info(),
                 ) from e
+        except urllib3.exceptions.HTTPError as e:
+            # Temporary for recovery detection
+            logger.error(
+                f"k8s_api_retry_creation_mutation: {e.__module__}.{e.__class__.__name__}: {e!s}"
+            )
+            if remaining_attempts > 0:
+                time.sleep(timeout)
+            else:
+                raise DagsterK8sAPIRetryLimitExceeded(
+                    msg_fn(),
+                    k8s_api_exception=e,
+                    max_retries=max_retries,
+                    original_exc_info=sys.exc_info(),
+                ) from e
     check.failed("Unreachable.")
 
 
@@ -531,6 +550,18 @@ class DagsterKubernetesClient:
 
     ### Pod operations ###
 
+    def get_pod_by_name(self, pod_name: str, namespace: str):
+        """Get a pod by name.
+
+        Args:
+            pod_name (str): Name of the pod to get.
+            namespace (str): Namespace in which the pod is located.
+        """
+        check.str_param(pod_name, "pod_name")
+        check.str_param(namespace, "namespace")
+
+        return self.core_api.read_namespaced_pod(pod_name, namespace=namespace)
+
     def get_pods_in_job(self, job_name, namespace):
         """Get the pods launched by the job ``job_name``.
 
@@ -740,13 +771,16 @@ class DagsterKubernetesClient:
             elif state.terminated is not None:
                 container_name = container_status.name
                 if state.terminated.exit_code != 0:
+                    tail_lines = int(
+                        os.getenv("DAGSTER_K8S_WAIT_FOR_POD_FAILURE_LOG_LINE_COUNT", "100")
+                    )
                     raw_logs = self.retrieve_pod_logs(
-                        pod_name, namespace, container_name=container_name
+                        pod_name, namespace, container_name=container_name, tail_lines=tail_lines
                     )
                     message = state.terminated.message
                     msg = (
-                        f'Container "{container_name}" failed with message: "{message}" '
-                        f'and pod logs: "{raw_logs}"'
+                        f'Container "{container_name}" failed with message: "{message}". '
+                        f'Last {tail_lines} log lines: "{raw_logs}"'
                     )
 
                     self.logger(msg)
@@ -1011,7 +1045,9 @@ class DagsterKubernetesClient:
         wait_time_between_attempts: float = DEFAULT_WAIT_BETWEEN_ATTEMPTS,
     ) -> None:
         k8s_api_retry_creation_mutation(
-            lambda: self.batch_api.create_namespaced_job(body=body, namespace=namespace),
+            lambda: self.batch_api.create_namespaced_job(
+                body=body, namespace=namespace, _request_timeout=DEFAULT_JOB_CREATION_TIMEOUT
+            ),
             max_retries=3,
             timeout=wait_time_between_attempts,
         )

dagster_k8s-0.28.12/dagster_k8s/component.py

@@ -0,0 +1,65 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from functools import cached_property
+from typing import Any, Optional, Union
+
+from dagster import (
+    AssetExecutionContext,
+    AssetsDefinition,
+    Component,
+    ComponentLoadContext,
+    Definitions,
+    Resolvable,
+    ResolvedAssetSpec,
+    multi_asset,
+)
+
+from dagster_k8s.pipes import PipesK8sClient, build_pod_body
+
+
+@dataclass
+class PipesK8sComponent(Component, Resolvable):
+    """Component that creates assets backed by kubernetes pod execution via Dagster Pipes."""
+
+    name: str
+    assets: Sequence[ResolvedAssetSpec]
+    image: Optional[str] = None
+    command: Optional[Union[str, Sequence[str]]] = None
+    namespace: Optional[str] = None
+    env: Optional[Mapping[str, str]] = None
+    base_pod_meta: Optional[Mapping[str, Any]] = None
+    base_pod_spec: Optional[Mapping[str, Any]] = None
+
+    def __post_init__(self):
+        # validate that we can build a pod for the given args
+        # i.e. image or base_pod_self.image
+        build_pod_body(
+            pod_name=self.name,
+            image=self.image,
+            command=self.command,
+            env_vars=self.env or {},
+            base_pod_meta=self.base_pod_meta,
+            base_pod_spec=self.base_pod_spec,
+        )
+
+    @cached_property
+    def client(self):
+        return PipesK8sClient()
+
+    def build_defs(self, context: ComponentLoadContext):
+        return Definitions(assets=[self.build_asset()])
+
+    def build_asset(self) -> AssetsDefinition:
+        @multi_asset(name=self.name, specs=self.assets)
+        def _asset(context: AssetExecutionContext):
+            return self.client.run(
+                context=context,
+                image=self.image,
+                command=self.command,
+                namespace=self.namespace,
+                env=self.env,
+                base_pod_meta=self.base_pod_meta,
+                base_pod_spec=self.base_pod_spec,
+            ).get_results()
+
+        return _asset

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/container_context.py

@@ -11,7 +11,7 @@ from dagster._core.container_context import process_shared_container_context_con
 from dagster._core.errors import DagsterInvalidConfigError
 from dagster._core.storage.dagster_run import DagsterRun
 from dagster._core.utils import parse_env_var
-from dagster._utils import hash_collection
+from dagster_shared.utils.hash import hash_collection
 
 if TYPE_CHECKING:
     from dagster_k8s import K8sRunLauncher
@@ -528,7 +528,7 @@ class K8sContainerContext(
                 run_k8s_container_context,
             )
 
-        processed_context_value = cast(dict, processed_container_context.value)
+        processed_context_value = cast("dict", processed_container_context.value)
 
         return shared_container_context.merge(
             K8sContainerContext(

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/executor.py

@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import Optional, cast
 
@@ -15,6 +16,10 @@ from dagster._core.definitions.executor_definition import multiple_process_execu
 from dagster._core.definitions.metadata import MetadataValue
 from dagster._core.events import DagsterEvent, EngineEventData
 from dagster._core.execution.retries import RetryMode, get_retries_config
+from dagster._core.execution.step_dependency_config import (
+    StepDependencyConfig,
+    get_step_dependency_config_field,
+)
 from dagster._core.execution.tags import get_tag_concurrency_limits_config
 from dagster._core.executor.base import Executor
 from dagster._core.executor.init import InitExecutorContext
@@ -24,6 +29,7 @@ from dagster._core.executor.step_delegating import (
     StepHandler,
     StepHandlerContext,
 )
+from dagster._utils.cached_method import cached_method
 from dagster._utils.merger import merge_dicts
 
 from dagster_k8s.client import DagsterKubernetesClient
@@ -31,12 +37,14 @@ from dagster_k8s.container_context import K8sContainerContext
 from dagster_k8s.job import (
     USER_DEFINED_K8S_JOB_CONFIG_SCHEMA,
     DagsterK8sJobConfig,
+    OwnerReference,
     UserDefinedDagsterK8sConfig,
     construct_dagster_k8s_job,
     get_k8s_job_name,
     get_user_defined_k8s_config,
 )
 from dagster_k8s.launcher import K8sRunLauncher
+from dagster_k8s.utils import get_deployment_id_label
 
 _K8S_EXECUTOR_CONFIG_SCHEMA = merge_dicts(
     DagsterK8sJobConfig.config_type_job(),
@@ -81,6 +89,15 @@ _K8S_EXECUTOR_CONFIG_SCHEMA = merge_dicts(
             default_value={},
             description="Per op k8s configuration overrides.",
         ),
+        "enable_owner_references": Field(
+            bool,
+            is_required=False,
+            default_value=False,
+            description="Whether to insert Kubernetes owner references on step jobs to their parent run pod."
+            " This ensures that step jobs and step pods are garbage collected when the run pod is deleted."
+            " For more information, see https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/",
+        ),
+        "step_dependency_config": get_step_dependency_config_field(),
     },
 )
 
@@ -154,12 +171,12 @@ def k8s_job_executor(init_context: InitExecutorContext) -> Executor:
     )
 
     if "load_incluster_config" in exc_cfg:
-        load_incluster_config = cast(bool, exc_cfg["load_incluster_config"])
+        load_incluster_config = cast("bool", exc_cfg["load_incluster_config"])
     else:
         load_incluster_config = run_launcher.load_incluster_config if run_launcher else True
 
     if "kubeconfig_file" in exc_cfg:
-        kubeconfig_file = cast(Optional[str], exc_cfg["kubeconfig_file"])
+        kubeconfig_file = cast("Optional[str]", exc_cfg["kubeconfig_file"])
     else:
         kubeconfig_file = run_launcher.kubeconfig_file if run_launcher else None
 
@@ -170,11 +187,17 @@ def k8s_job_executor(init_context: InitExecutorContext) -> Executor:
             load_incluster_config=load_incluster_config,
             kubeconfig_file=kubeconfig_file,
             per_step_k8s_config=exc_cfg.get("per_step_k8s_config", {}),
+            enable_owner_references=check.opt_bool_param(
+                exc_cfg.get("enable_owner_references"), "enable_owner_references", False
+            ),
         ),
         retries=RetryMode.from_config(exc_cfg["retries"]),  # type: ignore
         max_concurrent=check.opt_int_elem(exc_cfg, "max_concurrent"),
         tag_concurrency_limits=check.opt_list_elem(exc_cfg, "tag_concurrency_limits"),
         should_verify_step=True,
+        step_dependency_config=StepDependencyConfig.from_config(
+            exc_cfg.get("step_dependency_config")  # type: ignore
+        ),
     )
 
 
@@ -190,7 +213,9 @@ class K8sStepHandler(StepHandler):
         load_incluster_config: bool,
         kubeconfig_file: Optional[str],
         k8s_client_batch_api=None,
+        k8s_client_core_api=None,
         per_step_k8s_config=None,
+        enable_owner_references=False,
     ):
         super().__init__()
 
@@ -198,7 +223,7 @@ class K8sStepHandler(StepHandler):
         self._executor_container_context = check.inst_param(
             container_context, "container_context", K8sContainerContext
         )
-
+        self._kubeconfig_file = None
         if load_incluster_config:
             check.invariant(
                 kubeconfig_file is None,
@@ -208,17 +233,20 @@ class K8sStepHandler(StepHandler):
         else:
             check.opt_str_param(kubeconfig_file, "kubeconfig_file")
             kubernetes.config.load_kube_config(kubeconfig_file)
+            self._kubeconfig_file = kubeconfig_file
 
         self._api_client = DagsterKubernetesClient.production_client(
-            batch_api_override=k8s_client_batch_api
+            batch_api_override=k8s_client_batch_api,
+            core_api_override=k8s_client_core_api,
         )
         self._per_step_k8s_config = check.opt_dict_param(
             per_step_k8s_config, "per_step_k8s_config", key_type=str, value_type=dict
         )
+        self._enable_owner_references = enable_owner_references
 
     def _get_step_key(self, step_handler_context: StepHandlerContext) -> str:
         step_keys_to_execute = cast(
-            list[str], step_handler_context.execute_step_args.step_keys_to_execute
+            "list[str]", step_handler_context.execute_step_args.step_keys_to_execute
         )
         assert len(step_keys_to_execute) == 1, "Launching multiple steps is not currently supported"
         return step_keys_to_execute[0]
@@ -230,7 +258,7 @@ class K8sStepHandler(StepHandler):
 
         context = K8sContainerContext.create_for_run(
             step_handler_context.dagster_run,
-            cast(K8sRunLauncher, step_handler_context.instance.run_launcher),
+            cast("K8sRunLauncher", step_handler_context.instance.run_launcher),
             include_run_tags=False,  # For now don't include job-level dagster-k8s/config tags in step pods
         )
         context = context.merge(self._executor_container_context)
@@ -263,6 +291,25 @@ class K8sStepHandler(StepHandler):
 
         return f"dagster-step-{name_key}"
 
+    @cached_method
+    def _detect_current_name_and_uid(
+        self,
+    ) -> Optional[tuple[str, str]]:
+        """Get the current pod's pod name and uid, if available."""
+        from dagster_k8s.utils import detect_current_namespace
+
+        hostname = os.getenv("HOSTNAME")
+        if not hostname:
+            return None
+
+        namespace = detect_current_namespace(self._kubeconfig_file)
+        if not namespace:
+            return None
+
+        pod = self._api_client.get_pod_by_name(pod_name=hostname, namespace=namespace)
+
+        return pod.metadata.name, pod.metadata.uid
+
     def launch_step(self, step_handler_context: StepHandlerContext) -> Iterator[DagsterEvent]:
         step_key = self._get_step_key(step_handler_context)
 
@@ -297,6 +344,16 @@ class K8sStepHandler(StepHandler):
             labels["dagster/code-location"] = (
                 run.remote_job_origin.repository_origin.code_location_origin.location_name
             )
+        deployment_name_env_var = get_deployment_id_label(container_context.run_k8s_config)
+        if deployment_name_env_var:
+            labels["dagster/deployment-name"] = deployment_name_env_var
+
+        owner_references = []
+        if self._enable_owner_references:
+            my_pod = self._detect_current_name_and_uid()
+            if my_pod:
+                owner_references = [OwnerReference(kind="Pod", name=my_pod[0], uid=my_pod[1])]
+
         job = construct_dagster_k8s_job(
             job_config=job_config,
             args=args,
@@ -313,6 +370,7 @@ class K8sStepHandler(StepHandler):
                 },
                 {"name": "DAGSTER_RUN_STEP_KEY", "value": step_key},
             ],
+            owner_references=owner_references,
         )
 
         yield DagsterEvent.step_worker_starting(

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/job.py

@@ -85,6 +85,20 @@ DEFAULT_JOB_SPEC_CONFIG = {
 }
 
 
+class OwnerReference(NamedTuple):
+    kind: str
+    name: str
+    uid: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "api_version": "batch/v1" if self.kind == "Job" else "v1",
+            "kind": self.kind,
+            "name": self.name,
+            "uid": self.uid,
+        }
+
+
 class UserDefinedDagsterK8sConfig(
     NamedTuple(
         "_UserDefinedDagsterK8sConfig",
@@ -754,6 +768,7 @@ def construct_dagster_k8s_job(
     component: Optional[str] = None,
     labels: Optional[Mapping[str, str]] = None,
     env_vars: Optional[Sequence[Mapping[str, Any]]] = None,
+    owner_references: Optional[Sequence[OwnerReference]] = None,
 ) -> kubernetes.client.V1Job:
     """Constructs a Kubernetes Job object.
 
@@ -929,6 +944,16 @@ def construct_dagster_k8s_job(
     user_defined_job_metadata = copy.deepcopy(dict(user_defined_k8s_config.job_metadata))
     user_defined_job_labels = user_defined_job_metadata.pop("labels", {})
 
+    owner_reference_dicts = (
+        [owner_reference.to_dict() for owner_reference in owner_references]
+        if owner_references
+        else []
+    )
+    if "owner_references" in user_defined_job_metadata:
+        user_defined_job_metadata["owner_references"] = (
+            owner_reference_dicts + user_defined_job_metadata["owner_references"]
+        )
+
     job = k8s_model_from_dict(
         kubernetes.client.V1Job,
         merge_dicts(
@@ -944,6 +969,7 @@ def construct_dagster_k8s_job(
                             dagster_labels, user_defined_job_labels, job_config.labels
                         ),
                     },
+                    {"owner_references": owner_reference_dicts} if owner_reference_dicts else {},
                 ),
                 "spec": job_spec_config,
             },

dagster_k8s-0.28.12/dagster_k8s/kubernetes_version.py

@@ -0,0 +1 @@
+KUBERNETES_VERSION_UPPER_BOUND = "36"

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/launcher.py

@@ -18,6 +18,7 @@ from dagster._utils.error import serializable_error_info_from_exc_info
 from dagster_k8s.client import DagsterKubernetesClient
 from dagster_k8s.container_context import K8sContainerContext
 from dagster_k8s.job import DagsterK8sJobConfig, construct_dagster_k8s_job, get_job_name_from_run_id
+from dagster_k8s.utils import get_deployment_id_label
 
 
 class K8sRunLauncher(RunLauncher, ConfigurableClass):
@@ -231,6 +232,9 @@ class K8sRunLauncher(RunLauncher, ConfigurableClass):
             "dagster/job": job_origin.job_name,
             "dagster/run-id": run.run_id,
         }
+        deployment_name_env_var = get_deployment_id_label(user_defined_k8s_config)
+        if deployment_name_env_var:
+            labels["dagster/deployment-name"] = deployment_name_env_var
         if run.remote_job_origin:
             labels["dagster/code-location"] = (
                 run.remote_job_origin.repository_origin.code_location_origin.location_name
@@ -319,7 +323,7 @@ class K8sRunLauncher(RunLauncher, ConfigurableClass):
             return None
         return self._instance.count_resume_run_attempts(run.run_id)
 
-    def terminate(self, run_id):
+    def terminate(self, run_id):  # pyright: ignore[reportIncompatibleMethodOverride]
         check.str_param(run_id, "run_id")
         run = self._instance.get_run_by_id(run_id)
 
@@ -445,8 +449,8 @@ class K8sRunLauncher(RunLauncher, ConfigurableClass):
                 WorkerStatus.FAILED, "Run has not completed but K8s job has no active pods"
             )
 
-        if status.failed:
-            return CheckRunHealthResult(WorkerStatus.FAILED, "K8s job failed")
         if status.succeeded:
             return CheckRunHealthResult(WorkerStatus.SUCCESS)
+        if status.failed and not status.active:
+            return CheckRunHealthResult(WorkerStatus.FAILED, "K8s job failed")
         return CheckRunHealthResult(WorkerStatus.RUNNING)

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/ops/k8s_job_op.py

@@ -313,6 +313,14 @@ def execute_k8s_job(
         labels["dagster/code-location"] = (
             context.dagster_run.remote_job_origin.repository_origin.code_location_origin.location_name
         )
+    env = user_defined_k8s_config.container_config.get("env")
+    deployment_name_env_var = (
+        next((entry for entry in env if entry["name"] == "DAGSTER_CLOUD_DEPLOYMENT_NAME"), None)
+        if env
+        else None
+    )
+    if deployment_name_env_var:
+        labels["dagster/deployment-name"] = deployment_name_env_var["value"]
 
     job = construct_dagster_k8s_job(
         job_config=k8s_job_config,

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s/pipes.py

@@ -8,7 +8,6 @@ import time
 from collections.abc import Callable, Generator, Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from datetime import datetime
-from pathlib import Path
 from typing import Any, Optional, Union
 
 import kubernetes
@@ -34,6 +33,7 @@ from dagster._core.pipes.utils import (
     extract_message_or_forward_to_stdout,
     open_pipes_session,
 )
+from dagster._time import parse_time_string
 from dagster_pipes import (
     DAGSTER_PIPES_CONTEXT_ENV_VAR,
     DAGSTER_PIPES_MESSAGES_ENV_VAR,
@@ -41,14 +41,16 @@ from dagster_pipes import (
     PipesExtras,
     encode_env_var,
 )
+from urllib3.exceptions import ReadTimeoutError
 
 from dagster_k8s.client import (
     DEFAULT_WAIT_BETWEEN_ATTEMPTS,
+    DEFAULT_WAIT_TIMEOUT,
     DagsterKubernetesClient,
     WaitForPodState,
 )
 from dagster_k8s.models import k8s_model_from_dict, k8s_snake_case_dict
-from dagster_k8s.utils import get_common_labels
+from dagster_k8s.utils import detect_current_namespace, get_common_labels
 
 INIT_WAIT_TIMEOUT_FOR_READY = 1800.0  # 30mins
 INIT_WAIT_TIMEOUT_FOR_TERMINATE = 10.0  # 10s
@@ -62,9 +64,13 @@ def get_pod_name(run_id: str, op_name: str):
 
 
 DEFAULT_CONTAINER_NAME = "dagster-pipes-execution"
-_NAMESPACE_SECRET_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 _DEV_NULL_MESSAGE_WRITER = encode_env_var({"path": "/dev/null"})
 
+DEFAULT_CONSUME_POD_LOGS_RETRIES = 5
+
+# By default, timeout and reconnect to the log stream every hour.
+DEFAULT_DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT = 3600
+
 
 class PipesK8sPodLogsMessageReader(PipesMessageReader):
     """Message reader that reads messages from kubernetes pod logs."""
@@ -80,24 +86,97 @@ class PipesK8sPodLogsMessageReader(PipesMessageReader):
         finally:
             self._handler = None
 
+    def _get_consume_logs_request_timeout(self) -> Optional[int]:
+        request_timeout_env_var = os.getenv("DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT")
+        if request_timeout_env_var:
+            return int(request_timeout_env_var)
+
+        return DEFAULT_DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT
+
     def consume_pod_logs(
         self,
+        context: Union[OpExecutionContext, AssetExecutionContext],
         core_api: kubernetes.client.CoreV1Api,
         pod_name: str,
         namespace: str,
     ):
+        last_seen_timestamp = None
+        catching_up_after_retry = False
+
+        request_timeout = self._get_consume_logs_request_timeout()
+
+        retries_remaining = int(
+            os.getenv(
+                "DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_RETRIES", str(DEFAULT_CONSUME_POD_LOGS_RETRIES)
+            )
+        )
+
         handler = check.not_none(
             self._handler, "can only consume logs within scope of context manager"
         )
-        for line in core_api.read_namespaced_pod_log(
-            pod_name,
-            namespace,
-            follow=True,
-            _preload_content=False,  # avoid JSON processing
-        ).stream():
-            log_chunk = line.decode("utf-8")
-            for log_line in log_chunk.split("\n"):
-                extract_message_or_forward_to_stdout(handler, log_line)
+
+        while True:
+            # On retry, re-connect to the log stream for new messages since the last seen timestamp
+            # (with a buffer to ensure none are missed). The messages are deduplicated by timestamp below.
+            if last_seen_timestamp:
+                since_seconds = int(
+                    max(time.time() - parse_time_string(last_seen_timestamp).timestamp(), 0)
+                    + int(os.getenv("DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_BUFFER_SECONDS", "300"))
+                )
+            else:
+                since_seconds = None
+
+            try:
+                for log_item in _process_log_stream(
+                    core_api.read_namespaced_pod_log(
+                        pod_name,
+                        namespace,
+                        follow=True,
+                        timestamps=True,
+                        since_seconds=since_seconds,
+                        _preload_content=False,  # avoid JSON processing
+                        _request_timeout=request_timeout,
+                    )
+                ):
+                    timestamp = log_item.timestamp
+                    message = log_item.log
+
+                    if (
+                        catching_up_after_retry
+                        and timestamp
+                        and last_seen_timestamp
+                        and timestamp <= last_seen_timestamp
+                    ):
+                        # This is a log that we've already seen before from before we retried
+                        continue
+                    else:
+                        catching_up_after_retry = False
+                        extract_message_or_forward_to_stdout(handler, message)
+                        if timestamp:
+                            last_seen_timestamp = (
+                                max(last_seen_timestamp, timestamp)
+                                if last_seen_timestamp
+                                else timestamp
+                            )
+                return
+            except Exception as e:
+                # Expected read timeouts can occur for long-running pods if a request timeout is set
+                expected_read_timeout = isinstance(e, ReadTimeoutError) and request_timeout
+
+                if expected_read_timeout:
+                    # Expected so doesn't need to be logged to event log, but write to stdout
+                    # for visibility
+                    logging.getLogger("dagster").info("Re-connecting to pod logs stream")
+                else:
+                    if retries_remaining == 0:
+                        raise
+                    retries_remaining -= 1
+                    context.log.warning(
+                        f"Error consuming pod logs. {retries_remaining} retr{('y' if retries_remaining == 1 else 'ies')} remaining",
+                        exc_info=True,
+                    )
+
+                catching_up_after_retry = True
 
     @contextmanager
     def async_consume_pod_logs(
@@ -359,7 +438,7 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
             )
 
     @public
-    def run(
+    def run(  # pyright: ignore[reportIncompatibleMethodOverride]
         self,
         *,
         context: Union[OpExecutionContext, AssetExecutionContext],
@@ -372,6 +451,7 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
         base_pod_spec: Optional[Mapping[str, Any]] = None,
         ignore_containers: Optional[set] = None,
         enable_multi_container_logs: bool = False,
+        pod_wait_timeout: float = DEFAULT_WAIT_TIMEOUT,
     ) -> PipesClientCompletedInvocation:
         """Publish a kubernetes pod and wait for it to complete, enriched with the pipes protocol.
 
@@ -407,6 +487,8 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
             ignore_containers (Optional[Set]): Ignore certain containers from waiting for termination. Defaults to
                 None.
             enable_multi_container_logs (bool): Whether or not to enable multi-container log consumption.
+            pod_wait_timeout (float): How long to wait for the pod to terminate before raising an exception.
+                Defaults to 24h. Set to 0 to disable.
 
         Returns:
             PipesClientCompletedInvocation: Wrapper containing results reported by the external
@@ -422,7 +504,7 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
             context_injector=self.context_injector,
             message_reader=self.message_reader,
         ) as pipes_session:
-            namespace = namespace or _detect_current_namespace(self.kubeconfig_file) or "default"
+            namespace = namespace or detect_current_namespace(self.kubeconfig_file) or "default"
             pod_name = get_pod_name(context.run_id, context.op.name)
             pod_body = build_pod_body(
                 pod_name=pod_name,
@@ -446,13 +528,14 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
                     pod_name=pod_name,
                     enable_multi_container_logs=enable_multi_container_logs,
                 ):
-                    # We need to wait for the pod to start up so that the log streaming is successful afterwards.
+                    # wait until the pod is fully terminated (or raise an exception if it failed)
                     client.wait_for_pod(
                         pod_name,
                         namespace,
                         wait_for_state=WaitForPodState.Terminated,
                         ignore_containers=ignore_containers,
                         wait_time_between_attempts=self.poll_interval,
+                        wait_timeout=pod_wait_timeout,
                     )
             finally:
                 client.core_api.delete_namespaced_pod(pod_name, namespace)
@@ -504,6 +587,7 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
                     return
             else:
                 self.message_reader.consume_pod_logs(
+                    context=context,
                     core_api=client.core_api,
                     namespace=namespace,
                     pod_name=pod_name,
@@ -512,29 +596,6 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
         yield
 
 
-def _detect_current_namespace(
-    kubeconfig_file: Optional[str], namespace_secret_path: Path = _NAMESPACE_SECRET_PATH
-) -> Optional[str]:
-    """Get the current in-cluster namespace when operating within the cluster.
-
-    First attempt to read it from the `serviceaccount` secret or get it from the kubeconfig_file if it is possible.
-    It will attempt to take from the active context if it exists and returns None if it does not exist.
-    """
-    if namespace_secret_path.exists():
-        with namespace_secret_path.open() as f:
-            # We only need to read the first line, this guards us against bad input.
-            return f.read().strip()
-
-    if not kubeconfig_file:
-        return None
-
-    try:
-        _, active_context = kubernetes.config.list_kube_config_contexts(kubeconfig_file)
-        return active_context["context"]["namespace"]
-    except KeyError:
-        return None
-
-
 def build_pod_body(
     pod_name: str,
     image: Optional[str],

dagster_k8s-0.28.12/dagster_k8s/utils.py

@@ -0,0 +1,62 @@
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+import kubernetes
+from dagster import __version__ as dagster_version
+
+if TYPE_CHECKING:
+    from dagster_k8s.job import UserDefinedDagsterK8sConfig
+
+
+def sanitize_k8s_label(label_name: str):
+    # Truncate too long label values to fit into 63-characters limit and avoid invalid characters.
+    # https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+    label_name = label_name[:63]
+    return re.sub(r"[^a-zA-Z0-9\-_\.]", "-", label_name).strip("-").strip("_").strip(".")
+
+
+def get_common_labels():
+    # See: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
+    return {
+        "app.kubernetes.io/name": "dagster",
+        "app.kubernetes.io/instance": "dagster",
+        "app.kubernetes.io/version": sanitize_k8s_label(dagster_version),
+        "app.kubernetes.io/part-of": "dagster",
+    }
+
+
+def get_deployment_id_label(user_defined_k8s_config: "UserDefinedDagsterK8sConfig"):
+    env = user_defined_k8s_config.container_config.get("env")
+    deployment_name_env_var = (
+        next((entry for entry in env if entry["name"] == "DAGSTER_CLOUD_DEPLOYMENT_NAME"), None)
+        if env
+        else None
+    )
+    return deployment_name_env_var["value"] if deployment_name_env_var else None
+
+
+_NAMESPACE_SECRET_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+
+
+def detect_current_namespace(
+    kubeconfig_file: Optional[str], namespace_secret_path: Path = _NAMESPACE_SECRET_PATH
+) -> Optional[str]:
+    """Get the current in-cluster namespace when operating within the cluster.
+
+    First attempt to read it from the `serviceaccount` secret or get it from the kubeconfig_file if it is possible.
+    It will attempt to take from the active context if it exists and returns None if it does not exist.
+    """
+    if namespace_secret_path.exists():
+        with namespace_secret_path.open() as f:
+            # We only need to read the first line, this guards us against bad input.
+            return f.read().strip()
+
+    if not kubeconfig_file:
+        return None
+
+    try:
+        _, active_context = kubernetes.config.list_kube_config_contexts(kubeconfig_file)
+        return active_context["context"]["namespace"]
+    except KeyError:
+        return None

dagster_k8s-0.28.12/dagster_k8s/version.py

@@ -0,0 +1 @@
+__version__ = "0.28.12"

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12/dagster_k8s.egg-info}/PKG-INFO

@@ -1,16 +1,29 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-k8s
-Version: 0.26.2
+Version: 0.28.12
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.13
+Requires-Python: >=3.10,<3.15
 License-File: LICENSE
+Requires-Dist: dagster==1.12.12
+Requires-Dist: kubernetes<36
+Requires-Dist: google-auth!=2.23.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/dagster_k8s.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ setup.cfg
 setup.py
 dagster_k8s/__init__.py
 dagster_k8s/client.py
+dagster_k8s/component.py
 dagster_k8s/container_context.py
 dagster_k8s/executor.py
 dagster_k8s/job.py

dagster_k8s-0.28.12/dagster_k8s.egg-info/requires.txt

@@ -0,0 +1,3 @@
+dagster==1.12.12
+kubernetes<36
+google-auth!=2.23.1

{dagster-k8s-0.26.2 → dagster_k8s-0.28.12}/setup.py

@@ -31,18 +31,19 @@ setup(
     description="A Dagster integration for k8s",
     url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s",
     classifiers=[
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
     ],
     packages=find_packages(exclude=["dagster_k8s_tests*"]),
     include_package_data=True,
-    python_requires=">=3.9,<3.13",
+    python_requires=">=3.10,<3.15",
     install_requires=[
-        "dagster==1.10.2",
+        "dagster==1.12.12",
         f"kubernetes<{KUBERNETES_VERSION_UPPER_BOUND}",
         # exclude a google-auth release that added an overly restrictive urllib3 pin that confuses dependency resolvers
         "google-auth!=2.23.1",

dagster-k8s-0.26.2/README.md

@@ -1,4 +0,0 @@
-# dagster-k8s
-
-The docs for `dagster-k8s` can be found
-[here](https://docs.dagster.io/api/python-api/libraries/dagster-k8s).

dagster-k8s-0.26.2/dagster_k8s/kubernetes_version.py

@@ -1 +0,0 @@
-KUBERNETES_VERSION_UPPER_BOUND = "33"

dagster-k8s-0.26.2/dagster_k8s/utils.py

@@ -1,20 +0,0 @@
-import re
-
-from dagster import __version__ as dagster_version
-
-
-def sanitize_k8s_label(label_name: str):
-    # Truncate too long label values to fit into 63-characters limit and avoid invalid characters.
-    # https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
-    label_name = label_name[:63]
-    return re.sub(r"[^a-zA-Z0-9\-_\.]", "-", label_name).strip("-").strip("_").strip(".")
-
-
-def get_common_labels():
-    # See: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
-    return {
-        "app.kubernetes.io/name": "dagster",
-        "app.kubernetes.io/instance": "dagster",
-        "app.kubernetes.io/version": sanitize_k8s_label(dagster_version),
-        "app.kubernetes.io/part-of": "dagster",
-    }

dagster-k8s-0.26.2/dagster_k8s/version.py

@@ -1 +0,0 @@
-__version__ = "0.26.2"

dagster-k8s-0.26.2/dagster_k8s.egg-info/requires.txt

@@ -1,3 +0,0 @@
-dagster==1.10.2
-kubernetes<33
-google-auth!=2.23.1