dagster-k8s 0.26.9__tar.gz → 0.26.10__tar.gz

{dagster-k8s-0.26.9/dagster_k8s.egg-info → dagster-k8s-0.26.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.26.9
+Version: 0.26.10
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs

{dagster-k8s-0.26.9 → dagster-k8s-0.26.10}/dagster_k8s/pipes.py

@@ -34,6 +34,7 @@ from dagster._core.pipes.utils import (
     extract_message_or_forward_to_stdout,
     open_pipes_session,
 )
+from dagster._time import parse_time_string
 from dagster_pipes import (
     DAGSTER_PIPES_CONTEXT_ENV_VAR,
     DAGSTER_PIPES_MESSAGES_ENV_VAR,
@@ -41,6 +42,7 @@ from dagster_pipes import (
     PipesExtras,
     encode_env_var,
 )
+from urllib3.exceptions import ReadTimeoutError
 
 from dagster_k8s.client import (
     DEFAULT_WAIT_BETWEEN_ATTEMPTS,
@@ -65,6 +67,11 @@ DEFAULT_CONTAINER_NAME = "dagster-pipes-execution"
 _NAMESPACE_SECRET_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 _DEV_NULL_MESSAGE_WRITER = encode_env_var({"path": "/dev/null"})
 
+DEFAULT_CONSUME_POD_LOGS_RETRIES = 5
+
+# By default, timeout and reconnect to the log stream every hour.
+DEFAULT_DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT = 3600
+
 
 class PipesK8sPodLogsMessageReader(PipesMessageReader):
     """Message reader that reads messages from kubernetes pod logs."""
@@ -80,24 +87,97 @@ class PipesK8sPodLogsMessageReader(PipesMessageReader):
         finally:
             self._handler = None
 
+    def _get_consume_logs_request_timeout(self) -> Optional[int]:
+        request_timeout_env_var = os.getenv("DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT")
+        if request_timeout_env_var:
+            return int(request_timeout_env_var)
+
+        return DEFAULT_DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT
+
     def consume_pod_logs(
         self,
+        context: Union[OpExecutionContext, AssetExecutionContext],
         core_api: kubernetes.client.CoreV1Api,
         pod_name: str,
         namespace: str,
     ):
+        last_seen_timestamp = None
+        catching_up_after_retry = False
+
+        request_timeout = self._get_consume_logs_request_timeout()
+
+        retries_remaining = int(
+            os.getenv(
+                "DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_RETRIES", str(DEFAULT_CONSUME_POD_LOGS_RETRIES)
+            )
+        )
+
         handler = check.not_none(
             self._handler, "can only consume logs within scope of context manager"
         )
-        for line in core_api.read_namespaced_pod_log(
-            pod_name,
-            namespace,
-            follow=True,
-            _preload_content=False,  # avoid JSON processing
-        ).stream():
-            log_chunk = line.decode("utf-8")
-            for log_line in log_chunk.split("\n"):
-                extract_message_or_forward_to_stdout(handler, log_line)
+
+        while True:
+            # On retry, re-connect to the log stream for new messages since the last seen timestamp
+            # (with a buffer to ensure none are missed). The messages are deduplicated by timestamp below.
+            if last_seen_timestamp:
+                since_seconds = int(
+                    max(time.time() - parse_time_string(last_seen_timestamp).timestamp(), 0)
+                    + int(os.getenv("DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_BUFFER_SECONDS", "300"))
+                )
+            else:
+                since_seconds = None
+
+            try:
+                for log_item in _process_log_stream(
+                    core_api.read_namespaced_pod_log(
+                        pod_name,
+                        namespace,
+                        follow=True,
+                        timestamps=True,
+                        since_seconds=since_seconds,
+                        _preload_content=False,  # avoid JSON processing
+                        _request_timeout=request_timeout,
+                    )
+                ):
+                    timestamp = log_item.timestamp
+                    message = log_item.log
+
+                    if (
+                        catching_up_after_retry
+                        and timestamp
+                        and last_seen_timestamp
+                        and timestamp <= last_seen_timestamp
+                    ):
+                        # This is a log that we've already seen before from before we retried
+                        continue
+                    else:
+                        catching_up_after_retry = False
+                        extract_message_or_forward_to_stdout(handler, message)
+                        if timestamp:
+                            last_seen_timestamp = (
+                                max(last_seen_timestamp, timestamp)
+                                if last_seen_timestamp
+                                else timestamp
+                            )
+                return
+            except Exception as e:
+                # Expected read timeouts can occur for long-running pods if a request timeout is set
+                expected_read_timeout = isinstance(e, ReadTimeoutError) and request_timeout
+
+                if expected_read_timeout:
+                    # Expected so doesn't need to be logged to event log, but write to stdout
+                    # for visibility
+                    logging.getLogger("dagster").info("Re-connecting to pod logs stream")
+                else:
+                    if retries_remaining == 0:
+                        raise
+                    retries_remaining -= 1
+                    context.log.warning(
+                        f"Error consuming pod logs. {retries_remaining} retr{('y' if retries_remaining==1 else 'ies')} remaining",
+                        exc_info=True,
+                    )
+
+                catching_up_after_retry = True
 
     @contextmanager
     def async_consume_pod_logs(
@@ -504,6 +584,7 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
                     return
             else:
                 self.message_reader.consume_pod_logs(
+                    context=context,
                     core_api=client.core_api,
                     namespace=namespace,
                     pod_name=pod_name,

dagster-k8s-0.26.10/dagster_k8s/version.py

@@ -0,0 +1 @@
+__version__ = "0.26.10"

{dagster-k8s-0.26.9 → dagster-k8s-0.26.10/dagster_k8s.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.26.9
+Version: 0.26.10
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs

{dagster-k8s-0.26.9 → dagster-k8s-0.26.10}/dagster_k8s.egg-info/requires.txt

@@ -1,3 +1,3 @@
-dagster==1.10.9
+dagster==1.10.10
 kubernetes<33
 google-auth!=2.23.1

{dagster-k8s-0.26.9 → dagster-k8s-0.26.10}/setup.py

@@ -42,7 +42,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.13",
     install_requires=[
-        "dagster==1.10.9",
+        "dagster==1.10.10",
         f"kubernetes<{KUBERNETES_VERSION_UPPER_BOUND}",
         # exclude a google-auth release that added an overly restrictive urllib3 pin that confuses dependency resolvers
         "google-auth!=2.23.1",

dagster-k8s-0.26.9/dagster_k8s/version.py

@@ -1 +0,0 @@
-__version__ = "0.26.9"