dagster-k8s 0.26.9__py3-none-any.whl → 0.26.10__py3-none-any.whl

dagster_k8s/pipes.py

@@ -34,6 +34,7 @@ from dagster._core.pipes.utils import (
     extract_message_or_forward_to_stdout,
     open_pipes_session,
 )
+from dagster._time import parse_time_string
 from dagster_pipes import (
     DAGSTER_PIPES_CONTEXT_ENV_VAR,
     DAGSTER_PIPES_MESSAGES_ENV_VAR,
@@ -41,6 +42,7 @@ from dagster_pipes import (
     PipesExtras,
     encode_env_var,
 )
+from urllib3.exceptions import ReadTimeoutError
 
 from dagster_k8s.client import (
     DEFAULT_WAIT_BETWEEN_ATTEMPTS,
@@ -65,6 +67,11 @@ DEFAULT_CONTAINER_NAME = "dagster-pipes-execution"
 _NAMESPACE_SECRET_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 _DEV_NULL_MESSAGE_WRITER = encode_env_var({"path": "/dev/null"})
 
+DEFAULT_CONSUME_POD_LOGS_RETRIES = 5
+
+# By default, timeout and reconnect to the log stream every hour.
+DEFAULT_DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT = 3600
+
 
 class PipesK8sPodLogsMessageReader(PipesMessageReader):
     """Message reader that reads messages from kubernetes pod logs."""
@@ -80,24 +87,97 @@ class PipesK8sPodLogsMessageReader(PipesMessageReader):
         finally:
             self._handler = None
 
+    def _get_consume_logs_request_timeout(self) -> Optional[int]:
+        request_timeout_env_var = os.getenv("DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT")
+        if request_timeout_env_var:
+            return int(request_timeout_env_var)
+
+        return DEFAULT_DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_REQUEST_TIMEOUT
+
     def consume_pod_logs(
         self,
+        context: Union[OpExecutionContext, AssetExecutionContext],
         core_api: kubernetes.client.CoreV1Api,
         pod_name: str,
         namespace: str,
     ):
+        last_seen_timestamp = None
+        catching_up_after_retry = False
+
+        request_timeout = self._get_consume_logs_request_timeout()
+
+        retries_remaining = int(
+            os.getenv(
+                "DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_RETRIES", str(DEFAULT_CONSUME_POD_LOGS_RETRIES)
+            )
+        )
+
         handler = check.not_none(
             self._handler, "can only consume logs within scope of context manager"
         )
-        for line in core_api.read_namespaced_pod_log(
-            pod_name,
-            namespace,
-            follow=True,
-            _preload_content=False,  # avoid JSON processing
-        ).stream():
-            log_chunk = line.decode("utf-8")
-            for log_line in log_chunk.split("\n"):
-                extract_message_or_forward_to_stdout(handler, log_line)
+
+        while True:
+            # On retry, re-connect to the log stream for new messages since the last seen timestamp
+            # (with a buffer to ensure none are missed). The messages are deduplicated by timestamp below.
+            if last_seen_timestamp:
+                since_seconds = int(
+                    max(time.time() - parse_time_string(last_seen_timestamp).timestamp(), 0)
+                    + int(os.getenv("DAGSTER_PIPES_K8S_CONSUME_POD_LOGS_BUFFER_SECONDS", "300"))
+                )
+            else:
+                since_seconds = None
+
+            try:
+                for log_item in _process_log_stream(
+                    core_api.read_namespaced_pod_log(
+                        pod_name,
+                        namespace,
+                        follow=True,
+                        timestamps=True,
+                        since_seconds=since_seconds,
+                        _preload_content=False,  # avoid JSON processing
+                        _request_timeout=request_timeout,
+                    )
+                ):
+                    timestamp = log_item.timestamp
+                    message = log_item.log
+
+                    if (
+                        catching_up_after_retry
+                        and timestamp
+                        and last_seen_timestamp
+                        and timestamp <= last_seen_timestamp
+                    ):
+                        # This is a log that we've already seen before from before we retried
+                        continue
+                    else:
+                        catching_up_after_retry = False
+                        extract_message_or_forward_to_stdout(handler, message)
+                        if timestamp:
+                            last_seen_timestamp = (
+                                max(last_seen_timestamp, timestamp)
+                                if last_seen_timestamp
+                                else timestamp
+                            )
+                return
+            except Exception as e:
+                # Expected read timeouts can occur for long-running pods if a request timeout is set
+                expected_read_timeout = isinstance(e, ReadTimeoutError) and request_timeout
+
+                if expected_read_timeout:
+                    # Expected so doesn't need to be logged to event log, but write to stdout
+                    # for visibility
+                    logging.getLogger("dagster").info("Re-connecting to pod logs stream")
+                else:
+                    if retries_remaining == 0:
+                        raise
+                    retries_remaining -= 1
+                    context.log.warning(
+                        f"Error consuming pod logs. {retries_remaining} retr{('y' if retries_remaining==1 else 'ies')} remaining",
+                        exc_info=True,
+                    )
+
+                catching_up_after_retry = True
 
     @contextmanager
     def async_consume_pod_logs(
@@ -504,6 +584,7 @@ class PipesK8sClient(PipesClient, TreatAsResourceParam):
                     return
             else:
                 self.message_reader.consume_pod_logs(
+                    context=context,
                     core_api=client.core_api,
                     namespace=namespace,
                     pod_name=pod_name,

dagster_k8s/version.py

@@ -1 +1 @@
-__version__ = "0.26.9"
+__version__ = "0.26.10"

{dagster_k8s-0.26.9.dist-info → dagster_k8s-0.26.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.26.9
+Version: 0.26.10
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs
@@ -14,7 +14,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.10.9
+Requires-Dist: dagster ==1.10.10
 Requires-Dist: kubernetes <33
 Requires-Dist: google-auth !=2.23.1
 

{dagster_k8s-0.26.9.dist-info → dagster_k8s-0.26.10.dist-info}/RECORD

@@ -6,15 +6,15 @@ dagster_k8s/job.py,sha256=v7RXYLZHzy2sTC7HYdz_INqbLWyZxUz9QIMwORArjJg,43065
 dagster_k8s/kubernetes_version.py,sha256=T7MDA5pOOVE_woRUkKWxtiRj4RCLAVfjSn6ZXUr6sAA,38
 dagster_k8s/launcher.py,sha256=8dPYFkB6W6GS5q7Ab9Mf4Oqyu3riX-HBLYyf7TKV7HQ,16669
 dagster_k8s/models.py,sha256=REKzAY103pZw0pQFqvHaM6lL02hgbi0rxmm71qds4W8,6338
-dagster_k8s/pipes.py,sha256=ImEPlLCSDrLOhj6uHJ6Gd_J4KdEvVwmFWoh4AtClpnI,30487
+dagster_k8s/pipes.py,sha256=9-z_rPHYsAgEXE1QWFkRpI3_u6ifBXzG8OJefFCirtY,34045
 dagster_k8s/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
 dagster_k8s/test.py,sha256=cNtcbzxytiZtd01wY5ip7KPi01y0BUQuQhohoIfAFUM,684
 dagster_k8s/utils.py,sha256=c1bHqh5f1p5RZ0JCT6WEbPPjDvbgUp3pl4nYZRaaI4s,786
-dagster_k8s/version.py,sha256=ZG17eZIeBB7Zg7MM33Pu3Eg0xdJHjWdE5Wl10Wuq_wo,23
+dagster_k8s/version.py,sha256=ME_83c8BVbSixEHEcpqZ6-kklj3neFq1tYEhdbpNO5c,24
 dagster_k8s/ops/__init__.py,sha256=ur-9GrE_DRfnsFCpYan03qOY9cWbjagC8KHZFZuiCmc,113
 dagster_k8s/ops/k8s_job_op.py,sha256=CQHtTowGGLAHH5EUGNfqZxDT1MYMqslAJ9u11SzCf-I,22036
-dagster_k8s-0.26.9.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_k8s-0.26.9.dist-info/METADATA,sha256=XQhkCWNsRU3Vilw2sC3D0ScnFQcHCwMPAZEppXMUkng,733
-dagster_k8s-0.26.9.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_k8s-0.26.9.dist-info/top_level.txt,sha256=wFPjskoWPlk2hOLugYCaoZhSiZdUcbCA1QZe9I4dals,12
-dagster_k8s-0.26.9.dist-info/RECORD,,
+dagster_k8s-0.26.10.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_k8s-0.26.10.dist-info/METADATA,sha256=Q-8FhXNsNNOisiqHT1pheoSd_YInfFYq535OkxnK5Hk,735
+dagster_k8s-0.26.10.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+dagster_k8s-0.26.10.dist-info/top_level.txt,sha256=wFPjskoWPlk2hOLugYCaoZhSiZdUcbCA1QZe9I4dals,12
+dagster_k8s-0.26.10.dist-info/RECORD,,