prefect-client 3.6.27.dev3__tar.gz → 3.6.27.dev5__tar.gz

{prefect_client-3.6.27.dev3 → prefect_client-3.6.27.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect-client
-Version: 3.6.27.dev3
+Version: 3.6.27.dev5
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io

prefect_client-3.6.27.dev5/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.6.27.dev5"
+__build_date__ = "2026-04-16 21:12:02.023498+00:00"
+__git_commit__ = "03b8cad670c313d68d0bf8e87b0256e525713aa5"
+__dirty__ = False

{prefect_client-3.6.27.dev3 → prefect_client-3.6.27.dev5}/src/prefect/_experimental/bundles/__init__.py

@@ -24,6 +24,7 @@ from typing_extensions import Literal, NotRequired, TypeAlias
 import anyio
 import cloudpickle  # pyright: ignore[reportMissingTypeStubs]
 
+from prefect._internal.control_listener import configure_from_env
 from prefect.client.schemas.objects import FlowRun
 from prefect.context import SettingsContext, get_settings_context, serialize_context
 from prefect.engine import handle_engine_signals
@@ -32,6 +33,7 @@ from prefect.flows import Flow
 from prefect.logging import get_logger
 from prefect.settings.context import get_current_settings
 from prefect.settings.models.root import Settings
+from prefect.utilities.processutils import sanitize_subprocess_env
 from prefect.utilities.slugify import slugify
 
 from prefect._experimental._launchers import validate_bundle_step_launcher
@@ -509,7 +511,7 @@ def extract_flow_from_bundle(bundle: SerializedBundle) -> Flow[Any, Any]:
 def _extract_and_run_flow(
     bundle: SerializedBundle,
     cwd: Path | str | None = None,
-    env: dict[str, Any] | None = None,
+    env: dict[str, str | None] | None = None,
 ) -> None:
     """
     Extracts a flow from a bundle and runs it.
@@ -522,14 +524,20 @@ def _extract_and_run_flow(
         env: The environment to use when running the flow.
     """
 
-    os.environ.update(env or {})
+    os.environ.update(sanitize_subprocess_env(env))
     # TODO: make this a thing we can pass directly to the engine
     os.environ["PREFECT__ENABLE_CANCELLATION_AND_CRASHED_HOOKS"] = "false"
     settings_context = get_settings_context()
+    flow_run = FlowRun.model_validate(bundle["flow_run"])
+
+    # Consume the runner control-channel bootstrap env before deserializing
+    # bundled function/context objects, but do not connect yet. The actual
+    # listener socket is only opened while `capture_sigterm()` is active
+    # inside the flow engine.
+    configure_from_env()
 
     flow = _deserialize_bundle_object(bundle["function"])
     context = _deserialize_bundle_object(bundle["context"])
-    flow_run = FlowRun.model_validate(bundle["flow_run"])
 
     if cwd:
         os.chdir(cwd)
@@ -552,7 +560,7 @@ def _extract_and_run_flow(
 
 def execute_bundle_in_subprocess(
     bundle: SerializedBundle,
-    env: dict[str, Any] | None = None,
+    env: dict[str, str | None] | None = None,
     cwd: Path | str | None = None,
 ) -> multiprocessing.context.SpawnProcess:
     """
@@ -576,13 +584,16 @@ def execute_bundle_in_subprocess(
             env=os.environ,
         )
 
+    subprocess_env = sanitize_subprocess_env(
+        get_current_settings().to_environment_variables(exclude_unset=True)
+        | os.environ
+        | env
+    )
     process = ctx.Process(
         target=_extract_and_run_flow,
         kwargs={
             "bundle": bundle,
-            "env": get_current_settings().to_environment_variables(exclude_unset=True)
-            | os.environ
-            | env,
+            "env": subprocess_env,
             "cwd": cwd,
         },
     )

{prefect_client-3.6.27.dev3 → prefect_client-3.6.27.dev5}/src/prefect/_experimental/bundles/execute.py

@@ -34,7 +34,10 @@ async def execute_bundle(bundle: dict) -> None:
         flow_run = FlowRun.model_validate(bundle["flow_run"])
         executor = ctx.create_executor(
             flow_run,
-            BundleExecutionStarter(bundle=bundle),
+            BundleExecutionStarter(
+                bundle=bundle,
+                control_channel=ctx.control_channel,
+            ),
             resolve_flow=resolve_flow,
         )
         await executor.submit()

prefect_client-3.6.27.dev5/src/prefect/_internal/control_listener.py

@@ -0,0 +1,223 @@
+"""
+Child-process runner-control intent listener.
+
+This module is intentionally small. It only supports steady-state graceful
+control delivery after Prefect has already installed its SIGTERM bridge via
+`capture_sigterm()`.
+
+Protocol:
+
+1. The runner injects `PREFECT__CONTROL_PORT` and `PREFECT__CONTROL_TOKEN`
+   into the child environment.
+2. Process entrypoints call `configure_from_env()` early to consume those
+   one-shot env vars without connecting yet.
+3. The outermost `capture_sigterm()` calls `start()`, which connects back to
+   the runner and spawns a daemon thread blocked on the socket.
+4. The runner writes a single-byte intent (`b"c"` today).
+5. If Prefect still owns the live SIGTERM bridge, the child writes `b"a"`,
+   commits the intent for engine dispatch, and on Windows triggers
+   `_thread.interrupt_main(SIGTERM)`.
+6. The runner then sends its normal external termination signal. On POSIX,
+   that real `SIGTERM` is the only trigger that interrupts blocking code.
+
+If the child is not connected yet, or can no longer safely acknowledge the
+intent, the runner sees no ack and falls back to its existing crash-style
+termination path. There is deliberately no startup-time graceful cancellation
+contract in this implementation.
+"""
+
+from __future__ import annotations
+
+import _thread
+import os
+import signal
+import socket
+import threading
+from typing import Literal
+
+from prefect.utilities.engine import commit_control_intent_and_ack
+
+Intent = Literal["cancel"]
+
+_INTENT_FOR_BYTE: dict[bytes, Intent] = {
+    b"c": "cancel",
+}
+
+_intent: Intent | None = None
+_intent_lock = threading.Lock()
+
+_configured = False
+_configured_port: int | None = None
+_configured_token: str | None = None
+
+_started = False
+_started_lock = threading.Lock()
+_socket: socket.socket | None = None
+_reader_thread: threading.Thread | None = None
+
+
+def get_intent() -> Intent | None:
+    """Return the committed control intent, if any."""
+    with _intent_lock:
+        return _intent
+
+
+def _set_intent(value: Intent) -> None:
+    global _intent
+    with _intent_lock:
+        _intent = value
+
+
+def _clear_intent() -> None:
+    global _intent
+    with _intent_lock:
+        _intent = None
+
+
+def clear_intent() -> None:
+    """Clear the committed control intent after the current session consumes it."""
+    _clear_intent()
+
+
+def configure_from_env() -> None:
+    """Consume one-shot control-channel env vars without connecting yet."""
+    global _configured, _configured_port, _configured_token
+
+    with _started_lock:
+        if _configured:
+            return
+
+        _configured = True
+        port_str = os.environ.pop("PREFECT__CONTROL_PORT", None)
+        token = os.environ.pop("PREFECT__CONTROL_TOKEN", None)
+        if not port_str or not token:
+            return
+
+        try:
+            _configured_port = int(port_str)
+        except ValueError:
+            _configured_port = None
+            _configured_token = None
+            return
+
+        _configured_token = token
+
+
+def _acknowledge_intent(sock: socket.socket, intent: Intent) -> bool:
+    """Commit intent and acknowledge it to the runner."""
+    if not commit_control_intent_and_ack(
+        commit_intent=lambda: _set_intent(intent),
+        clear_intent=_clear_intent,
+        send_ack=lambda: sock.sendall(b"a"),
+        trigger_cancel=(
+            (lambda: _thread.interrupt_main(signal.SIGTERM))
+            if os.name == "nt"
+            else None
+        ),
+    ):
+        return False
+
+    return True
+
+
+def _reader_loop(sock: socket.socket) -> None:
+    """Block on the runner's control channel and act on a single intent byte."""
+    try:
+        while True:
+            try:
+                data = sock.recv(1)
+            except OSError:
+                return
+            if not data:
+                return
+
+            intent = _INTENT_FOR_BYTE.get(data)
+            if intent is None:
+                return
+
+            _acknowledge_intent(sock, intent)
+            return
+    finally:
+        try:
+            sock.close()
+        except OSError:
+            pass
+
+
+def start() -> None:
+    """Connect to the runner's control channel if bootstrap config is present."""
+    global _started, _socket, _reader_thread
+
+    configure_from_env()
+
+    with _started_lock:
+        if _started:
+            return
+        if _configured_port is None or _configured_token is None:
+            return
+
+        # A new listener session must not inherit a committed intent from an
+        # earlier flow run in the same interpreter.
+        _clear_intent()
+
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect(("127.0.0.1", _configured_port))
+            sock.sendall(_configured_token.encode("ascii") + b"\n")
+        except OSError:
+            try:
+                sock.close()
+            except (OSError, UnboundLocalError):
+                pass
+            return
+
+        thread = threading.Thread(
+            target=_reader_loop,
+            args=(sock,),
+            name="prefect-control-listener",
+            daemon=True,
+        )
+        _socket = sock
+        _reader_thread = thread
+        _started = True
+        thread.start()
+
+
+def stop() -> None:
+    """Close the active control connection, if any."""
+    global _configured, _configured_port, _configured_token
+    global _started, _socket, _reader_thread
+
+    with _started_lock:
+        sock = _socket
+        _socket = None
+        _reader_thread = None
+        _started = False
+        _configured = False
+        _configured_port = None
+        _configured_token = None
+
+    if sock is not None:
+        try:
+            sock.shutdown(socket.SHUT_RDWR)
+        except OSError:
+            pass
+        try:
+            sock.close()
+        except OSError:
+            pass
+
+
+def reset_for_testing() -> None:
+    """Reset module state. Tests only."""
+    global _intent, _configured, _configured_port, _configured_token
+
+    with _intent_lock:
+        _intent = None
+
+    stop()
+
+    with _started_lock:
+        _configured = False
+        _configured_port = None
+        _configured_token = None

{prefect_client-3.6.27.dev3 → prefect_client-3.6.27.dev5}/src/prefect/deployments/runner.py

@@ -963,7 +963,7 @@ class RunnerDeployment(BaseModel):
             concurrency_limit
         )
 
-        deployment = cls(
+        kwargs: dict[str, Any] = dict(
             name=name,
             flow_name=flow.name,
             schedules=constructed_schedules,
@@ -977,10 +977,14 @@ class RunnerDeployment(BaseModel):
             version=version,
             version_type=version_type,
             enforce_parameter_schema=enforce_parameter_schema,
-            work_pool_name=work_pool_name,
-            work_queue_name=work_queue_name,
             job_variables=job_variables,
         )
+        if work_pool_name is not None:
+            kwargs["work_pool_name"] = work_pool_name
+        if work_queue_name is not None:
+            kwargs["work_queue_name"] = work_queue_name
+
+        deployment = cls(**kwargs)
         deployment._sla = _sla
 
         if not deployment.entrypoint:
@@ -1110,7 +1114,7 @@ class RunnerDeployment(BaseModel):
             concurrency_limit
         )
 
-        deployment = cls(
+        kwargs: dict[str, Any] = dict(
             name=name,
             flow_name=flow_name or flow.name,
             schedules=constructed_schedules,
@@ -1124,10 +1128,14 @@ class RunnerDeployment(BaseModel):
             version=version,
             entrypoint=entrypoint,
             enforce_parameter_schema=enforce_parameter_schema,
-            work_pool_name=work_pool_name,
-            work_queue_name=work_queue_name,
             job_variables=job_variables,
         )
+        if work_pool_name is not None:
+            kwargs["work_pool_name"] = work_pool_name
+        if work_queue_name is not None:
+            kwargs["work_queue_name"] = work_queue_name
+
+        deployment = cls(**kwargs)
         deployment._sla = _sla
         deployment._path = str(Path.cwd())
 
@@ -1237,7 +1245,7 @@ class RunnerDeployment(BaseModel):
                 if ":" not in entrypoint:
                     sys.path.remove(str(storage.destination))
 
-        deployment = cls(
+        kwargs: dict[str, Any] = dict(
             name=name,
             flow_name=flow_name or flow.name,
             schedules=constructed_schedules,
@@ -1253,10 +1261,14 @@ class RunnerDeployment(BaseModel):
             entrypoint=entrypoint,
             enforce_parameter_schema=enforce_parameter_schema,
             storage=storage,
-            work_pool_name=work_pool_name,
-            work_queue_name=work_queue_name,
             job_variables=job_variables,
         )
+        if work_pool_name is not None:
+            kwargs["work_pool_name"] = work_pool_name
+        if work_queue_name is not None:
+            kwargs["work_queue_name"] = work_queue_name
+
+        deployment = cls(**kwargs)
         deployment._sla = _sla
         deployment._entrypoint_type = (
             EntrypointType.FILE_PATH
@@ -1372,7 +1384,7 @@ class RunnerDeployment(BaseModel):
                 if ":" not in entrypoint:
                     sys.path.remove(str(storage.destination))
 
-        deployment = cls(
+        kwargs: dict[str, Any] = dict(
             name=name,
             flow_name=flow_name or flow.name,
             schedules=constructed_schedules,
@@ -1388,10 +1400,14 @@ class RunnerDeployment(BaseModel):
             entrypoint=entrypoint,
             enforce_parameter_schema=enforce_parameter_schema,
             storage=storage,
-            work_pool_name=work_pool_name,
-            work_queue_name=work_queue_name,
             job_variables=job_variables,
         )
+        if work_pool_name is not None:
+            kwargs["work_pool_name"] = work_pool_name
+        if work_queue_name is not None:
+            kwargs["work_queue_name"] = work_queue_name
+
+        deployment = cls(**kwargs)
         deployment._sla = _sla
         deployment._entrypoint_type = (
             EntrypointType.FILE_PATH

{prefect_client-3.6.27.dev3 → prefect_client-3.6.27.dev5}/src/prefect/engine.py

@@ -1,22 +1,26 @@
 from __future__ import annotations
 
+import asyncio
+import inspect
 import os
 import sys
 from contextlib import contextmanager
+from types import GeneratorType
 from typing import TYPE_CHECKING, Any, Callable
 from uuid import UUID
 
 from prefect._internal.compatibility.migration import getattr_migration
+from prefect._internal.control_listener import (
+    clear_intent,
+    configure_from_env,
+    get_intent,
+)
 from prefect.exceptions import (
     Abort,
     Pause,
+    TerminationSignal,
 )
-from prefect.logging.loggers import (
-    get_logger,
-)
-from prefect.utilities.asyncutils import (
-    run_coro_as_sync,
-)
+from prefect.logging.loggers import get_logger
 
 if TYPE_CHECKING:
     import logging
@@ -28,6 +32,32 @@ if TYPE_CHECKING:
 engine_logger: "logging.Logger" = get_logger("engine")
 
 
+def _drive_run_flow_result(flow: Any, run_result: object) -> None:
+    """Execute deferred work for generator flows returned by `run_flow()`."""
+    if getattr(flow, "isasync", False) and getattr(flow, "isgenerator", False):
+        if not inspect.isasyncgen(run_result):
+            return
+
+        async def _consume_asyncgen() -> None:
+            async for _ in run_result:
+                pass
+
+        asyncio.run(_consume_asyncgen())
+        return
+
+    if not getattr(flow, "isasync", False) and getattr(flow, "isgenerator", False):
+        if not isinstance(run_result, GeneratorType):
+            return
+        for _ in run_result:
+            pass
+        return
+
+    if getattr(flow, "isasync", False) and not getattr(flow, "isgenerator", False):
+        if not asyncio.iscoroutine(run_result):
+            return
+        asyncio.run(run_result)
+
+
 @contextmanager
 def handle_engine_signals(flow_run_id: UUID | None = None):
     """
@@ -69,6 +99,21 @@ def handle_engine_signals(flow_run_id: UUID | None = None):
             msg = "Execution is paused."
         engine_logger.info(msg)
         exit(0)
+    except TerminationSignal:
+        # A TerminationSignal can mean either:
+        # - an expected runner-driven control action (today: cancel intent),
+        # - or a raw external termination with no runner intent attached.
+        #
+        # Only the first case should translate to a clean process exit.
+        if get_intent() == "cancel":
+            if flow_run_id:
+                msg = f"Execution of flow run '{flow_run_id}' was cancelled."
+            else:
+                msg = "Execution was cancelled."
+            engine_logger.info(msg)
+            clear_intent()
+            exit(0)
+        raise
     except Exception:
         if flow_run_id:
             msg = f"Execution of flow run '{flow_run_id}' exited with unexpected exception"
@@ -97,6 +142,13 @@ if __name__ == "__main__":
         )
         exit(1)
 
+    # Consume the runner control-channel bootstrap env before loading any
+    # flow code, but do not connect yet. The actual socket connection is
+    # established only while `capture_sigterm()` is active inside the flow
+    # engine; cancels that land before then fall back to the existing
+    # crash-style termination path.
+    configure_from_env()
+
     with handle_engine_signals(flow_run_id):
         from prefect.flow_engine import (
             flow_run_logger,
@@ -118,14 +170,15 @@ if __name__ == "__main__":
             )
             raise
 
-        # run the flow
+        # Run async flows on a main-thread event loop in this subprocess so
+        # `capture_sigterm()` can install Prefect's SIGTERM bridge. Using the
+        # shared run-sync loop here moves execution off the main thread, which
+        # prevents graceful cancellation from ever becoming ready.
         with RunMetrics(flow_run, flow):
-            if flow.isasync:
-                run_coro_as_sync(
-                    run_flow(flow, flow_run=flow_run, error_logger=run_logger)
-                )
-            else:
-                run_flow(flow, flow_run=flow_run, error_logger=run_logger)
+            _run_result: object = run_flow(
+                flow, flow_run=flow_run, error_logger=run_logger
+            )
+            _drive_run_flow_result(flow, _run_result)
 
 
 __getattr__: Callable[[str], Any] = getattr_migration(__name__)