prefect-client 3.0.0rc16__py3-none-any.whl → 3.0.0rc18__py3-none-any.whl

prefect/blocks/core.py

@@ -1046,7 +1046,7 @@ class Block(BaseModel, ABC):
     @classmethod
     @sync_compatible
     @inject_client
-    async def register_type_and_schema(cls, client: "PrefectClient" = None):
+    async def register_type_and_schema(cls, client: Optional["PrefectClient"] = None):
         """
         Makes block available for configuration with current Prefect API.
         Recursively registers all nested blocks. Registration is idempotent.

prefect/cache_policies.py

@@ -143,8 +143,8 @@ class TaskSource(CachePolicy):
     def compute_key(
         self,
         task_ctx: TaskRunContext,
-        inputs: Dict[str, Any],
-        flow_parameters: Dict[str, Any],
+        inputs: Optional[Dict[str, Any]],
+        flow_parameters: Optional[Dict[str, Any]],
         **kwargs,
     ) -> Optional[str]:
         if not task_ctx:
@@ -153,6 +153,11 @@ class TaskSource(CachePolicy):
             lines = inspect.getsource(task_ctx.task)
         except TypeError:
             lines = inspect.getsource(task_ctx.task.fn.__class__)
+        except OSError as exc:
+            if "could not get source code" in str(exc):
+                lines = task_ctx.task.fn.__code__.co_code
+            else:
+                raise
 
         return hash_objects(lines)
 

prefect/client/schemas/actions.py

@@ -157,6 +157,10 @@ class DeploymentCreate(ActionBaseModel):
         default_factory=list,
         description="A list of schedules for the deployment.",
     )
+    concurrency_limit: Optional[int] = Field(
+        default=None,
+        description="The concurrency limit for the deployment.",
+    )
     enforce_parameter_schema: Optional[bool] = Field(
         default=None,
         description=(
@@ -229,6 +233,10 @@ class DeploymentUpdate(ActionBaseModel):
         default=None,
         description="A list of schedules for the deployment.",
     )
+    concurrency_limit: Optional[int] = Field(
+        default=None,
+        description="The concurrency limit for the deployment.",
+    )
     tags: List[str] = Field(default_factory=list)
     work_queue_name: Optional[str] = Field(None)
     work_pool_name: Optional[str] = Field(
@@ -607,11 +615,11 @@ class WorkQueueCreate(ActionBaseModel):
         default=False,
         description="Whether the work queue is paused.",
     )
-    concurrency_limit: Optional[int] = Field(
+    concurrency_limit: Optional[NonNegativeInteger] = Field(
         default=None,
         description="A concurrency limit for the work queue.",
     )
-    priority: Optional[int] = Field(
+    priority: Optional[PositiveInteger] = Field(
         default=None,
         description=(
             "The queue's priority. Lower values are higher priority (1 is the highest)."
@@ -635,8 +643,10 @@ class WorkQueueUpdate(ActionBaseModel):
     is_paused: bool = Field(
         default=False, description="Whether or not the work queue is paused."
     )
-    concurrency_limit: Optional[int] = Field(None)
-    priority: Optional[int] = Field(None)
+    concurrency_limit: Optional[NonNegativeInteger] = Field(None)
+    priority: Optional[PositiveInteger] = Field(
+        None, description="The queue's priority."
+    )
     last_polled: Optional[DateTime] = Field(None)
 
     # DEPRECATED

prefect/client/schemas/filters.py

@@ -505,6 +505,23 @@ class DeploymentFilterTags(PrefectBaseModel, OperatorMixin):
     )
 
 
+class DeploymentFilterConcurrencyLimit(PrefectBaseModel):
+    """Filter by `Deployment.concurrency_limit`."""
+
+    ge_: Optional[int] = Field(
+        default=None,
+        description="Only include deployments with a concurrency limit greater than or equal to this value",
+    )
+    le_: Optional[int] = Field(
+        default=None,
+        description="Only include deployments with a concurrency limit less than or equal to this value",
+    )
+    is_null_: Optional[bool] = Field(
+        default=None,
+        description="If true, only include deployments without a concurrency limit",
+    )
+
+
 class DeploymentFilter(PrefectBaseModel, OperatorMixin):
     """Filter for deployments. Only deployments matching all criteria will be returned."""
 
@@ -520,6 +537,9 @@ class DeploymentFilter(PrefectBaseModel, OperatorMixin):
     work_queue_name: Optional[DeploymentFilterWorkQueueName] = Field(
         default=None, description="Filter criteria for `Deployment.work_queue_name`"
     )
+    concurrency_limit: Optional[DeploymentFilterConcurrencyLimit] = Field(
+        default=None, description="Filter criteria for `Deployment.concurrency_limit`"
+    )
 
 
 class LogFilterName(PrefectBaseModel):

prefect/client/schemas/objects.py

@@ -996,6 +996,9 @@ class Deployment(ObjectBaseModel):
     paused: bool = Field(
         default=False, description="Whether or not the deployment is paused."
     )
+    concurrency_limit: Optional[int] = Field(
+        default=None, description="The concurrency limit for the deployment."
+    )
     schedules: List[DeploymentSchedule] = Field(
         default_factory=list, description="A list of schedules for the deployment."
     )

prefect/client/schemas/responses.py

@@ -313,6 +313,9 @@ class DeploymentResponse(ObjectBaseModel):
     flow_id: UUID = Field(
         default=..., description="The flow id associated with the deployment."
     )
+    concurrency_limit: Optional[int] = Field(
+        default=None, description="The concurrency limit for the deployment."
+    )
     paused: bool = Field(
         default=False, description="Whether or not the deployment is paused."
     )

prefect/client/schemas/sorting.py

@@ -59,6 +59,8 @@ class DeploymentSort(AutoEnum):
     UPDATED_DESC = AutoEnum.auto()
     NAME_ASC = AutoEnum.auto()
     NAME_DESC = AutoEnum.auto()
+    CONCURRENCY_LIMIT_ASC = AutoEnum.auto()
+    CONCURRENCY_LIMIT_DESC = AutoEnum.auto()
 
 
 class ArtifactSort(AutoEnum):

prefect/flows.py

@@ -611,12 +611,17 @@ class Flow(Generic[P, R]):
             # do not serialize the bound self object
             if self.ismethod and value is self.fn.__prefect_self__:
                 continue
+            if isinstance(value, (PrefectFuture, State)):
+                # Don't call jsonable_encoder() on a PrefectFuture or State to
+                # avoid triggering a __getitem__ call
+                serialized_parameters[key] = f"<{type(value).__name__}>"
+                continue
             try:
                 serialized_parameters[key] = jsonable_encoder(value)
             except (TypeError, ValueError):
                 logger.debug(
-                    f"Parameter {key!r} for flow {self.name!r} is of unserializable "
-                    f"type {type(value).__name__!r} and will not be stored "
+                    f"Parameter {key!r} for flow {self.name!r} is unserializable. "
+                    f"Type {type(value).__name__!r} and will not be stored "
                     "in the backend."
                 )
                 serialized_parameters[key] = f"<{type(value).__name__}>"

prefect/records/__init__.py

@@ -1 +1 @@
-from .store import RecordStore
+from .base import RecordStore

prefect/records/base.py

@@ -0,0 +1,167 @@
+import os
+import socket
+import threading
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Optional
+
+from prefect.results import BaseResult
+
+
+class RecordStore:
+    def read(self, key: str):
+        raise NotImplementedError
+
+    def write(self, key: str, value: dict):
+        raise NotImplementedError
+
+    def exists(self, key: str) -> bool:
+        return False
+
+    def acquire_lock(
+        self,
+        key: str,
+        holder: Optional[str] = None,
+        acquire_timeout: Optional[float] = None,
+        hold_timeout: Optional[float] = None,
+    ) -> bool:
+        """
+        Acquire a lock for a transaction record with the given key. Will block other
+        actors from updating this transaction record until the lock is
+        released.
+
+        Args:
+            key: Unique identifier for the transaction record.
+            holder: Unique identifier for the holder of the lock. If not provided,
+                a default holder based on the current host, process, and thread will
+                be used.
+            acquire_timeout: Max number of seconds to wait for the record to become
+                available if it is locked while attempting to acquire a lock. Pass 0
+                to attempt to acquire a lock without waiting. Blocks indefinitely by
+                default.
+            hold_timeout: Max number of seconds to hold the lock for. Holds the lock
+                indefinitely by default.
+
+        Returns:
+            bool: True if the lock was successfully acquired; False otherwise.
+        """
+        raise NotImplementedError
+
+    def release_lock(self, key: str, holder: Optional[str] = None):
+        """
+        Releases the lock on the corresponding transaction record.
+
+        Args:
+            key: Unique identifier for the transaction record.
+            holder: Unique identifier for the holder of the lock. Must match the
+                holder provided when acquiring the lock.
+        """
+        raise NotImplementedError
+
+    def is_locked(self, key: str) -> bool:
+        """
+        Simple check to see if the corresponding record is currently locked.
+
+        Args:
+            key: Unique identifier for the transaction record.
+
+        Returns:
+            True is the record is locked; False otherwise.
+        """
+        raise NotImplementedError
+
+    def is_lock_holder(self, key: str, holder: Optional[str] = None) -> bool:
+        """
+        Check if the current holder is the lock holder for the transaction record.
+
+        Args:
+            key: Unique identifier for the transaction record.
+            holder: Unique identifier for the holder of the lock. If not provided,
+                a default holder based on the current host, process, and thread will
+                be used.
+
+        Returns:
+            bool: True if the current holder is the lock holder; False otherwise.
+        """
+        raise NotImplementedError
+
+    def wait_for_lock(self, key: str, timeout: Optional[float] = None) -> bool:
+        """
+        Wait for the corresponding transaction record to become free.
+
+        Args:
+            key: Unique identifier for the transaction record.
+            timeout: Maximum time to wait. None means to wait indefinitely.
+
+        Returns:
+            bool: True if the lock becomes free within the timeout; False
+                otherwise.
+        """
+        ...
+
+    @staticmethod
+    def generate_default_holder() -> str:
+        """
+        Generate a default holder string using hostname, PID, and thread ID.
+
+        Returns:
+            str: A unique identifier string.
+        """
+        hostname = socket.gethostname()
+        pid = os.getpid()
+        thread_name = threading.current_thread().name
+        thread_id = threading.get_ident()
+        return f"{hostname}:{pid}:{thread_id}:{thread_name}"
+
+    @contextmanager
+    def lock(
+        self,
+        key: str,
+        holder: Optional[str] = None,
+        acquire_timeout: Optional[float] = None,
+        hold_timeout: Optional[float] = None,
+    ):
+        """
+        Context manager to lock the transaction record during the execution
+        of the nested code block.
+
+        Args:
+            key: Unique identifier for the transaction record.
+            holder: Unique identifier for the holder of the lock. If not provided,
+                a default holder based on the current host, process, and thread will
+                be used.
+            acquire_timeout: Max number of seconds to wait for the record to become
+                available if it is locked while attempting to acquire a lock. Pass 0
+                to attempt to acquire a lock without waiting. Blocks indefinitely by
+                default.
+            hold_timeout: Max number of seconds to hold the lock for. Holds the lock
+                indefinitely by default.
+
+        Example:
+            Hold a lock while during an operation:
+                ```python
+                    with TransactionRecord(key="my-transaction-record-key").lock():
+                        do_stuff()
+                ```
+        """
+        self.acquire_lock(
+            key=key,
+            holder=holder,
+            acquire_timeout=acquire_timeout,
+            hold_timeout=hold_timeout,
+        )
+
+        try:
+            yield
+        finally:
+            self.release_lock(key=key, holder=holder)
+
+
+@dataclass
+class TransactionRecord:
+    """
+    A dataclass representation of a transaction record.
+    """
+
+    key: str
+    result: Optional[BaseResult] = None

prefect/records/memory.py

@@ -0,0 +1,165 @@
+import threading
+from typing import Dict, Optional, TypedDict
+
+from prefect.results import BaseResult
+
+from .base import RecordStore, TransactionRecord
+
+
+class _LockInfo(TypedDict):
+    """
+    A dictionary containing information about a lock.
+
+    Attributes:
+        holder: The holder of the lock.
+        lock: The lock object.
+        expiration_timer: The timer for the lock expiration
+    """
+
+    holder: str
+    lock: threading.Lock
+    expiration_timer: Optional[threading.Timer]
+
+
+class MemoryRecordStore(RecordStore):
+    """
+    A record store that stores data in memory.
+    """
+
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        self._locks_dict_lock = threading.Lock()
+        self._locks: Dict[str, _LockInfo] = {}
+        self._records: Dict[str, TransactionRecord] = {}
+
+    def read(self, key: str) -> Optional[TransactionRecord]:
+        return self._records.get(key)
+
+    def write(self, key: str, value: BaseResult, holder: Optional[str] = None) -> None:
+        holder = holder or self.generate_default_holder()
+
+        with self._locks_dict_lock:
+            if self.is_locked(key) and not self.is_lock_holder(key, holder):
+                raise ValueError(
+                    f"Cannot write to transaction with key {key} because it is locked by another holder."
+                )
+        self._records[key] = TransactionRecord(key=key, result=value)
+
+    def exists(self, key: str) -> bool:
+        return key in self._records
+
+    def _expire_lock(self, key: str):
+        """
+        Expire the lock for the given key.
+
+        Used as a callback for the expiration timer of a lock.
+
+        Args:
+            key: The key of the lock to expire.
+        """
+        with self._locks_dict_lock:
+            if key in self._locks:
+                lock_info = self._locks[key]
+                if lock_info["lock"].locked():
+                    lock_info["lock"].release()
+                if lock_info["expiration_timer"]:
+                    lock_info["expiration_timer"].cancel()
+                del self._locks[key]
+
+    def acquire_lock(
+        self,
+        key: str,
+        holder: Optional[str] = None,
+        acquire_timeout: Optional[float] = None,
+        hold_timeout: Optional[float] = None,
+    ) -> bool:
+        holder = holder or self.generate_default_holder()
+        with self._locks_dict_lock:
+            if key not in self._locks:
+                lock = threading.Lock()
+                lock.acquire()
+                expiration_timer = None
+                if hold_timeout is not None:
+                    expiration_timer = threading.Timer(
+                        hold_timeout, self._expire_lock, args=(key,)
+                    )
+                    expiration_timer.start()
+                self._locks[key] = _LockInfo(
+                    holder=holder, lock=lock, expiration_timer=expiration_timer
+                )
+                return True
+            elif self._locks[key]["holder"] == holder:
+                return True
+            else:
+                existing_lock_info = self._locks[key]
+
+        if acquire_timeout is not None:
+            existing_lock_acquired = existing_lock_info["lock"].acquire(
+                timeout=acquire_timeout
+            )
+        else:
+            existing_lock_acquired = existing_lock_info["lock"].acquire()
+
+        if existing_lock_acquired:
+            with self._locks_dict_lock:
+                if (
+                    expiration_timer := existing_lock_info["expiration_timer"]
+                ) is not None:
+                    expiration_timer.cancel()
+                expiration_timer = None
+                if hold_timeout is not None:
+                    expiration_timer = threading.Timer(
+                        hold_timeout, self._expire_lock, args=(key,)
+                    )
+                    expiration_timer.start()
+                self._locks[key] = _LockInfo(
+                    holder=holder,
+                    lock=existing_lock_info["lock"],
+                    expiration_timer=expiration_timer,
+                )
+            return True
+        return False
+
+    def release_lock(self, key: str, holder: Optional[str] = None) -> None:
+        holder = holder or self.generate_default_holder()
+        with self._locks_dict_lock:
+            if key in self._locks and self._locks[key]["holder"] == holder:
+                if (
+                    expiration_timer := self._locks[key]["expiration_timer"]
+                ) is not None:
+                    expiration_timer.cancel()
+                self._locks[key]["lock"].release()
+                del self._locks[key]
+            else:
+                raise ValueError(
+                    f"No lock held by {holder} for transaction with key {key}"
+                )
+
+    def is_locked(self, key: str) -> bool:
+        return key in self._locks and self._locks[key]["lock"].locked()
+
+    def is_lock_holder(self, key: str, holder: Optional[str] = None) -> bool:
+        holder = holder or self.generate_default_holder()
+        lock_info = self._locks.get(key)
+        return (
+            lock_info is not None
+            and lock_info["lock"].locked()
+            and lock_info["holder"] == holder
+        )
+
+    def wait_for_lock(self, key: str, timeout: Optional[float] = None) -> bool:
+        if lock := self._locks.get(key, {}).get("lock"):
+            if timeout is not None:
+                lock_acquired = lock.acquire(timeout=timeout)
+            else:
+                lock_acquired = lock.acquire()
+            if lock_acquired:
+                lock.release()
+            return lock_acquired
+        return True

prefect/records/result_store.py

@@ -6,7 +6,7 @@ import pendulum
 from prefect.results import BaseResult, PersistedResult, ResultFactory
 from prefect.utilities.asyncutils import run_coro_as_sync
 
-from .store import RecordStore
+from .base import RecordStore
 
 
 @dataclass

prefect/task_engine.py

@@ -211,6 +211,9 @@ class BaseTaskRunEngine(Generic[P, R]):
         return task_run.state.is_running() or task_run.state.is_scheduled()
 
     def log_finished_message(self):
+        if not self.task_run:
+            return
+
         # If debugging, use the more complete `repr` than the usual `str` description
         display_state = repr(self.state) if PREFECT_DEBUG_MODE else str(self.state)
         level = logging.INFO if self.state.is_completed() else logging.ERROR
@@ -363,7 +366,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
             self.task_run.run_count += 1
 
             flow_run_context = FlowRunContext.get()
-            if flow_run_context:
+            if flow_run_context and flow_run_context.flow_run:
                 # Carry forward any task run information from the flow run
                 flow_run = flow_run_context.flow_run
                 self.task_run.flow_run_run_count = flow_run.run_count
@@ -622,21 +625,24 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
 
             self.logger = task_run_logger(task_run=self.task_run, task=self.task)  # type: ignore
 
-            if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
-                # update the task run name if necessary
-                if not self._task_name_set and self.task.task_run_name:
-                    task_run_name = _resolve_custom_task_run_name(
-                        task=self.task, parameters=self.parameters
-                    )
+            # update the task run name if necessary
+            if not self._task_name_set and self.task.task_run_name:
+                task_run_name = _resolve_custom_task_run_name(
+                    task=self.task, parameters=self.parameters
+                )
+
+                if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
+                    # update the task run name if necessary
                     self.client.set_task_run_name(
                         task_run_id=self.task_run.id, name=task_run_name
                     )
-                    self.logger.extra["task_run_name"] = task_run_name
-                    self.logger.debug(
-                        f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
-                    )
-                    self.task_run.name = task_run_name
-                    self._task_name_set = True
+
+                self.logger.extra["task_run_name"] = task_run_name
+                self.logger.debug(
+                    f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
+                )
+                self.task_run.name = task_run_name
+                self._task_name_set = True
             yield
 
     @contextmanager
@@ -655,21 +661,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                 self._is_started = True
                 try:
                     if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
-                        from prefect.utilities.engine import (
-                            _resolve_custom_task_run_name,
-                        )
-
-                        task_run_name = (
-                            _resolve_custom_task_run_name(
-                                task=self.task, parameters=self.parameters
-                            )
-                            if self.task.task_run_name
-                            else None
-                        )
-
-                        if self.task_run and task_run_name:
-                            self.task_run.name = task_run_name
-
                         if not self.task_run:
                             self.task_run = run_coro_as_sync(
                                 self.task.create_local_run(
@@ -679,7 +670,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                                     parent_task_run_context=TaskRunContext.get(),
                                     wait_for=self.wait_for,
                                     extra_task_inputs=dependencies,
-                                    task_run_name=task_run_name,
                                 )
                             )
                             # Emit an event to capture that the task run was in the `PENDING` state.
@@ -1185,21 +1175,21 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
 
             self.logger = task_run_logger(task_run=self.task_run, task=self.task)  # type: ignore
 
-            if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
-                # update the task run name if necessary
-                if not self._task_name_set and self.task.task_run_name:
-                    task_run_name = _resolve_custom_task_run_name(
-                        task=self.task, parameters=self.parameters
-                    )
+            if not self._task_name_set and self.task.task_run_name:
+                task_run_name = _resolve_custom_task_run_name(
+                    task=self.task, parameters=self.parameters
+                )
+                if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
+                    # update the task run name if necessary
                     await self.client.set_task_run_name(
                         task_run_id=self.task_run.id, name=task_run_name
                     )
-                    self.logger.extra["task_run_name"] = task_run_name
-                    self.logger.debug(
-                        f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
-                    )
-                    self.task_run.name = task_run_name
-                    self._task_name_set = True
+                self.logger.extra["task_run_name"] = task_run_name
+                self.logger.debug(
+                    f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
+                )
+                self.task_run.name = task_run_name
+                self._task_name_set = True
             yield
 
     @asynccontextmanager
@@ -1218,21 +1208,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                 self._is_started = True
                 try:
                     if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
-                        from prefect.utilities.engine import (
-                            _resolve_custom_task_run_name,
-                        )
-
-                        task_run_name = (
-                            _resolve_custom_task_run_name(
-                                task=self.task, parameters=self.parameters
-                            )
-                            if self.task.task_run_name
-                            else None
-                        )
-
-                        if self.task_run and task_run_name:
-                            self.task_run.name = task_run_name
-
                         if not self.task_run:
                             self.task_run = await self.task.create_local_run(
                                 id=task_run_id,
@@ -1241,7 +1216,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                                 parent_task_run_context=TaskRunContext.get(),
                                 wait_for=self.wait_for,
                                 extra_task_inputs=dependencies,
-                                task_run_name=task_run_name,
                             )
                             # Emit an event to capture that the task run was in the `PENDING` state.
                             self._last_event = emit_task_run_state_change_event(

prefect/tasks.py

@@ -814,7 +814,6 @@ class Task(Generic[P, R]):
         wait_for: Optional[Iterable[PrefectFuture]] = None,
         extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
         deferred: bool = False,
-        task_run_name: Optional[str] = None,
     ) -> TaskRun:
         if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
             raise RuntimeError(
@@ -839,12 +838,12 @@ class Task(Generic[P, R]):
         async with client:
             if not flow_run_context:
                 dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
-                task_run_name = task_run_name or self.name
+                task_run_name = self.name
             else:
                 dynamic_key = _dynamic_key_for_task_run(
-                    context=flow_run_context, task=self
+                    context=flow_run_context, task=self, stable=False
                 )
-                task_run_name = task_run_name or f"{self.name}-{dynamic_key}"
+                task_run_name = f"{self.name}-{dynamic_key[:3]}"
 
             if deferred:
                 state = Scheduled()

prefect/transactions.py

@@ -1,3 +1,4 @@
+import copy
 import logging
 from contextlib import contextmanager
 from contextvars import ContextVar, Token
@@ -25,6 +26,7 @@ from prefect.results import (
     ResultFactory,
     get_default_result_storage,
 )
+from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.collections import AutoEnum
 from prefect.utilities.engine import _get_hook_name
@@ -72,8 +74,10 @@ class Transaction(ContextModel):
     def set(self, name: str, value: Any) -> None:
         self._stored_values[name] = value
 
-    def get(self, name: str) -> Any:
+    def get(self, name: str, default: Any = NotSet) -> Any:
         if name not in self._stored_values:
+            if default is not NotSet:
+                return default
             raise ValueError(f"Could not retrieve value for unknown key: {name}")
         return self._stored_values.get(name)
 
@@ -104,6 +108,7 @@ class Transaction(ContextModel):
             # either inherit from parent or set a default of eager
             if parent:
                 self.commit_mode = parent.commit_mode
+                self._stored_values = copy.deepcopy(parent._stored_values)
             else:
                 self.commit_mode = CommitMode.LAZY
 

prefect/utilities/engine.py

@@ -559,8 +559,12 @@ def propose_state_sync(
         )
 
 
-def _dynamic_key_for_task_run(context: FlowRunContext, task: Task) -> Union[int, str]:
-    if context.detached:  # this task is running on remote infrastructure
+def _dynamic_key_for_task_run(
+    context: FlowRunContext, task: Task, stable: bool = True
+) -> Union[int, str]:
+    if (
+        stable is False or context.detached
+    ):  # this task is running on remote infrastructure
         return str(uuid4())
     elif context.flow_run is None:  # this is an autonomous task run
         context.task_run_dynamic_keys[task.task_key] = getattr(

prefect/workers/base.py

@@ -930,7 +930,12 @@ class BaseWorker(abc.ABC):
 
         deployment_vars = deployment.job_variables or {}
         flow_run_vars = flow_run.job_variables or {}
-        job_variables = {**deployment_vars, **flow_run_vars}
+        job_variables = {**deployment_vars}
+
+        # merge environment variables carefully, otherwise full override
+        if isinstance(job_variables.get("env"), dict):
+            job_variables["env"].update(flow_run_vars.pop("env", {}))
+        job_variables.update(flow_run_vars)
 
         configuration = await self.job_configuration.from_template_and_values(
             base_job_template=self._work_pool.base_job_template,

{prefect_client-3.0.0rc16.dist-info → prefect_client-3.0.0rc18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.0.0rc16
+Version: 3.0.0rc18
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.

{prefect_client-3.0.0rc16.dist-info → prefect_client-3.0.0rc18.dist-info}/RECORD

@@ -4,14 +4,14 @@ prefect/_version.py,sha256=I9JsXwt7BjAAbMEZgtmE3a6dJ2jqV-wqWto9D6msb3k,24597
 prefect/agent.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
 prefect/artifacts.py,sha256=wet3coxBtqK0914uTf-slYpXRVP0mjbZn804hXB-RS4,13011
 prefect/automations.py,sha256=NlQ62GPJzy-gnWQqX7c6CQJKw7p60WLGDAFcy82vtg4,5613
-prefect/cache_policies.py,sha256=uEKNGO-PJ3N35B2tjhRDtQULN6ok72D9raIoJaUyXk0,6365
+prefect/cache_policies.py,sha256=779dI23HIPmrtImx9X6eVEH5jc9M7wE9T2isivNo8uM,6570
 prefect/context.py,sha256=Q04o0F1zc9O9y7H0Y6po1hvyajrviYAzuQmpKdNvgbM,21859
 prefect/engine.py,sha256=BpmDbe6miZcTl1vRkxfCPYcWSXADLigGPCagFwucMz0,1976
 prefect/exceptions.py,sha256=3s69Z_IC3HKF6BKxcHrMPXkKdYwfbEfaTjy4-5LOtQ0,11132
 prefect/filesystems.py,sha256=rbFvlqHXeeo71nK1Y5I0-ucmGOYUcdkbb6N2vpsRcWE,17229
 prefect/flow_engine.py,sha256=c8mIffc57zLtHFRo4sVtQOXGihVA_y2mZiXYzjJlOHY,29445
 prefect/flow_runs.py,sha256=EaXRIQTOnwnA0fO7_EjwafFRmS57K_CRy0Xsz3JDIhc,16070
-prefect/flows.py,sha256=Sl3m8Q3mXwY1MSgJSeViiXag4jtUBN7stMP1Fs4iepo,88039
+prefect/flows.py,sha256=9TQZaDYSY3Ao72YHF5pvmEjXD8Z5vgTp2aLEAEWiJ2Y,88326
 prefect/futures.py,sha256=Zt5U7PnNpKUQuyfAhWAZZxpG0hQ6HXuA4KVg6E9sQf8,16208
 prefect/main.py,sha256=bab5nBn37a6gmxdPbTlRS2a9Cf0KY0GaCotDOSbcQ7M,1930
 prefect/manifests.py,sha256=477XcmfdC_yE81wT6zIAKnEUEJ0lH9ZLfOVSgX2FohE,676
@@ -22,12 +22,12 @@ prefect/results.py,sha256=3mVkVWZn_VSQ9Pik79StNy113rB_SEiP83SdoUsFvTM,24635
 prefect/serializers.py,sha256=Lo41EM0_qGzcfB_63390Izeo3DdK6cY6VZfxa9hpSGQ,8712
 prefect/settings.py,sha256=RpO6XMM2qf6KC_rQMHmQpcjul1t2rZmLOkK89Ta3OcM,72076
 prefect/states.py,sha256=lw22xucH46cN9stkxiV9ByIvq689mH5iL3gErri-Y18,22207
-prefect/task_engine.py,sha256=z5raK4OLgCRp6cubrTDL0XZQsx8Z5xTFwKn2QnITGcA,64053
+prefect/task_engine.py,sha256=KIKspqIJpmT_8egEufaDThb1SvDGA9aPwPF5Y87UHe8,62799
 prefect/task_runners.py,sha256=W1n0yMwbDIqnvffFVJADo9MGEbLaYkzWk52rqgnkMY4,15019
 prefect/task_runs.py,sha256=jkaQOkRKOHS8fgHUijteriFpjMSKv4zldn1D8tZHkUI,8777
 prefect/task_worker.py,sha256=DX4NYERghB8RZeFleZE0xOq3yJVunjUaAKHtiz8wuRo,17992
-prefect/tasks.py,sha256=tpYQhH4wZcOeVTOOGus0Med4prLlmMM2jSf3TKuFdUQ,68230
-prefect/transactions.py,sha256=YyWuY99mBeEsY8JGR3_eyWbFIUxWIzbHfU2UMBKTaik,11038
+prefect/tasks.py,sha256=AGh4D8zZhvmmZiixeFjda11RjhAMlK3Myr0ABaIapCQ,68169
+prefect/transactions.py,sha256=6j69ClTNr10XxrsVu9QmsRU6vYU-_14umk_fvAQsBNM,11266
 prefect/variables.py,sha256=-t5LVY0N-K4f0fa6YwruVVQqwnU3fGWBMYXXE32XPkA,4821
 prefect/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 prefect/_internal/_logging.py,sha256=HvNHY-8P469o5u4LYEDBTem69XZEt1QUeUaLToijpak,810
@@ -62,7 +62,7 @@ prefect/_internal/schemas/serializers.py,sha256=G_RGHfObjisUiRvd29p-zc6W4bwt5rE1
 prefect/_internal/schemas/validators.py,sha256=Y8bHb3EsLJTiHsffg_TPbknj0Nmln8vd6qySLFbfGzY,26546
 prefect/blocks/__init__.py,sha256=BUfh6gIwA6HEjRyVCAiv0he3M1zfM-oY-JrlBfeWeY8,182
 prefect/blocks/abstract.py,sha256=YLzCaf3yXv6wFCF5ZqCIHJNwH7fME1rLxC-SijARHzk,16319
-prefect/blocks/core.py,sha256=EAvmPga9fyUfK9QW_HFLSit0dn6Szh_KbDK3n8HX4q0,52155
+prefect/blocks/core.py,sha256=b3cdh-MsEi2BfE8EZ2jLyzf1vPWMmHNp4AboC-vSthc,52165
 prefect/blocks/fields.py,sha256=1m507VVmkpOnMF_7N-qboRjtw4_ceIuDneX3jZ3Jm54,63
 prefect/blocks/notifications.py,sha256=_ARWqq0NHFPxaaW27W6VieBVBwAKELeHtAkHXXP3R_g,29076
 prefect/blocks/redis.py,sha256=GUKYyx2QLtyNvgf5FT_dJxbgQcOzWCja3I23J1-AXhM,5629
@@ -77,12 +77,12 @@ prefect/client/orchestration.py,sha256=G3zkz3HTfy9ZMELKAi8HRqvybOhdlOsia2Vt7ElWE
 prefect/client/subscriptions.py,sha256=J9uK9NGHO4VX4Y3NGgBJ4pIG_0cf-dJWPhF3f3PGYL4,3388
 prefect/client/utilities.py,sha256=89fmza0cRMOayxgXRdO51TKb11TczJ0ByOZmcZVrt44,3286
 prefect/client/schemas/__init__.py,sha256=KlyqFV-hMulMkNstBn_0ijoHoIwJZaBj6B1r07UmgvE,607
-prefect/client/schemas/actions.py,sha256=i0izw0JPdlvAeEC2YEbmskr7kU0-tyzEwEtCKXMPi74,28101
-prefect/client/schemas/filters.py,sha256=_Ibx8VDICS0wkqo-M8zlpOyAbkrRw8sfgqyD9_x2Vvc,34881
-prefect/client/schemas/objects.py,sha256=PNxOTBwQDQ57703i6ZrV_kxYDSuJSrJ3O_42hDwOliA,53296
-prefect/client/schemas/responses.py,sha256=3Y-md6uXCF7X8gcZYvvKqcu5CXqvJfkJyfocHFypp9g,14637
+prefect/client/schemas/actions.py,sha256=LJrioOmj34drliWEMF65_6xZgtv9UxyJgB5ksi1aXhU,28484
+prefect/client/schemas/filters.py,sha256=Pu0wB58_dsNwTeItpQdYII2mwGe0VlLV3ULsuI2PyCg,35648
+prefect/client/schemas/objects.py,sha256=52eJM22Di5C5yuyfbd1mv4j2P6GpAWJlu2b1b8lJmUE,53426
+prefect/client/schemas/responses.py,sha256=J61c8ZqrxfOjKkdf6MLLrRBScsS0tql6Q2SPVr04Ids,14767
 prefect/client/schemas/schedules.py,sha256=8rpqjOYtknu2-1n5_WD4cOplgu93P3mCyX86B22LfL4,13070
-prefect/client/schemas/sorting.py,sha256=EIQ6FUjUWMwk6fn6ckVLQLXOP-GI5kce7ftjUkDFWV0,2490
+prefect/client/schemas/sorting.py,sha256=L-2Mx-igZPtsUoRUguTcG3nIEstMEMPD97NwPM2Ox5s,2579
 prefect/client/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 prefect/client/types/flexible_schedule_list.py,sha256=F1VFAXGLM89dJRBLnVsxwAMGLmrRF2i81FirEMpbB5s,368
 prefect/concurrency/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -135,9 +135,10 @@ prefect/logging/handlers.py,sha256=eIf-0LFH8XUu8Ybnc3LXoocSsa8M8EdAIwbPTVFzZjI,1
 prefect/logging/highlighters.py,sha256=BpSXOy0n3lFVvlKWa7jC-HetAiClFi9jnQtEq5-rgok,1681
 prefect/logging/loggers.py,sha256=xmJ4GIuSXyFV4yMMDK-VokMFGbfcX64PLPyzdM_aV9c,11544
 prefect/logging/logging.yml,sha256=UkEewf0c3_dURI2uCU4RrxkhI5Devoa1s93fl7hilcg,3160
-prefect/records/__init__.py,sha256=7q-lwyevfVgb5S7K9frzawmiJmpZ5ET0m5yXIHBYcVA,31
-prefect/records/result_store.py,sha256=6Yh9zqqXMWjn0qWSfcjQBSfXCM7jVg9pve5TVsOodDc,1734
-prefect/records/store.py,sha256=eQM1p2vZDshXZYg6SkJwL-DP3kUehL_Zgs8xa2-0DZs,224
+prefect/records/__init__.py,sha256=rJJhaJBa0AWu63fJhtB-yHBi64qL6p4svo7F0qvm2sc,30
+prefect/records/base.py,sha256=yJet1guDtA7RWC2nZBvggeI_JF3NaX_7WODDwTauBvs,5545
+prefect/records/memory.py,sha256=dgNNOrib6cbe8mjGELxpyOaHCFFtco_DE9ULK54_Ca0,5768
+prefect/records/result_store.py,sha256=Ac_zlLcKfEzXRh2O0hTP0jmgZTmBRUE_JvG69NxH7Ao,1733
 prefect/runner/__init__.py,sha256=7U-vAOXFkzMfRz1q8Uv6Otsvc0OrPYLLP44srwkJ_8s,89
 prefect/runner/runner.py,sha256=yrBG5fCO7dCpE-lwPOkx8ToBtfJ3-BKQIUm9eNbjA2g,47388
 prefect/runner/server.py,sha256=2o5vhrL7Zbn-HBStWhCjqqViex5Ye9GiQ1EW9RSEzdo,10500
@@ -161,7 +162,7 @@ prefect/utilities/compat.py,sha256=mNQZDnzyKaOqy-OV-DnmH_dc7CNF5nQgW_EsA4xMr7g,9
 prefect/utilities/context.py,sha256=BThuUW94-IYgFYTeMIM9KMo8ShT3oiI7w5ajZHzU1j0,1377
 prefect/utilities/dispatch.py,sha256=c8G-gBo7hZlyoD7my9nO50Rzy8Retk-np5WGq9_E2AM,5856
 prefect/utilities/dockerutils.py,sha256=kRozGQ7JO6Uxl-ljWtDryzxhf96rHL78aHYDh255Em4,20324
-prefect/utilities/engine.py,sha256=KNr6VVPL_EBxMAc06bAV4yHpF4lkaZKndXG6BodW3T0,31659
+prefect/utilities/engine.py,sha256=_rfhPBNuIghCGd1-AN_Csg_Ar046GO3n48KB_EpHHbU,31721
 prefect/utilities/filesystem.py,sha256=frAyy6qOeYa7c-jVbEUGZQEe6J1yF8I_SvUepPd59gI,4415
 prefect/utilities/hashing.py,sha256=EOwZLmoIZImuSTxAvVqInabxJ-4RpEfYeg9e2EDQF8o,1752
 prefect/utilities/importtools.py,sha256=Wo4Tj9hSf7gghP83MxW3w9FK3jkaGKPEobDYjabPqT0,19389
@@ -181,14 +182,14 @@ prefect/utilities/schema_tools/__init__.py,sha256=KsFsTEHQqgp89TkDpjggkgBBywoHQP
 prefect/utilities/schema_tools/hydration.py,sha256=Nitnmr35Mcn5z9NXIvh9DuZW5nCZxpjyMc9RFawMsgs,8376
 prefect/utilities/schema_tools/validation.py,sha256=2GCjxwApTFwzey40ul9OkcAXrU3r-kWK__9ucMo0qbk,9744
 prefect/workers/__init__.py,sha256=8dP8SLZbWYyC_l9DRTQSE3dEbDgns5DZDhxkp_NfsbQ,35
-prefect/workers/base.py,sha256=gjv-ZxwJOSr9mytY5bVwj8rtFriLGacGghQhwcX9hQI,43457
+prefect/workers/base.py,sha256=T1G4EkiJlK7n1PZthmYseuzOr21pQypwZ2lQU1qElFA,43683
 prefect/workers/block.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
 prefect/workers/cloud.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
 prefect/workers/process.py,sha256=t1f1EYRoPL5B25KbLgUX2b5q-lCCAXb2Gpf6T2M9WfY,19822
 prefect/workers/server.py,sha256=lgh2FfSuaNU7b6HPxSFm8JtKvAvHsZGkiOo4y4tW1Cw,2022
 prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
-prefect_client-3.0.0rc16.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
-prefect_client-3.0.0rc16.dist-info/METADATA,sha256=u1Slx4EdzeZnEU0qZ_7V28FsWgCbcfRsjYVCk-1KNcY,7404
-prefect_client-3.0.0rc16.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-prefect_client-3.0.0rc16.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
-prefect_client-3.0.0rc16.dist-info/RECORD,,
+prefect_client-3.0.0rc18.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
+prefect_client-3.0.0rc18.dist-info/METADATA,sha256=-ar0iFrvdZa5j0kUwS1uJn09jfVzrdRmSxSECZq8A5Q,7404
+prefect_client-3.0.0rc18.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+prefect_client-3.0.0rc18.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
+prefect_client-3.0.0rc18.dist-info/RECORD,,

prefect/records/store.py

@@ -1,9 +0,0 @@
-class RecordStore:
-    def read(self, key: str):
-        raise NotImplementedError
-
-    def write(self, key: str, value: dict):
-        raise NotImplementedError
-
-    def exists(self, key: str) -> bool:
-        return False