prefect-client 3.0.0rc11__py3-none-any.whl → 3.0.0rc12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/concurrency/services.py +9 -0
- prefect/artifacts.py +12 -0
- prefect/client/schemas/actions.py +4 -0
- prefect/events/schemas/events.py +10 -0
- prefect/flow_engine.py +5 -2
- prefect/flows.py +17 -5
- prefect/results.py +1 -46
- prefect/runner/runner.py +3 -3
- prefect/settings.py +0 -17
- prefect/task_engine.py +158 -48
- prefect/tasks.py +164 -17
- prefect/transactions.py +2 -31
- prefect/utilities/asyncutils.py +12 -9
- {prefect_client-3.0.0rc11.dist-info → prefect_client-3.0.0rc12.dist-info}/METADATA +2 -2
- {prefect_client-3.0.0rc11.dist-info → prefect_client-3.0.0rc12.dist-info}/RECORD +18 -18
- {prefect_client-3.0.0rc11.dist-info → prefect_client-3.0.0rc12.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc11.dist-info → prefect_client-3.0.0rc12.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc11.dist-info → prefect_client-3.0.0rc12.dist-info}/top_level.txt +0 -0
@@ -151,6 +151,7 @@ class QueueService(abc.ABC, Generic[T]):
|
|
151
151
|
|
152
152
|
if item is None:
|
153
153
|
logger.debug("Exiting service %r", self)
|
154
|
+
self._queue.task_done()
|
154
155
|
break
|
155
156
|
|
156
157
|
try:
|
@@ -164,6 +165,8 @@ class QueueService(abc.ABC, Generic[T]):
|
|
164
165
|
item,
|
165
166
|
exc_info=log_traceback,
|
166
167
|
)
|
168
|
+
finally:
|
169
|
+
self._queue.task_done()
|
167
170
|
|
168
171
|
@abc.abstractmethod
|
169
172
|
async def _handle(self, item: T):
|
@@ -235,6 +238,12 @@ class QueueService(abc.ABC, Generic[T]):
|
|
235
238
|
else:
|
236
239
|
return concurrent.futures.wait(futures, timeout=timeout)
|
237
240
|
|
241
|
+
def wait_until_empty(self):
|
242
|
+
"""
|
243
|
+
Wait until the queue is empty and all items have been processed.
|
244
|
+
"""
|
245
|
+
self._queue.join()
|
246
|
+
|
238
247
|
@classmethod
|
239
248
|
def instance(cls: Type[Self], *args) -> Self:
|
240
249
|
"""
|
prefect/artifacts.py
CHANGED
@@ -6,6 +6,7 @@ from __future__ import annotations
|
|
6
6
|
|
7
7
|
import json # noqa: I001
|
8
8
|
import math
|
9
|
+
import warnings
|
9
10
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
10
11
|
from uuid import UUID
|
11
12
|
|
@@ -54,8 +55,19 @@ class Artifact(ArtifactRequest):
|
|
54
55
|
Returns:
|
55
56
|
- The created artifact.
|
56
57
|
"""
|
58
|
+
from prefect.context import MissingContextError, get_run_context
|
59
|
+
|
57
60
|
client, _ = get_or_create_client(client)
|
58
61
|
task_run_id, flow_run_id = get_task_and_flow_run_ids()
|
62
|
+
|
63
|
+
try:
|
64
|
+
get_run_context()
|
65
|
+
except MissingContextError:
|
66
|
+
warnings.warn(
|
67
|
+
"Artifact creation outside of a flow or task run is deprecated and will be removed in a later version.",
|
68
|
+
FutureWarning,
|
69
|
+
)
|
70
|
+
|
59
71
|
return await client.create_artifact(
|
60
72
|
artifact=ArtifactRequest(
|
61
73
|
type=self.type,
|
@@ -377,6 +377,10 @@ class DeploymentFlowRunCreate(ActionBaseModel):
|
|
377
377
|
parameters: Dict[str, Any] = Field(
|
378
378
|
default_factory=dict, description="The parameters for the flow run."
|
379
379
|
)
|
380
|
+
enforce_parameter_schema: Optional[bool] = Field(
|
381
|
+
default=None,
|
382
|
+
description="Whether or not to enforce the parameter schema on this run.",
|
383
|
+
)
|
380
384
|
context: Dict[str, Any] = Field(
|
381
385
|
default_factory=dict, description="The context for the flow run."
|
382
386
|
)
|
prefect/events/schemas/events.py
CHANGED
@@ -60,6 +60,16 @@ class Resource(Labelled):
|
|
60
60
|
def name(self) -> Optional[str]:
|
61
61
|
return self.get("prefect.resource.name")
|
62
62
|
|
63
|
+
def prefect_object_id(self, kind: str) -> UUID:
|
64
|
+
"""Extracts the UUID from an event's resource ID if it's the expected kind
|
65
|
+
of prefect resource"""
|
66
|
+
prefix = f"{kind}." if not kind.endswith(".") else kind
|
67
|
+
|
68
|
+
if not self.id.startswith(prefix):
|
69
|
+
raise ValueError(f"Resource ID {self.id} does not start with {prefix}")
|
70
|
+
|
71
|
+
return UUID(self.id[len(prefix) :])
|
72
|
+
|
63
73
|
|
64
74
|
class RelatedResource(Resource):
|
65
75
|
"""A Resource with a specific role in an Event"""
|
prefect/flow_engine.py
CHANGED
@@ -91,9 +91,12 @@ def load_flow_and_flow_run(flow_run_id: UUID) -> Tuple[FlowRun, Flow]:
|
|
91
91
|
|
92
92
|
flow_run = client.read_flow_run(flow_run_id)
|
93
93
|
if entrypoint:
|
94
|
-
flow
|
94
|
+
# we should not accept a placeholder flow at runtime
|
95
|
+
flow = load_flow_from_entrypoint(entrypoint, use_placeholder_flow=False)
|
95
96
|
else:
|
96
|
-
flow = run_coro_as_sync(
|
97
|
+
flow = run_coro_as_sync(
|
98
|
+
load_flow_from_flow_run(flow_run, use_placeholder_flow=False)
|
99
|
+
)
|
97
100
|
|
98
101
|
return flow_run, flow
|
99
102
|
|
prefect/flows.py
CHANGED
@@ -798,7 +798,7 @@ class Flow(Generic[P, R]):
|
|
798
798
|
cron: Optional[Union[Iterable[str], str]] = None,
|
799
799
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
800
800
|
paused: Optional[bool] = None,
|
801
|
-
schedules: Optional[
|
801
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
802
802
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
803
803
|
is_schedule_active: Optional[bool] = None,
|
804
804
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
@@ -1704,6 +1704,7 @@ def select_flow(
|
|
1704
1704
|
|
1705
1705
|
def load_flow_from_entrypoint(
|
1706
1706
|
entrypoint: str,
|
1707
|
+
use_placeholder_flow: bool = True,
|
1707
1708
|
) -> Flow:
|
1708
1709
|
"""
|
1709
1710
|
Extract a flow object from a script at an entrypoint by running all of the code in the file.
|
@@ -1711,6 +1712,8 @@ def load_flow_from_entrypoint(
|
|
1711
1712
|
Args:
|
1712
1713
|
entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
|
1713
1714
|
to a flow function
|
1715
|
+
use_placeholder_flow: if True, use a placeholder Flow object if the actual flow object
|
1716
|
+
cannot be loaded from the entrypoint (e.g. dependencies are missing)
|
1714
1717
|
|
1715
1718
|
Returns:
|
1716
1719
|
The flow object from the script
|
@@ -1737,8 +1740,10 @@ def load_flow_from_entrypoint(
|
|
1737
1740
|
# drawback of this approach is that we're unable to actually load the
|
1738
1741
|
# function, so we create a placeholder flow that will re-raise this
|
1739
1742
|
# exception when called.
|
1740
|
-
|
1741
|
-
|
1743
|
+
if use_placeholder_flow:
|
1744
|
+
flow = load_placeholder_flow(entrypoint=entrypoint, raises=exc)
|
1745
|
+
else:
|
1746
|
+
raise
|
1742
1747
|
|
1743
1748
|
if not isinstance(flow, Flow):
|
1744
1749
|
raise MissingFlowError(
|
@@ -1856,6 +1861,7 @@ async def load_flow_from_flow_run(
|
|
1856
1861
|
flow_run: "FlowRun",
|
1857
1862
|
ignore_storage: bool = False,
|
1858
1863
|
storage_base_path: Optional[str] = None,
|
1864
|
+
use_placeholder_flow: bool = True,
|
1859
1865
|
) -> Flow:
|
1860
1866
|
"""
|
1861
1867
|
Load a flow from the location/script provided in a deployment's storage document.
|
@@ -1882,7 +1888,9 @@ async def load_flow_from_flow_run(
|
|
1882
1888
|
f"Importing flow code from module path {deployment.entrypoint}"
|
1883
1889
|
)
|
1884
1890
|
flow = await run_sync_in_worker_thread(
|
1885
|
-
load_flow_from_entrypoint,
|
1891
|
+
load_flow_from_entrypoint,
|
1892
|
+
deployment.entrypoint,
|
1893
|
+
use_placeholder_flow=use_placeholder_flow,
|
1886
1894
|
)
|
1887
1895
|
return flow
|
1888
1896
|
|
@@ -1924,7 +1932,11 @@ async def load_flow_from_flow_run(
|
|
1924
1932
|
import_path = relative_path_to_current_platform(deployment.entrypoint)
|
1925
1933
|
run_logger.debug(f"Importing flow code from '{import_path}'")
|
1926
1934
|
|
1927
|
-
flow = await run_sync_in_worker_thread(
|
1935
|
+
flow = await run_sync_in_worker_thread(
|
1936
|
+
load_flow_from_entrypoint,
|
1937
|
+
str(import_path),
|
1938
|
+
use_placeholder_flow=use_placeholder_flow,
|
1939
|
+
)
|
1928
1940
|
|
1929
1941
|
return flow
|
1930
1942
|
|
prefect/results.py
CHANGED
@@ -25,7 +25,7 @@ from typing_extensions import ParamSpec, Self
|
|
25
25
|
import prefect
|
26
26
|
from prefect.blocks.core import Block
|
27
27
|
from prefect.client.utilities import inject_client
|
28
|
-
from prefect.exceptions import MissingResult
|
28
|
+
from prefect.exceptions import MissingResult
|
29
29
|
from prefect.filesystems import (
|
30
30
|
LocalFileSystem,
|
31
31
|
WritableFileSystem,
|
@@ -64,51 +64,6 @@ R = TypeVar("R")
|
|
64
64
|
_default_storages: Dict[Tuple[str, str], WritableFileSystem] = {}
|
65
65
|
|
66
66
|
|
67
|
-
async def _get_or_create_default_storage(block_document_slug: str) -> ResultStorage:
|
68
|
-
"""
|
69
|
-
Generate a default file system for storage.
|
70
|
-
"""
|
71
|
-
default_storage_name, storage_path = cache_key = (
|
72
|
-
block_document_slug,
|
73
|
-
PREFECT_LOCAL_STORAGE_PATH.value(),
|
74
|
-
)
|
75
|
-
|
76
|
-
async def get_storage() -> WritableFileSystem:
|
77
|
-
try:
|
78
|
-
return await Block.load(default_storage_name)
|
79
|
-
except ValueError as e:
|
80
|
-
if "Unable to find" not in str(e):
|
81
|
-
raise e
|
82
|
-
|
83
|
-
block_type_slug, name = default_storage_name.split("/")
|
84
|
-
if block_type_slug == "local-file-system":
|
85
|
-
block = LocalFileSystem(basepath=storage_path)
|
86
|
-
else:
|
87
|
-
raise ValueError(
|
88
|
-
"The default storage block does not exist, but it is of type "
|
89
|
-
f"'{block_type_slug}' which cannot be created implicitly. Please create "
|
90
|
-
"the block manually."
|
91
|
-
)
|
92
|
-
|
93
|
-
try:
|
94
|
-
await block.save(name, overwrite=False)
|
95
|
-
except ValueError as e:
|
96
|
-
if "already in use" not in str(e):
|
97
|
-
raise e
|
98
|
-
except ObjectAlreadyExists:
|
99
|
-
# Another client created the block before we reached this line
|
100
|
-
block = await Block.load(default_storage_name)
|
101
|
-
|
102
|
-
return block
|
103
|
-
|
104
|
-
try:
|
105
|
-
return _default_storages[cache_key]
|
106
|
-
except KeyError:
|
107
|
-
storage = await get_storage()
|
108
|
-
_default_storages[cache_key] = storage
|
109
|
-
return storage
|
110
|
-
|
111
|
-
|
112
67
|
@sync_compatible
|
113
68
|
async def get_default_result_storage() -> ResultStorage:
|
114
69
|
"""
|
prefect/runner/runner.py
CHANGED
@@ -167,9 +167,7 @@ class Runner:
|
|
167
167
|
self.query_seconds = query_seconds or PREFECT_RUNNER_POLL_FREQUENCY.value()
|
168
168
|
self._prefetch_seconds = prefetch_seconds
|
169
169
|
|
170
|
-
self._limiter: Optional[anyio.CapacityLimiter] =
|
171
|
-
self.limit
|
172
|
-
)
|
170
|
+
self._limiter: Optional[anyio.CapacityLimiter] = None
|
173
171
|
self._client = get_client()
|
174
172
|
self._submitting_flow_run_ids = set()
|
175
173
|
self._cancelling_flow_run_ids = set()
|
@@ -1227,6 +1225,8 @@ class Runner:
|
|
1227
1225
|
self._client = get_client()
|
1228
1226
|
self._tmp_dir.mkdir(parents=True)
|
1229
1227
|
|
1228
|
+
self._limiter = anyio.CapacityLimiter(self.limit)
|
1229
|
+
|
1230
1230
|
if not hasattr(self, "_loop") or not self._loop:
|
1231
1231
|
self._loop = asyncio.get_event_loop()
|
1232
1232
|
|
prefect/settings.py
CHANGED
@@ -481,18 +481,6 @@ PREFECT_HOME = Setting(
|
|
481
481
|
directory may be created automatically when required.
|
482
482
|
"""
|
483
483
|
|
484
|
-
PREFECT_EXTRA_ENTRYPOINTS = Setting(
|
485
|
-
str,
|
486
|
-
default="",
|
487
|
-
)
|
488
|
-
"""
|
489
|
-
Modules for Prefect to import when Prefect is imported.
|
490
|
-
|
491
|
-
Values should be separated by commas, e.g. `my_module,my_other_module`.
|
492
|
-
Objects within modules may be specified by a ':' partition, e.g. `my_module:my_object`.
|
493
|
-
If a callable object is provided, it will be called with no arguments on import.
|
494
|
-
"""
|
495
|
-
|
496
484
|
PREFECT_DEBUG_MODE = Setting(
|
497
485
|
bool,
|
498
486
|
default=False,
|
@@ -1438,11 +1426,6 @@ a task worker should move a task from PENDING to RUNNING very quickly, so runs s
|
|
1438
1426
|
PENDING for a while is a sign that the task worker may have crashed.
|
1439
1427
|
"""
|
1440
1428
|
|
1441
|
-
PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT = Setting(bool, default=False)
|
1442
|
-
"""
|
1443
|
-
Whether or not to disable the sync_compatible decorator utility.
|
1444
|
-
"""
|
1445
|
-
|
1446
1429
|
PREFECT_EXPERIMENTAL_ENABLE_SCHEDULE_CONCURRENCY = Setting(bool, default=False)
|
1447
1430
|
|
1448
1431
|
# Defaults -----------------------------------------------------------------------------
|
prefect/task_engine.py
CHANGED
@@ -5,6 +5,7 @@ import time
|
|
5
5
|
from asyncio import CancelledError
|
6
6
|
from contextlib import ExitStack, contextmanager
|
7
7
|
from dataclasses import dataclass, field
|
8
|
+
from functools import wraps
|
8
9
|
from textwrap import dedent
|
9
10
|
from typing import (
|
10
11
|
Any,
|
@@ -53,6 +54,7 @@ from prefect.records.result_store import ResultFactoryStore
|
|
53
54
|
from prefect.results import BaseResult, ResultFactory, _format_user_supplied_storage_key
|
54
55
|
from prefect.settings import (
|
55
56
|
PREFECT_DEBUG_MODE,
|
57
|
+
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION,
|
56
58
|
PREFECT_TASKS_REFRESH_CACHE,
|
57
59
|
)
|
58
60
|
from prefect.states import (
|
@@ -124,8 +126,7 @@ class TaskRunEngine(Generic[P, R]):
|
|
124
126
|
raise ValueError("Task run is not set")
|
125
127
|
return self.task_run.state
|
126
128
|
|
127
|
-
|
128
|
-
def can_retry(self) -> bool:
|
129
|
+
def can_retry(self, exc: Exception) -> bool:
|
129
130
|
retry_condition: Optional[
|
130
131
|
Callable[[Task[P, Coroutine[Any, Any, R]], TaskRun, State], bool]
|
131
132
|
] = self.task.retry_condition_fn
|
@@ -136,9 +137,19 @@ class TaskRunEngine(Generic[P, R]):
|
|
136
137
|
f"Running `retry_condition_fn` check {retry_condition!r} for task"
|
137
138
|
f" {self.task.name!r}"
|
138
139
|
)
|
139
|
-
|
140
|
-
|
140
|
+
state = Failed(
|
141
|
+
data=exc,
|
142
|
+
message=f"Task run encountered unexpected exception: {repr(exc)}",
|
141
143
|
)
|
144
|
+
if inspect.iscoroutinefunction(retry_condition):
|
145
|
+
should_retry = run_coro_as_sync(
|
146
|
+
retry_condition(self.task, self.task_run, state)
|
147
|
+
)
|
148
|
+
elif inspect.isfunction(retry_condition):
|
149
|
+
should_retry = retry_condition(self.task, self.task_run, state)
|
150
|
+
else:
|
151
|
+
should_retry = not retry_condition
|
152
|
+
return should_retry
|
142
153
|
except Exception:
|
143
154
|
self.logger.error(
|
144
155
|
(
|
@@ -269,6 +280,17 @@ class TaskRunEngine(Generic[P, R]):
|
|
269
280
|
return
|
270
281
|
|
271
282
|
new_state = Running()
|
283
|
+
|
284
|
+
if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
285
|
+
self.task_run.start_time = new_state.timestamp
|
286
|
+
self.task_run.run_count += 1
|
287
|
+
|
288
|
+
flow_run_context = FlowRunContext.get()
|
289
|
+
if flow_run_context:
|
290
|
+
# Carry forward any task run information from the flow run
|
291
|
+
flow_run = flow_run_context.flow_run
|
292
|
+
self.task_run.flow_run_run_count = flow_run.run_count
|
293
|
+
|
272
294
|
state = self.set_state(new_state)
|
273
295
|
|
274
296
|
# TODO: this is temporary until the API stops rejecting state transitions
|
@@ -298,24 +320,37 @@ class TaskRunEngine(Generic[P, R]):
|
|
298
320
|
last_state = self.state
|
299
321
|
if not self.task_run:
|
300
322
|
raise ValueError("Task run is not set")
|
301
|
-
try:
|
302
|
-
new_state = propose_state_sync(
|
303
|
-
self.client, state, task_run_id=self.task_run.id, force=force
|
304
|
-
)
|
305
|
-
except Pause as exc:
|
306
|
-
# We shouldn't get a pause signal without a state, but if this happens,
|
307
|
-
# just use a Paused state to assume an in-process pause.
|
308
|
-
new_state = exc.state if exc.state else Paused()
|
309
|
-
if new_state.state_details.pause_reschedule:
|
310
|
-
# If we're being asked to pause and reschedule, we should exit the
|
311
|
-
# task and expect to be resumed later.
|
312
|
-
raise
|
313
323
|
|
314
|
-
|
315
|
-
|
324
|
+
if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
325
|
+
self.task_run.state = new_state = state
|
326
|
+
|
327
|
+
# Ensure that the state_details are populated with the current run IDs
|
328
|
+
new_state.state_details.task_run_id = self.task_run.id
|
329
|
+
new_state.state_details.flow_run_id = self.task_run.flow_run_id
|
330
|
+
|
331
|
+
# Predictively update the de-normalized task_run.state_* attributes
|
332
|
+
self.task_run.state_id = new_state.id
|
333
|
+
self.task_run.state_type = new_state.type
|
334
|
+
self.task_run.state_name = new_state.name
|
335
|
+
else:
|
336
|
+
try:
|
337
|
+
new_state = propose_state_sync(
|
338
|
+
self.client, state, task_run_id=self.task_run.id, force=force
|
339
|
+
)
|
340
|
+
except Pause as exc:
|
341
|
+
# We shouldn't get a pause signal without a state, but if this happens,
|
342
|
+
# just use a Paused state to assume an in-process pause.
|
343
|
+
new_state = exc.state if exc.state else Paused()
|
344
|
+
if new_state.state_details.pause_reschedule:
|
345
|
+
# If we're being asked to pause and reschedule, we should exit the
|
346
|
+
# task and expect to be resumed later.
|
347
|
+
raise
|
348
|
+
|
349
|
+
# currently this is a hack to keep a reference to the state object
|
350
|
+
# that has an in-memory result attached to it; using the API state
|
351
|
+
# could result in losing that reference
|
352
|
+
self.task_run.state = new_state
|
316
353
|
|
317
|
-
# could result in losing that reference
|
318
|
-
self.task_run.state = new_state
|
319
354
|
# emit a state change event
|
320
355
|
self._last_event = emit_task_run_state_change_event(
|
321
356
|
task_run=self.task_run,
|
@@ -323,6 +358,7 @@ class TaskRunEngine(Generic[P, R]):
|
|
323
358
|
validated_state=self.task_run.state,
|
324
359
|
follows=self._last_event,
|
325
360
|
)
|
361
|
+
|
326
362
|
return new_state
|
327
363
|
|
328
364
|
def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
|
@@ -367,11 +403,19 @@ class TaskRunEngine(Generic[P, R]):
|
|
367
403
|
)
|
368
404
|
transaction.stage(
|
369
405
|
terminal_state.data,
|
370
|
-
on_rollback_hooks=
|
371
|
-
|
406
|
+
on_rollback_hooks=[
|
407
|
+
_with_transaction_hook_logging(hook, "rollback", self.logger)
|
408
|
+
for hook in self.task.on_rollback_hooks
|
409
|
+
],
|
410
|
+
on_commit_hooks=[
|
411
|
+
_with_transaction_hook_logging(hook, "commit", self.logger)
|
412
|
+
for hook in self.task.on_commit_hooks
|
413
|
+
],
|
372
414
|
)
|
373
415
|
if transaction.is_committed():
|
374
416
|
terminal_state.name = "Cached"
|
417
|
+
|
418
|
+
self.record_terminal_state_timing(terminal_state)
|
375
419
|
self.set_state(terminal_state)
|
376
420
|
self._return_value = result
|
377
421
|
return result
|
@@ -383,7 +427,7 @@ class TaskRunEngine(Generic[P, R]):
|
|
383
427
|
- If the task has a retry delay, place in AwaitingRetry state with a delayed scheduled time.
|
384
428
|
- If the task has no retries left, or the retry condition is not met, return False.
|
385
429
|
"""
|
386
|
-
if self.retries < self.task.retries and self.can_retry:
|
430
|
+
if self.retries < self.task.retries and self.can_retry(exc):
|
387
431
|
if self.task.retry_delay_seconds:
|
388
432
|
delay = (
|
389
433
|
self.task.retry_delay_seconds[
|
@@ -398,6 +442,8 @@ class TaskRunEngine(Generic[P, R]):
|
|
398
442
|
else:
|
399
443
|
delay = None
|
400
444
|
new_state = Retrying()
|
445
|
+
if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
446
|
+
self.task_run.run_count += 1
|
401
447
|
|
402
448
|
self.logger.info(
|
403
449
|
"Task run failed with exception: %r - " "Retry %s/%s will start %s",
|
@@ -432,6 +478,7 @@ class TaskRunEngine(Generic[P, R]):
|
|
432
478
|
result_factory=getattr(context, "result_factory", None),
|
433
479
|
)
|
434
480
|
)
|
481
|
+
self.record_terminal_state_timing(state)
|
435
482
|
self.set_state(state)
|
436
483
|
self._raised = exc
|
437
484
|
|
@@ -454,9 +501,20 @@ class TaskRunEngine(Generic[P, R]):
|
|
454
501
|
state = run_coro_as_sync(exception_to_crashed_state(exc))
|
455
502
|
self.logger.error(f"Crash detected! {state.message}")
|
456
503
|
self.logger.debug("Crash details:", exc_info=exc)
|
504
|
+
self.record_terminal_state_timing(state)
|
457
505
|
self.set_state(state, force=True)
|
458
506
|
self._raised = exc
|
459
507
|
|
508
|
+
def record_terminal_state_timing(self, state: State) -> None:
|
509
|
+
if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
510
|
+
if self.task_run.start_time and not self.task_run.end_time:
|
511
|
+
self.task_run.end_time = state.timestamp
|
512
|
+
|
513
|
+
if self.task_run.state.is_running():
|
514
|
+
self.task_run.total_run_time += (
|
515
|
+
state.timestamp - self.task_run.state.timestamp
|
516
|
+
)
|
517
|
+
|
460
518
|
@contextmanager
|
461
519
|
def setup_run_context(self, client: Optional[SyncPrefectClient] = None):
|
462
520
|
from prefect.utilities.engine import (
|
@@ -469,7 +527,8 @@ class TaskRunEngine(Generic[P, R]):
|
|
469
527
|
if not self.task_run:
|
470
528
|
raise ValueError("Task run is not set")
|
471
529
|
|
472
|
-
|
530
|
+
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
531
|
+
self.task_run = client.read_task_run(self.task_run.id)
|
473
532
|
with ExitStack() as stack:
|
474
533
|
if log_prints := should_log_prints(self.task):
|
475
534
|
stack.enter_context(patch_print())
|
@@ -483,23 +542,24 @@ class TaskRunEngine(Generic[P, R]):
|
|
483
542
|
client=client,
|
484
543
|
)
|
485
544
|
)
|
486
|
-
|
545
|
+
|
487
546
|
self.logger = task_run_logger(task_run=self.task_run, task=self.task) # type: ignore
|
488
547
|
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
548
|
+
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
549
|
+
# update the task run name if necessary
|
550
|
+
if not self._task_name_set and self.task.task_run_name:
|
551
|
+
task_run_name = _resolve_custom_task_run_name(
|
552
|
+
task=self.task, parameters=self.parameters
|
553
|
+
)
|
554
|
+
self.client.set_task_run_name(
|
555
|
+
task_run_id=self.task_run.id, name=task_run_name
|
556
|
+
)
|
557
|
+
self.logger.extra["task_run_name"] = task_run_name
|
558
|
+
self.logger.debug(
|
559
|
+
f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
|
560
|
+
)
|
561
|
+
self.task_run.name = task_run_name
|
562
|
+
self._task_name_set = True
|
503
563
|
yield
|
504
564
|
|
505
565
|
@contextmanager
|
@@ -511,22 +571,47 @@ class TaskRunEngine(Generic[P, R]):
|
|
511
571
|
"""
|
512
572
|
Enters a client context and creates a task run if needed.
|
513
573
|
"""
|
574
|
+
|
514
575
|
with hydrated_context(self.context):
|
515
576
|
with ClientContext.get_or_create() as client_ctx:
|
516
577
|
self._client = client_ctx.sync_client
|
517
578
|
self._is_started = True
|
518
579
|
try:
|
519
580
|
if not self.task_run:
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
581
|
+
if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
582
|
+
# TODO - this maybe should be a method on Task?
|
583
|
+
from prefect.utilities.engine import (
|
584
|
+
_resolve_custom_task_run_name,
|
585
|
+
)
|
586
|
+
|
587
|
+
task_run_name = None
|
588
|
+
if not self._task_name_set and self.task.task_run_name:
|
589
|
+
task_run_name = _resolve_custom_task_run_name(
|
590
|
+
task=self.task, parameters=self.parameters
|
591
|
+
)
|
592
|
+
|
593
|
+
self.task_run = run_coro_as_sync(
|
594
|
+
self.task.create_local_run(
|
595
|
+
id=task_run_id,
|
596
|
+
parameters=self.parameters,
|
597
|
+
flow_run_context=FlowRunContext.get(),
|
598
|
+
parent_task_run_context=TaskRunContext.get(),
|
599
|
+
wait_for=self.wait_for,
|
600
|
+
extra_task_inputs=dependencies,
|
601
|
+
task_run_name=task_run_name,
|
602
|
+
)
|
603
|
+
)
|
604
|
+
else:
|
605
|
+
self.task_run = run_coro_as_sync(
|
606
|
+
self.task.create_run(
|
607
|
+
id=task_run_id,
|
608
|
+
parameters=self.parameters,
|
609
|
+
flow_run_context=FlowRunContext.get(),
|
610
|
+
parent_task_run_context=TaskRunContext.get(),
|
611
|
+
wait_for=self.wait_for,
|
612
|
+
extra_task_inputs=dependencies,
|
613
|
+
)
|
528
614
|
)
|
529
|
-
)
|
530
615
|
# Emit an event to capture that the task run was in the `PENDING` state.
|
531
616
|
self._last_event = emit_task_run_state_change_event(
|
532
617
|
task_run=self.task_run,
|
@@ -916,3 +1001,28 @@ def run_task(
|
|
916
1001
|
return run_task_async(**kwargs)
|
917
1002
|
else:
|
918
1003
|
return run_task_sync(**kwargs)
|
1004
|
+
|
1005
|
+
|
1006
|
+
def _with_transaction_hook_logging(
|
1007
|
+
hook: Callable[[Transaction], None],
|
1008
|
+
hook_type: Literal["rollback", "commit"],
|
1009
|
+
logger: logging.Logger,
|
1010
|
+
) -> Callable[[Transaction], None]:
|
1011
|
+
@wraps(hook)
|
1012
|
+
def _hook(txn: Transaction) -> None:
|
1013
|
+
hook_name = _get_hook_name(hook)
|
1014
|
+
logger.info(f"Running {hook_type} hook {hook_name!r}")
|
1015
|
+
|
1016
|
+
try:
|
1017
|
+
hook(txn)
|
1018
|
+
except Exception as exc:
|
1019
|
+
logger.error(
|
1020
|
+
f"An error was encountered while running {hook_type} hook {hook_name!r}",
|
1021
|
+
)
|
1022
|
+
raise exc
|
1023
|
+
else:
|
1024
|
+
logger.info(
|
1025
|
+
f"{hook_type.capitalize()} hook {hook_name!r} finished running successfully"
|
1026
|
+
)
|
1027
|
+
|
1028
|
+
return _hook
|
prefect/tasks.py
CHANGED
@@ -6,7 +6,6 @@ Module containing the base workflow task class and decorator - for most use case
|
|
6
6
|
|
7
7
|
import datetime
|
8
8
|
import inspect
|
9
|
-
import os
|
10
9
|
from copy import copy
|
11
10
|
from functools import partial, update_wrapper
|
12
11
|
from typing import (
|
@@ -33,13 +32,19 @@ from uuid import UUID, uuid4
|
|
33
32
|
|
34
33
|
from typing_extensions import Literal, ParamSpec
|
35
34
|
|
35
|
+
import prefect.states
|
36
36
|
from prefect._internal.compatibility.deprecated import (
|
37
37
|
deprecated_async_method,
|
38
38
|
)
|
39
39
|
from prefect.cache_policies import DEFAULT, NONE, CachePolicy
|
40
40
|
from prefect.client.orchestration import get_client
|
41
41
|
from prefect.client.schemas import TaskRun
|
42
|
-
from prefect.client.schemas.objects import
|
42
|
+
from prefect.client.schemas.objects import (
|
43
|
+
StateDetails,
|
44
|
+
TaskRunInput,
|
45
|
+
TaskRunPolicy,
|
46
|
+
TaskRunResult,
|
47
|
+
)
|
43
48
|
from prefect.context import (
|
44
49
|
FlowRunContext,
|
45
50
|
TagsContext,
|
@@ -50,6 +55,7 @@ from prefect.futures import PrefectDistributedFuture, PrefectFuture, PrefectFutu
|
|
50
55
|
from prefect.logging.loggers import get_logger
|
51
56
|
from prefect.results import ResultFactory, ResultSerializer, ResultStorage
|
52
57
|
from prefect.settings import (
|
58
|
+
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION,
|
53
59
|
PREFECT_TASK_DEFAULT_RETRIES,
|
54
60
|
PREFECT_TASK_DEFAULT_RETRY_DELAY_SECONDS,
|
55
61
|
)
|
@@ -181,6 +187,31 @@ def _infer_parent_task_runs(
|
|
181
187
|
return parents
|
182
188
|
|
183
189
|
|
190
|
+
def _generate_task_key(fn: Callable[..., Any]) -> str:
|
191
|
+
"""Generate a task key based on the function name and source code.
|
192
|
+
|
193
|
+
We may eventually want some sort of top-level namespace here to
|
194
|
+
disambiguate tasks with the same function name in different modules,
|
195
|
+
in a more human-readable way, while avoiding relative import problems (see #12337).
|
196
|
+
|
197
|
+
As long as the task implementations are unique (even if named the same), we should
|
198
|
+
not have any collisions.
|
199
|
+
|
200
|
+
Args:
|
201
|
+
fn: The function to generate a task key for.
|
202
|
+
"""
|
203
|
+
if not hasattr(fn, "__qualname__"):
|
204
|
+
return to_qualified_name(type(fn))
|
205
|
+
|
206
|
+
qualname = fn.__qualname__.split(".")[-1]
|
207
|
+
|
208
|
+
code_hash = (
|
209
|
+
h[:NUM_CHARS_DYNAMIC_KEY] if (h := hash_objects(fn.__code__)) else "unknown"
|
210
|
+
)
|
211
|
+
|
212
|
+
return f"{qualname}-{code_hash}"
|
213
|
+
|
214
|
+
|
184
215
|
class Task(Generic[P, R]):
|
185
216
|
"""
|
186
217
|
A Prefect task definition.
|
@@ -263,7 +294,7 @@ class Task(Generic[P, R]):
|
|
263
294
|
description: Optional[str] = None,
|
264
295
|
tags: Optional[Iterable[str]] = None,
|
265
296
|
version: Optional[str] = None,
|
266
|
-
cache_policy:
|
297
|
+
cache_policy: Union[CachePolicy, Type[NotSet]] = NotSet,
|
267
298
|
cache_key_fn: Optional[
|
268
299
|
Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
|
269
300
|
] = None,
|
@@ -362,17 +393,7 @@ class Task(Generic[P, R]):
|
|
362
393
|
|
363
394
|
self.tags = set(tags if tags else [])
|
364
395
|
|
365
|
-
|
366
|
-
self.task_key = to_qualified_name(type(self.fn))
|
367
|
-
else:
|
368
|
-
try:
|
369
|
-
task_origin_hash = hash_objects(
|
370
|
-
self.name, os.path.abspath(inspect.getsourcefile(self.fn))
|
371
|
-
)
|
372
|
-
except TypeError:
|
373
|
-
task_origin_hash = "unknown-source-file"
|
374
|
-
|
375
|
-
self.task_key = f"{self.fn.__qualname__}-{task_origin_hash}"
|
396
|
+
self.task_key = _generate_task_key(self.fn)
|
376
397
|
|
377
398
|
if cache_policy is not NotSet and cache_key_fn is not None:
|
378
399
|
logger.warning(
|
@@ -786,6 +807,130 @@ class Task(Generic[P, R]):
|
|
786
807
|
|
787
808
|
return task_run
|
788
809
|
|
810
|
+
async def create_local_run(
|
811
|
+
self,
|
812
|
+
client: Optional["PrefectClient"] = None,
|
813
|
+
id: Optional[UUID] = None,
|
814
|
+
parameters: Optional[Dict[str, Any]] = None,
|
815
|
+
flow_run_context: Optional[FlowRunContext] = None,
|
816
|
+
parent_task_run_context: Optional[TaskRunContext] = None,
|
817
|
+
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
818
|
+
extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
|
819
|
+
deferred: bool = False,
|
820
|
+
task_run_name: Optional[str] = None,
|
821
|
+
) -> TaskRun:
|
822
|
+
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
823
|
+
raise RuntimeError(
|
824
|
+
"Cannot call `Task.create_local_run` unless "
|
825
|
+
"PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION is True"
|
826
|
+
)
|
827
|
+
|
828
|
+
from prefect.utilities.engine import (
|
829
|
+
_dynamic_key_for_task_run,
|
830
|
+
collect_task_run_inputs_sync,
|
831
|
+
)
|
832
|
+
|
833
|
+
if flow_run_context is None:
|
834
|
+
flow_run_context = FlowRunContext.get()
|
835
|
+
if parent_task_run_context is None:
|
836
|
+
parent_task_run_context = TaskRunContext.get()
|
837
|
+
if parameters is None:
|
838
|
+
parameters = {}
|
839
|
+
if client is None:
|
840
|
+
client = get_client()
|
841
|
+
|
842
|
+
async with client:
|
843
|
+
if not flow_run_context:
|
844
|
+
dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
|
845
|
+
task_run_name = task_run_name or self.name
|
846
|
+
else:
|
847
|
+
dynamic_key = _dynamic_key_for_task_run(
|
848
|
+
context=flow_run_context, task=self
|
849
|
+
)
|
850
|
+
task_run_name = task_run_name or f"{self.name}-{dynamic_key}"
|
851
|
+
|
852
|
+
if deferred:
|
853
|
+
state = Scheduled()
|
854
|
+
state.state_details.deferred = True
|
855
|
+
else:
|
856
|
+
state = Pending()
|
857
|
+
|
858
|
+
# store parameters for background tasks so that task worker
|
859
|
+
# can retrieve them at runtime
|
860
|
+
if deferred and (parameters or wait_for):
|
861
|
+
parameters_id = uuid4()
|
862
|
+
state.state_details.task_parameters_id = parameters_id
|
863
|
+
|
864
|
+
# TODO: Improve use of result storage for parameter storage / reference
|
865
|
+
self.persist_result = True
|
866
|
+
|
867
|
+
factory = await ResultFactory.from_autonomous_task(self, client=client)
|
868
|
+
context = serialize_context()
|
869
|
+
data: Dict[str, Any] = {"context": context}
|
870
|
+
if parameters:
|
871
|
+
data["parameters"] = parameters
|
872
|
+
if wait_for:
|
873
|
+
data["wait_for"] = wait_for
|
874
|
+
await factory.store_parameters(parameters_id, data)
|
875
|
+
|
876
|
+
# collect task inputs
|
877
|
+
task_inputs = {
|
878
|
+
k: collect_task_run_inputs_sync(v) for k, v in parameters.items()
|
879
|
+
}
|
880
|
+
|
881
|
+
# collect all parent dependencies
|
882
|
+
if task_parents := _infer_parent_task_runs(
|
883
|
+
flow_run_context=flow_run_context,
|
884
|
+
task_run_context=parent_task_run_context,
|
885
|
+
parameters=parameters,
|
886
|
+
):
|
887
|
+
task_inputs["__parents__"] = task_parents
|
888
|
+
|
889
|
+
# check wait for dependencies
|
890
|
+
if wait_for:
|
891
|
+
task_inputs["wait_for"] = collect_task_run_inputs_sync(wait_for)
|
892
|
+
|
893
|
+
# Join extra task inputs
|
894
|
+
for k, extras in (extra_task_inputs or {}).items():
|
895
|
+
task_inputs[k] = task_inputs[k].union(extras)
|
896
|
+
|
897
|
+
flow_run_id = (
|
898
|
+
getattr(flow_run_context.flow_run, "id", None)
|
899
|
+
if flow_run_context and flow_run_context.flow_run
|
900
|
+
else None
|
901
|
+
)
|
902
|
+
task_run_id = id or uuid4()
|
903
|
+
state = prefect.states.Pending(
|
904
|
+
state_details=StateDetails(
|
905
|
+
task_run_id=task_run_id,
|
906
|
+
flow_run_id=flow_run_id,
|
907
|
+
)
|
908
|
+
)
|
909
|
+
task_run = TaskRun(
|
910
|
+
id=task_run_id,
|
911
|
+
name=task_run_name,
|
912
|
+
flow_run_id=flow_run_id,
|
913
|
+
task_key=self.task_key,
|
914
|
+
dynamic_key=str(dynamic_key),
|
915
|
+
task_version=self.version,
|
916
|
+
empirical_policy=TaskRunPolicy(
|
917
|
+
retries=self.retries,
|
918
|
+
retry_delay=self.retry_delay_seconds,
|
919
|
+
retry_jitter_factor=self.retry_jitter_factor,
|
920
|
+
),
|
921
|
+
tags=list(set(self.tags).union(TagsContext.get().current_tags or [])),
|
922
|
+
task_inputs=task_inputs or {},
|
923
|
+
expected_start_time=state.timestamp,
|
924
|
+
state_id=state.id,
|
925
|
+
state_type=state.type,
|
926
|
+
state_name=state.name,
|
927
|
+
state=state,
|
928
|
+
created=state.timestamp,
|
929
|
+
updated=state.timestamp,
|
930
|
+
)
|
931
|
+
|
932
|
+
return task_run
|
933
|
+
|
789
934
|
@overload
|
790
935
|
def __call__(
|
791
936
|
self: "Task[P, NoReturn]",
|
@@ -1365,7 +1510,7 @@ class Task(Generic[P, R]):
|
|
1365
1510
|
|
1366
1511
|
Args:
|
1367
1512
|
task_runner: The task runner to use for serving the task. If not provided,
|
1368
|
-
the default
|
1513
|
+
the default task runner will be used.
|
1369
1514
|
|
1370
1515
|
Examples:
|
1371
1516
|
Serve a task using the default task runner
|
@@ -1392,7 +1537,7 @@ def task(
|
|
1392
1537
|
description: Optional[str] = None,
|
1393
1538
|
tags: Optional[Iterable[str]] = None,
|
1394
1539
|
version: Optional[str] = None,
|
1395
|
-
cache_policy: CachePolicy = NotSet,
|
1540
|
+
cache_policy: Union[CachePolicy, Type[NotSet]] = NotSet,
|
1396
1541
|
cache_key_fn: Optional[
|
1397
1542
|
Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
|
1398
1543
|
] = None,
|
@@ -1430,7 +1575,9 @@ def task(
|
|
1430
1575
|
tags: Optional[Iterable[str]] = None,
|
1431
1576
|
version: Optional[str] = None,
|
1432
1577
|
cache_policy: Union[CachePolicy, Type[NotSet]] = NotSet,
|
1433
|
-
cache_key_fn:
|
1578
|
+
cache_key_fn: Union[
|
1579
|
+
Callable[["TaskRunContext", Dict[str, Any]], Optional[str]], None
|
1580
|
+
] = None,
|
1434
1581
|
cache_expiration: Optional[datetime.timedelta] = None,
|
1435
1582
|
task_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1436
1583
|
retries: Optional[int] = None,
|
prefect/transactions.py
CHANGED
@@ -26,7 +26,6 @@ from prefect.results import (
|
|
26
26
|
)
|
27
27
|
from prefect.utilities.asyncutils import run_coro_as_sync
|
28
28
|
from prefect.utilities.collections import AutoEnum
|
29
|
-
from prefect.utilities.engine import _get_hook_name
|
30
29
|
|
31
30
|
|
32
31
|
class IsolationLevel(AutoEnum):
|
@@ -180,39 +179,20 @@ class Transaction(ContextModel):
|
|
180
179
|
return False
|
181
180
|
|
182
181
|
try:
|
183
|
-
hook_name = None
|
184
|
-
|
185
182
|
for child in self.children:
|
186
183
|
child.commit()
|
187
184
|
|
188
185
|
for hook in self.on_commit_hooks:
|
189
|
-
hook_name = _get_hook_name(hook)
|
190
|
-
if self.logger:
|
191
|
-
self.logger.info(f"Running commit hook {hook_name!r}")
|
192
|
-
|
193
186
|
hook(self)
|
194
187
|
|
195
|
-
if self.logger:
|
196
|
-
self.logger.info(
|
197
|
-
f"Commit hook {hook_name!r} finished running successfully"
|
198
|
-
)
|
199
|
-
|
200
188
|
if self.store and self.key:
|
201
189
|
self.store.write(key=self.key, value=self._staged_value)
|
202
190
|
self.state = TransactionState.COMMITTED
|
203
191
|
return True
|
204
192
|
except Exception:
|
205
193
|
if self.logger:
|
206
|
-
if hook_name:
|
207
|
-
msg = (
|
208
|
-
f"An error was encountered while running commit hook {hook_name!r}",
|
209
|
-
)
|
210
|
-
else:
|
211
|
-
msg = (
|
212
|
-
f"An error was encountered while committing transaction {self.key!r}",
|
213
|
-
)
|
214
194
|
self.logger.exception(
|
215
|
-
|
195
|
+
f"An error was encountered while committing transaction {self.key!r}",
|
216
196
|
exc_info=True,
|
217
197
|
)
|
218
198
|
self.rollback()
|
@@ -242,17 +222,8 @@ class Transaction(ContextModel):
|
|
242
222
|
|
243
223
|
try:
|
244
224
|
for hook in reversed(self.on_rollback_hooks):
|
245
|
-
hook_name = _get_hook_name(hook)
|
246
|
-
if self.logger:
|
247
|
-
self.logger.info(f"Running rollback hook {hook_name!r}")
|
248
|
-
|
249
225
|
hook(self)
|
250
226
|
|
251
|
-
if self.logger:
|
252
|
-
self.logger.info(
|
253
|
-
f"Rollback hook {hook_name!r} finished running successfully"
|
254
|
-
)
|
255
|
-
|
256
227
|
self.state = TransactionState.ROLLED_BACK
|
257
228
|
|
258
229
|
for child in reversed(self.children):
|
@@ -262,7 +233,7 @@ class Transaction(ContextModel):
|
|
262
233
|
except Exception:
|
263
234
|
if self.logger:
|
264
235
|
self.logger.exception(
|
265
|
-
f"An error was encountered while
|
236
|
+
f"An error was encountered while rolling back transaction {self.key!r}",
|
266
237
|
exc_info=True,
|
267
238
|
)
|
268
239
|
return False
|
prefect/utilities/asyncutils.py
CHANGED
@@ -267,11 +267,17 @@ async def run_sync_in_worker_thread(
|
|
267
267
|
Note that cancellation of threads will not result in interrupted computation, the
|
268
268
|
thread may continue running — the outcome will just be ignored.
|
269
269
|
"""
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
270
|
+
# When running a sync function in a worker thread, we set this flag so that
|
271
|
+
# any root sync compatible functions will run as sync functions
|
272
|
+
token = RUNNING_ASYNC_FLAG.set(False)
|
273
|
+
try:
|
274
|
+
call = partial(__fn, *args, **kwargs)
|
275
|
+
result = await anyio.to_thread.run_sync(
|
276
|
+
call_with_mark, call, abandon_on_cancel=True, limiter=get_thread_limiter()
|
277
|
+
)
|
278
|
+
return result
|
279
|
+
finally:
|
280
|
+
RUNNING_ASYNC_FLAG.reset(token)
|
275
281
|
|
276
282
|
|
277
283
|
def call_with_mark(call):
|
@@ -348,11 +354,8 @@ def sync_compatible(
|
|
348
354
|
*args: Any, _sync: Optional[bool] = None, **kwargs: Any
|
349
355
|
) -> Union[R, Coroutine[Any, Any, R]]:
|
350
356
|
from prefect.context import MissingContextError, get_run_context
|
351
|
-
from prefect.settings import (
|
352
|
-
PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT,
|
353
|
-
)
|
354
357
|
|
355
|
-
if
|
358
|
+
if _sync is False:
|
356
359
|
return async_fn(*args, **kwargs)
|
357
360
|
|
358
361
|
is_async = True
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: prefect-client
|
3
|
-
Version: 3.0.
|
3
|
+
Version: 3.0.0rc12
|
4
4
|
Summary: Workflow orchestration and management.
|
5
5
|
Home-page: https://www.prefect.io
|
6
6
|
Author: Prefect Technologies, Inc.
|
@@ -29,7 +29,7 @@ Requires-Dist: asgi-lifespan <3.0,>=1.0
|
|
29
29
|
Requires-Dist: cachetools <6.0,>=5.3
|
30
30
|
Requires-Dist: cloudpickle <4.0,>=2.0
|
31
31
|
Requires-Dist: coolname <3.0.0,>=1.0.4
|
32
|
-
Requires-Dist: croniter <
|
32
|
+
Requires-Dist: croniter <4.0.0,>=1.0.12
|
33
33
|
Requires-Dist: exceptiongroup >=1.0.0
|
34
34
|
Requires-Dist: fastapi <1.0.0,>=0.111.0
|
35
35
|
Requires-Dist: fsspec >=2022.5.0
|
@@ -2,32 +2,32 @@ prefect/.prefectignore,sha256=awSprvKT0vI8a64mEOLrMxhxqcO-b0ERQeYpA2rNKVQ,390
|
|
2
2
|
prefect/__init__.py,sha256=rFlBikby0TcAmljqECcleQE_se15eh1gLp5iac0ZhsU,3301
|
3
3
|
prefect/_version.py,sha256=I9JsXwt7BjAAbMEZgtmE3a6dJ2jqV-wqWto9D6msb3k,24597
|
4
4
|
prefect/agent.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
5
|
-
prefect/artifacts.py,sha256=
|
5
|
+
prefect/artifacts.py,sha256=wet3coxBtqK0914uTf-slYpXRVP0mjbZn804hXB-RS4,13011
|
6
6
|
prefect/automations.py,sha256=NlQ62GPJzy-gnWQqX7c6CQJKw7p60WLGDAFcy82vtg4,5613
|
7
7
|
prefect/cache_policies.py,sha256=uEKNGO-PJ3N35B2tjhRDtQULN6ok72D9raIoJaUyXk0,6365
|
8
8
|
prefect/context.py,sha256=40FLSXI3Qd9dMwP8nQ7fGZFKpQIuEkjTpekT4V1MvF8,19942
|
9
9
|
prefect/engine.py,sha256=BpmDbe6miZcTl1vRkxfCPYcWSXADLigGPCagFwucMz0,1976
|
10
10
|
prefect/exceptions.py,sha256=3s69Z_IC3HKF6BKxcHrMPXkKdYwfbEfaTjy4-5LOtQ0,11132
|
11
11
|
prefect/filesystems.py,sha256=rbFvlqHXeeo71nK1Y5I0-ucmGOYUcdkbb6N2vpsRcWE,17229
|
12
|
-
prefect/flow_engine.py,sha256=
|
12
|
+
prefect/flow_engine.py,sha256=oNsRxLIZjjf-PyYl5m2cxWk1iRONI-U_KBZESfS4EAw,29328
|
13
13
|
prefect/flow_runs.py,sha256=EaXRIQTOnwnA0fO7_EjwafFRmS57K_CRy0Xsz3JDIhc,16070
|
14
|
-
prefect/flows.py,sha256=
|
14
|
+
prefect/flows.py,sha256=jskTdjUxH7Le1n3ME3TbkytRXrWs_B1LVRUy2-xOixk,84721
|
15
15
|
prefect/futures.py,sha256=w5M_iZwt5aI0AUfia0JC1FkitRmQ6Oxtc_L7g_INTOM,13695
|
16
16
|
prefect/main.py,sha256=bab5nBn37a6gmxdPbTlRS2a9Cf0KY0GaCotDOSbcQ7M,1930
|
17
17
|
prefect/manifests.py,sha256=477XcmfdC_yE81wT6zIAKnEUEJ0lH9ZLfOVSgX2FohE,676
|
18
18
|
prefect/plugins.py,sha256=7AICJzHIu8iAeF9vl9wAYm28pR_k7dkdnm3OyJRfCv4,2229
|
19
19
|
prefect/profiles.toml,sha256=Fs8hD_BdWHZgAijgk8pK_Zx-Pm-YFixqDIfEP6fM-qU,38
|
20
20
|
prefect/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
21
|
-
prefect/results.py,sha256=
|
21
|
+
prefect/results.py,sha256=3mVkVWZn_VSQ9Pik79StNy113rB_SEiP83SdoUsFvTM,24635
|
22
22
|
prefect/serializers.py,sha256=Lo41EM0_qGzcfB_63390Izeo3DdK6cY6VZfxa9hpSGQ,8712
|
23
|
-
prefect/settings.py,sha256=
|
23
|
+
prefect/settings.py,sha256=Xmgy4S_zQWKiT7VEXiRtSjhjihrqK5feNkH3QVdDTMQ,69090
|
24
24
|
prefect/states.py,sha256=lw22xucH46cN9stkxiV9ByIvq689mH5iL3gErri-Y18,22207
|
25
|
-
prefect/task_engine.py,sha256=
|
25
|
+
prefect/task_engine.py,sha256=GXWIDMNUl0--RecH9yc41mSSpCNnIFmapi8fMIFDsdQ,39911
|
26
26
|
prefect/task_runners.py,sha256=W1n0yMwbDIqnvffFVJADo9MGEbLaYkzWk52rqgnkMY4,15019
|
27
27
|
prefect/task_runs.py,sha256=eDWYH5H1K4SyduhKmn3GzO6vM3fZSwOZxAb8KhkMGsk,7798
|
28
28
|
prefect/task_worker.py,sha256=lV9rQb9YOaO28DZLW_avw6p0pTSVYtsA1gqODWxB7J0,17334
|
29
|
-
prefect/tasks.py,sha256=
|
30
|
-
prefect/transactions.py,sha256=
|
29
|
+
prefect/tasks.py,sha256=A-sQB8S5dxcLtl10crDThsfdKpMzm9Rknfwhu27rnvY,68085
|
30
|
+
prefect/transactions.py,sha256=XBbOjAUnDWw9QcxVwEamRaWxvRA_Ao-MkIN5dFL7h54,10008
|
31
31
|
prefect/variables.py,sha256=-t5LVY0N-K4f0fa6YwruVVQqwnU3fGWBMYXXE32XPkA,4821
|
32
32
|
prefect/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
33
|
prefect/_internal/_logging.py,sha256=HvNHY-8P469o5u4LYEDBTem69XZEt1QUeUaLToijpak,810
|
@@ -45,7 +45,7 @@ prefect/_internal/concurrency/cancellation.py,sha256=D1B_I2cBSGPNXcLaXNaLN_9_QAg
|
|
45
45
|
prefect/_internal/concurrency/event_loop.py,sha256=1VBW862QZ6DV9dExWOT398A0fti4A7jW2kcY7Y5V-lI,2073
|
46
46
|
prefect/_internal/concurrency/inspection.py,sha256=xfyUNr5CoES8LFhybi2DmzHeW4RpTAbrGPiZOYMVLbQ,3464
|
47
47
|
prefect/_internal/concurrency/primitives.py,sha256=BQ0vObO7NUEq-IMbu5aTlfoZpWMbYwinzYP89GIyw68,2609
|
48
|
-
prefect/_internal/concurrency/services.py,sha256=
|
48
|
+
prefect/_internal/concurrency/services.py,sha256=6RYcGFrHh_xO5BXeRDgNOmWsjQnQrX9jyW7SN9zuOF0,12209
|
49
49
|
prefect/_internal/concurrency/threads.py,sha256=90Wi00pTebfihiFgP-P71DeGTnhlJKctzP69mtjphKU,8860
|
50
50
|
prefect/_internal/concurrency/waiters.py,sha256=X6xxsKcHB9ULnCkeUf0cLTI267kI_GC4k96XRuhPhnw,8790
|
51
51
|
prefect/_internal/pydantic/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
@@ -77,7 +77,7 @@ prefect/client/orchestration.py,sha256=W3tiqjND1lf0GtunLBayMRrUD5ykAcW0GfwxqT9fT
|
|
77
77
|
prefect/client/subscriptions.py,sha256=J9uK9NGHO4VX4Y3NGgBJ4pIG_0cf-dJWPhF3f3PGYL4,3388
|
78
78
|
prefect/client/utilities.py,sha256=Qh1WdKLs8F_GuA04FeZ1GJsPYtiCN4DjKmLEaMfKmpQ,3264
|
79
79
|
prefect/client/schemas/__init__.py,sha256=KlyqFV-hMulMkNstBn_0ijoHoIwJZaBj6B1r07UmgvE,607
|
80
|
-
prefect/client/schemas/actions.py,sha256=
|
80
|
+
prefect/client/schemas/actions.py,sha256=sZFVfN-Xks7x-sfp0kX97lSzHDbMx99E-rzIngOUU6A,28184
|
81
81
|
prefect/client/schemas/filters.py,sha256=HyIYZQowhkHa_D6syj83zUp5uFEzA8UADLaS9mt1MTo,35305
|
82
82
|
prefect/client/schemas/objects.py,sha256=3-qhF8qTUSB-wax4s5_zBs6A1K2hdDr1WLxpTryQbRE,53547
|
83
83
|
prefect/client/schemas/responses.py,sha256=xW9QKmVgBftSEmtuSr5gp9HBFvIDzF6aSFq-mhv7dE8,14948
|
@@ -114,7 +114,7 @@ prefect/events/cli/automations.py,sha256=WIZ3-EcDibjQB5BrMEx7OZ7UfOqP8VjCI1dNh64
|
|
114
114
|
prefect/events/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
115
115
|
prefect/events/schemas/automations.py,sha256=he9_v0Oq8AtCJe5gMti5GDQiaGa50sM4Jz9soDf-upU,14396
|
116
116
|
prefect/events/schemas/deployment_triggers.py,sha256=i_BtKscXU9kOHAeqmxrYQr8itEYfuPIxAnCW3fo1YeE,3114
|
117
|
-
prefect/events/schemas/events.py,sha256=
|
117
|
+
prefect/events/schemas/events.py,sha256=erAIoMtfFjPWeTPpja6Rg3WqUNZhlX7YM3b4CzzuomQ,9053
|
118
118
|
prefect/events/schemas/labelling.py,sha256=bU-XYaHXhI2MEBIHngth96R9D02m8HHb85KNcHZ_1Gc,3073
|
119
119
|
prefect/infrastructure/__init__.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
120
120
|
prefect/infrastructure/base.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
@@ -138,7 +138,7 @@ prefect/records/__init__.py,sha256=7q-lwyevfVgb5S7K9frzawmiJmpZ5ET0m5yXIHBYcVA,3
|
|
138
138
|
prefect/records/result_store.py,sha256=6Yh9zqqXMWjn0qWSfcjQBSfXCM7jVg9pve5TVsOodDc,1734
|
139
139
|
prefect/records/store.py,sha256=eQM1p2vZDshXZYg6SkJwL-DP3kUehL_Zgs8xa2-0DZs,224
|
140
140
|
prefect/runner/__init__.py,sha256=7U-vAOXFkzMfRz1q8Uv6Otsvc0OrPYLLP44srwkJ_8s,89
|
141
|
-
prefect/runner/runner.py,sha256=
|
141
|
+
prefect/runner/runner.py,sha256=zoW90-rYm6NLSVqmdXdQe7_aIBu6HuICe0gkz8FsKJc,47998
|
142
142
|
prefect/runner/server.py,sha256=2o5vhrL7Zbn-HBStWhCjqqViex5Ye9GiQ1EW9RSEzdo,10500
|
143
143
|
prefect/runner/storage.py,sha256=FFHk28iF_OLw-cnXQtJIgGXUV4xecxF70mobiszP8C4,24557
|
144
144
|
prefect/runner/submit.py,sha256=RuyDr-ved9wjYYarXiehY5oJVFf_HE3XKKACNWpxpPc,8131
|
@@ -153,7 +153,7 @@ prefect/types/__init__.py,sha256=SAHJDtWEGidTKXQACJ38nj6fq8r57Gj0Pwo4Gy7pVWs,223
|
|
153
153
|
prefect/types/entrypoint.py,sha256=2FF03-wLPgtnqR_bKJDB2BsXXINPdu8ptY9ZYEZnXg8,328
|
154
154
|
prefect/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
155
155
|
prefect/utilities/annotations.py,sha256=bXB43j5Zsq5gaBcJe9qnszBlnNwCTwqSTgcu2OkkRLo,2776
|
156
|
-
prefect/utilities/asyncutils.py,sha256=
|
156
|
+
prefect/utilities/asyncutils.py,sha256=8bOtYljRoOn-LKaMhaQWx80MM2KwUQQC0jzOuO--ZX4,20123
|
157
157
|
prefect/utilities/callables.py,sha256=rkPPzwiVFRoVszSUq612s9S0v3nxcMC-rIwfXoJTn0E,24915
|
158
158
|
prefect/utilities/collections.py,sha256=pPa_SZZq80cja6l99b3TV7hRQy366WnuWpOW_FnutMI,17259
|
159
159
|
prefect/utilities/compat.py,sha256=mNQZDnzyKaOqy-OV-DnmH_dc7CNF5nQgW_EsA4xMr7g,906
|
@@ -186,8 +186,8 @@ prefect/workers/cloud.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
|
186
186
|
prefect/workers/process.py,sha256=t1f1EYRoPL5B25KbLgUX2b5q-lCCAXb2Gpf6T2M9WfY,19822
|
187
187
|
prefect/workers/server.py,sha256=lgh2FfSuaNU7b6HPxSFm8JtKvAvHsZGkiOo4y4tW1Cw,2022
|
188
188
|
prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
|
189
|
-
prefect_client-3.0.
|
190
|
-
prefect_client-3.0.
|
191
|
-
prefect_client-3.0.
|
192
|
-
prefect_client-3.0.
|
193
|
-
prefect_client-3.0.
|
189
|
+
prefect_client-3.0.0rc12.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
|
190
|
+
prefect_client-3.0.0rc12.dist-info/METADATA,sha256=mpzyZ76ECL9XsRq4Squ61UkfenYkdhJzZM7FekCZjHg,7432
|
191
|
+
prefect_client-3.0.0rc12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
192
|
+
prefect_client-3.0.0rc12.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
|
193
|
+
prefect_client-3.0.0rc12.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|