prefect-client 3.0.0rc2__py3-none-any.whl → 3.0.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/compatibility/migration.py +124 -0
- prefect/_internal/concurrency/__init__.py +2 -2
- prefect/_internal/concurrency/primitives.py +1 -0
- prefect/_internal/pydantic/annotations/pendulum.py +2 -2
- prefect/_internal/pytz.py +1 -1
- prefect/blocks/core.py +1 -1
- prefect/client/orchestration.py +96 -22
- prefect/client/schemas/actions.py +1 -1
- prefect/client/schemas/filters.py +6 -0
- prefect/client/schemas/objects.py +10 -3
- prefect/client/subscriptions.py +3 -2
- prefect/context.py +1 -27
- prefect/deployments/__init__.py +3 -0
- prefect/deployments/base.py +4 -2
- prefect/deployments/deployments.py +3 -0
- prefect/deployments/steps/pull.py +1 -0
- prefect/deployments/steps/utility.py +2 -1
- prefect/engine.py +3 -0
- prefect/events/cli/automations.py +1 -1
- prefect/events/clients.py +7 -1
- prefect/exceptions.py +9 -0
- prefect/filesystems.py +22 -11
- prefect/flow_engine.py +116 -154
- prefect/flows.py +83 -34
- prefect/infrastructure/provisioners/container_instance.py +1 -0
- prefect/infrastructure/provisioners/ecs.py +2 -2
- prefect/input/__init__.py +4 -0
- prefect/logging/formatters.py +2 -2
- prefect/logging/handlers.py +2 -2
- prefect/logging/loggers.py +1 -1
- prefect/plugins.py +1 -0
- prefect/records/cache_policies.py +3 -3
- prefect/records/result_store.py +10 -3
- prefect/results.py +27 -55
- prefect/runner/runner.py +1 -1
- prefect/runner/server.py +1 -1
- prefect/runtime/__init__.py +1 -0
- prefect/runtime/deployment.py +1 -0
- prefect/runtime/flow_run.py +1 -0
- prefect/runtime/task_run.py +1 -0
- prefect/settings.py +15 -2
- prefect/states.py +15 -4
- prefect/task_engine.py +190 -33
- prefect/task_runners.py +9 -3
- prefect/task_runs.py +3 -3
- prefect/task_worker.py +29 -9
- prefect/tasks.py +133 -57
- prefect/transactions.py +87 -15
- prefect/types/__init__.py +1 -1
- prefect/utilities/asyncutils.py +3 -3
- prefect/utilities/callables.py +16 -4
- prefect/utilities/dockerutils.py +5 -3
- prefect/utilities/engine.py +11 -0
- prefect/utilities/filesystem.py +4 -5
- prefect/utilities/importtools.py +29 -0
- prefect/utilities/services.py +2 -2
- prefect/utilities/urls.py +195 -0
- prefect/utilities/visualization.py +1 -0
- prefect/variables.py +4 -0
- prefect/workers/base.py +35 -0
- {prefect_client-3.0.0rc2.dist-info → prefect_client-3.0.0rc3.dist-info}/METADATA +2 -2
- {prefect_client-3.0.0rc2.dist-info → prefect_client-3.0.0rc3.dist-info}/RECORD +65 -62
- {prefect_client-3.0.0rc2.dist-info → prefect_client-3.0.0rc3.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc2.dist-info → prefect_client-3.0.0rc3.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc2.dist-info → prefect_client-3.0.0rc3.dist-info}/top_level.txt +0 -0
prefect/tasks.py
CHANGED
@@ -22,6 +22,7 @@ from typing import (
|
|
22
22
|
Optional,
|
23
23
|
Set,
|
24
24
|
Tuple,
|
25
|
+
Type,
|
25
26
|
TypeVar,
|
26
27
|
Union,
|
27
28
|
cast,
|
@@ -122,6 +123,57 @@ def exponential_backoff(backoff_factor: float) -> Callable[[int], List[float]]:
|
|
122
123
|
return retry_backoff_callable
|
123
124
|
|
124
125
|
|
126
|
+
def _infer_parent_task_runs(
|
127
|
+
flow_run_context: Optional[FlowRunContext],
|
128
|
+
task_run_context: Optional[TaskRunContext],
|
129
|
+
parameters: Dict[str, Any],
|
130
|
+
):
|
131
|
+
"""
|
132
|
+
Attempt to infer the parent task runs for this task run based on the
|
133
|
+
provided flow run and task run contexts, as well as any parameters. It is
|
134
|
+
assumed that the task run is running within those contexts.
|
135
|
+
If any parameter comes from a running task run, that task run is considered
|
136
|
+
a parent. This is expected to happen when task inputs are yielded from
|
137
|
+
generator tasks.
|
138
|
+
"""
|
139
|
+
parents = []
|
140
|
+
|
141
|
+
# check if this task has a parent task run based on running in another
|
142
|
+
# task run's existing context. A task run is only considered a parent if
|
143
|
+
# it is in the same flow run (because otherwise presumably the child is
|
144
|
+
# in a subflow, so the subflow serves as the parent) or if there is no
|
145
|
+
# flow run
|
146
|
+
if task_run_context:
|
147
|
+
# there is no flow run
|
148
|
+
if not flow_run_context:
|
149
|
+
parents.append(TaskRunResult(id=task_run_context.task_run.id))
|
150
|
+
# there is a flow run and the task run is in the same flow run
|
151
|
+
elif flow_run_context and task_run_context.task_run.flow_run_id == getattr(
|
152
|
+
flow_run_context.flow_run, "id", None
|
153
|
+
):
|
154
|
+
parents.append(TaskRunResult(id=task_run_context.task_run.id))
|
155
|
+
|
156
|
+
# parent dependency tracking: for every provided parameter value, try to
|
157
|
+
# load the corresponding task run state. If the task run state is still
|
158
|
+
# running, we consider it a parent task run. Note this is only done if
|
159
|
+
# there is an active flow run context because dependencies are only
|
160
|
+
# tracked within the same flow run.
|
161
|
+
if flow_run_context:
|
162
|
+
for v in parameters.values():
|
163
|
+
if isinstance(v, State):
|
164
|
+
upstream_state = v
|
165
|
+
elif isinstance(v, PrefectFuture):
|
166
|
+
upstream_state = v.state
|
167
|
+
else:
|
168
|
+
upstream_state = flow_run_context.task_run_results.get(id(v))
|
169
|
+
if upstream_state and upstream_state.is_running():
|
170
|
+
parents.append(
|
171
|
+
TaskRunResult(id=upstream_state.state_details.task_run_id)
|
172
|
+
)
|
173
|
+
|
174
|
+
return parents
|
175
|
+
|
176
|
+
|
125
177
|
@PrefectObjectRegistry.register_instances
|
126
178
|
class Task(Generic[P, R]):
|
127
179
|
"""
|
@@ -268,7 +320,18 @@ class Task(Generic[P, R]):
|
|
268
320
|
self.description = description or inspect.getdoc(fn)
|
269
321
|
update_wrapper(self, fn)
|
270
322
|
self.fn = fn
|
271
|
-
|
323
|
+
|
324
|
+
# the task is considered async if its function is async or an async
|
325
|
+
# generator
|
326
|
+
self.isasync = inspect.iscoroutinefunction(
|
327
|
+
self.fn
|
328
|
+
) or inspect.isasyncgenfunction(self.fn)
|
329
|
+
|
330
|
+
# the task is considered a generator if its function is a generator or
|
331
|
+
# an async generator
|
332
|
+
self.isgenerator = inspect.isgeneratorfunction(
|
333
|
+
self.fn
|
334
|
+
) or inspect.isasyncgenfunction(self.fn)
|
272
335
|
|
273
336
|
if not name:
|
274
337
|
if not hasattr(self.fn, "__name__"):
|
@@ -367,34 +430,57 @@ class Task(Generic[P, R]):
|
|
367
430
|
self.retry_condition_fn = retry_condition_fn
|
368
431
|
self.viz_return_value = viz_return_value
|
369
432
|
|
433
|
+
@property
|
434
|
+
def ismethod(self) -> bool:
|
435
|
+
return hasattr(self.fn, "__prefect_self__")
|
436
|
+
|
437
|
+
def __get__(self, instance, owner):
|
438
|
+
"""
|
439
|
+
Implement the descriptor protocol so that the task can be used as an instance method.
|
440
|
+
When an instance method is loaded, this method is called with the "self" instance as
|
441
|
+
an argument. We return a copy of the task with that instance bound to the task's function.
|
442
|
+
"""
|
443
|
+
|
444
|
+
# if no instance is provided, it's being accessed on the class
|
445
|
+
if instance is None:
|
446
|
+
return self
|
447
|
+
|
448
|
+
# if the task is being accessed on an instance, bind the instance to the __prefect_self__ attribute
|
449
|
+
# of the task's function. This will allow it to be automatically added to the task's parameters
|
450
|
+
else:
|
451
|
+
bound_task = copy(self)
|
452
|
+
bound_task.fn.__prefect_self__ = instance
|
453
|
+
return bound_task
|
454
|
+
|
370
455
|
def with_options(
|
371
456
|
self,
|
372
457
|
*,
|
373
|
-
name: str = None,
|
374
|
-
description: str = None,
|
375
|
-
tags: Iterable[str] = None,
|
376
|
-
cache_policy: CachePolicy = NotSet,
|
377
|
-
cache_key_fn:
|
378
|
-
["TaskRunContext", Dict[str, Any]], Optional[str]
|
458
|
+
name: Optional[str] = None,
|
459
|
+
description: Optional[str] = None,
|
460
|
+
tags: Optional[Iterable[str]] = None,
|
461
|
+
cache_policy: Union[CachePolicy, Type[NotSet]] = NotSet,
|
462
|
+
cache_key_fn: Optional[
|
463
|
+
Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
|
379
464
|
] = None,
|
380
465
|
task_run_name: Optional[Union[Callable[[], str], str]] = None,
|
381
|
-
cache_expiration: datetime.timedelta = None,
|
382
|
-
retries:
|
466
|
+
cache_expiration: Optional[datetime.timedelta] = None,
|
467
|
+
retries: Union[int, Type[NotSet]] = NotSet,
|
383
468
|
retry_delay_seconds: Union[
|
384
469
|
float,
|
385
470
|
int,
|
386
471
|
List[float],
|
387
472
|
Callable[[int], List[float]],
|
473
|
+
Type[NotSet],
|
388
474
|
] = NotSet,
|
389
|
-
retry_jitter_factor:
|
390
|
-
persist_result:
|
391
|
-
result_storage:
|
392
|
-
result_serializer:
|
393
|
-
result_storage_key:
|
475
|
+
retry_jitter_factor: Union[float, Type[NotSet]] = NotSet,
|
476
|
+
persist_result: Union[bool, Type[NotSet]] = NotSet,
|
477
|
+
result_storage: Union[ResultStorage, Type[NotSet]] = NotSet,
|
478
|
+
result_serializer: Union[ResultSerializer, Type[NotSet]] = NotSet,
|
479
|
+
result_storage_key: Union[str, Type[NotSet]] = NotSet,
|
394
480
|
cache_result_in_memory: Optional[bool] = None,
|
395
|
-
timeout_seconds: Union[int, float] = None,
|
396
|
-
log_prints:
|
397
|
-
refresh_cache:
|
481
|
+
timeout_seconds: Union[int, float, None] = None,
|
482
|
+
log_prints: Union[bool, Type[NotSet]] = NotSet,
|
483
|
+
refresh_cache: Union[bool, Type[NotSet]] = NotSet,
|
398
484
|
on_completion: Optional[
|
399
485
|
List[Callable[["Task", TaskRun, State], Union[Awaitable[None], None]]]
|
400
486
|
] = None,
|
@@ -588,7 +674,7 @@ class Task(Generic[P, R]):
|
|
588
674
|
async with client:
|
589
675
|
if not flow_run_context:
|
590
676
|
dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
|
591
|
-
task_run_name =
|
677
|
+
task_run_name = self.name
|
592
678
|
else:
|
593
679
|
dynamic_key = _dynamic_key_for_task_run(
|
594
680
|
context=flow_run_context, task=self
|
@@ -624,27 +710,15 @@ class Task(Generic[P, R]):
|
|
624
710
|
k: collect_task_run_inputs_sync(v) for k, v in parameters.items()
|
625
711
|
}
|
626
712
|
|
627
|
-
#
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
if not flow_run_context:
|
635
|
-
task_inputs["__parents__"] = [
|
636
|
-
TaskRunResult(id=parent_task_run_context.task_run.id)
|
637
|
-
]
|
638
|
-
# there is a flow run and the task run is in the same flow run
|
639
|
-
elif (
|
640
|
-
flow_run_context
|
641
|
-
and parent_task_run_context.task_run.flow_run_id
|
642
|
-
== getattr(flow_run_context.flow_run, "id", None)
|
643
|
-
):
|
644
|
-
task_inputs["__parents__"] = [
|
645
|
-
TaskRunResult(id=parent_task_run_context.task_run.id)
|
646
|
-
]
|
713
|
+
# collect all parent dependencies
|
714
|
+
if task_parents := _infer_parent_task_runs(
|
715
|
+
flow_run_context=flow_run_context,
|
716
|
+
task_run_context=parent_task_run_context,
|
717
|
+
parameters=parameters,
|
718
|
+
):
|
719
|
+
task_inputs["__parents__"] = task_parents
|
647
720
|
|
721
|
+
# check wait for dependencies
|
648
722
|
if wait_for:
|
649
723
|
task_inputs["wait_for"] = collect_task_run_inputs_sync(wait_for)
|
650
724
|
|
@@ -1234,13 +1308,15 @@ def task(__fn: Callable[P, R]) -> Task[P, R]:
|
|
1234
1308
|
@overload
|
1235
1309
|
def task(
|
1236
1310
|
*,
|
1237
|
-
name: str = None,
|
1238
|
-
description: str = None,
|
1239
|
-
tags: Iterable[str] = None,
|
1240
|
-
version: str = None,
|
1311
|
+
name: Optional[str] = None,
|
1312
|
+
description: Optional[str] = None,
|
1313
|
+
tags: Optional[Iterable[str]] = None,
|
1314
|
+
version: Optional[str] = None,
|
1241
1315
|
cache_policy: CachePolicy = NotSet,
|
1242
|
-
cache_key_fn:
|
1243
|
-
|
1316
|
+
cache_key_fn: Optional[
|
1317
|
+
Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
|
1318
|
+
] = None,
|
1319
|
+
cache_expiration: Optional[datetime.timedelta] = None,
|
1244
1320
|
task_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1245
1321
|
retries: int = 0,
|
1246
1322
|
retry_delay_seconds: Union[
|
@@ -1255,7 +1331,7 @@ def task(
|
|
1255
1331
|
result_storage_key: Optional[str] = None,
|
1256
1332
|
result_serializer: Optional[ResultSerializer] = None,
|
1257
1333
|
cache_result_in_memory: bool = True,
|
1258
|
-
timeout_seconds: Union[int, float] = None,
|
1334
|
+
timeout_seconds: Union[int, float, None] = None,
|
1259
1335
|
log_prints: Optional[bool] = None,
|
1260
1336
|
refresh_cache: Optional[bool] = None,
|
1261
1337
|
on_completion: Optional[List[Callable[["Task", TaskRun, State], None]]] = None,
|
@@ -1269,20 +1345,17 @@ def task(
|
|
1269
1345
|
def task(
|
1270
1346
|
__fn=None,
|
1271
1347
|
*,
|
1272
|
-
name: str = None,
|
1273
|
-
description: str = None,
|
1274
|
-
tags: Iterable[str] = None,
|
1275
|
-
version: str = None,
|
1276
|
-
cache_policy: CachePolicy = NotSet,
|
1348
|
+
name: Optional[str] = None,
|
1349
|
+
description: Optional[str] = None,
|
1350
|
+
tags: Optional[Iterable[str]] = None,
|
1351
|
+
version: Optional[str] = None,
|
1352
|
+
cache_policy: Union[CachePolicy, Type[NotSet]] = NotSet,
|
1277
1353
|
cache_key_fn: Callable[["TaskRunContext", Dict[str, Any]], Optional[str]] = None,
|
1278
|
-
cache_expiration: datetime.timedelta = None,
|
1354
|
+
cache_expiration: Optional[datetime.timedelta] = None,
|
1279
1355
|
task_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1280
|
-
retries: int = None,
|
1356
|
+
retries: Optional[int] = None,
|
1281
1357
|
retry_delay_seconds: Union[
|
1282
|
-
float,
|
1283
|
-
int,
|
1284
|
-
List[float],
|
1285
|
-
Callable[[int], List[float]],
|
1358
|
+
float, int, List[float], Callable[[int], List[float]], None
|
1286
1359
|
] = None,
|
1287
1360
|
retry_jitter_factor: Optional[float] = None,
|
1288
1361
|
persist_result: Optional[bool] = None,
|
@@ -1290,7 +1363,7 @@ def task(
|
|
1290
1363
|
result_storage_key: Optional[str] = None,
|
1291
1364
|
result_serializer: Optional[ResultSerializer] = None,
|
1292
1365
|
cache_result_in_memory: bool = True,
|
1293
|
-
timeout_seconds: Union[int, float] = None,
|
1366
|
+
timeout_seconds: Union[int, float, None] = None,
|
1294
1367
|
log_prints: Optional[bool] = None,
|
1295
1368
|
refresh_cache: Optional[bool] = None,
|
1296
1369
|
on_completion: Optional[List[Callable[["Task", TaskRun, State], None]]] = None,
|
@@ -1408,6 +1481,9 @@ def task(
|
|
1408
1481
|
"""
|
1409
1482
|
|
1410
1483
|
if __fn:
|
1484
|
+
if isinstance(__fn, (classmethod, staticmethod)):
|
1485
|
+
method_decorator = type(__fn).__name__
|
1486
|
+
raise TypeError(f"@{method_decorator} should be applied on top of @task")
|
1411
1487
|
return cast(
|
1412
1488
|
Task[P, R],
|
1413
1489
|
Task(
|
prefect/transactions.py
CHANGED
@@ -7,17 +7,19 @@ from typing import (
|
|
7
7
|
List,
|
8
8
|
Optional,
|
9
9
|
Type,
|
10
|
-
TypeVar,
|
11
10
|
)
|
12
11
|
|
13
12
|
from pydantic import Field
|
13
|
+
from typing_extensions import Self
|
14
14
|
|
15
|
-
from prefect.context import ContextModel
|
15
|
+
from prefect.context import ContextModel, FlowRunContext, TaskRunContext
|
16
16
|
from prefect.records import RecordStore
|
17
|
+
from prefect.records.result_store import ResultFactoryStore
|
18
|
+
from prefect.results import BaseResult, ResultFactory, get_default_result_storage
|
19
|
+
from prefect.settings import PREFECT_DEFAULT_RESULT_STORAGE_BLOCK
|
20
|
+
from prefect.utilities.asyncutils import run_coro_as_sync
|
17
21
|
from prefect.utilities.collections import AutoEnum
|
18
22
|
|
19
|
-
T = TypeVar("T")
|
20
|
-
|
21
23
|
|
22
24
|
class IsolationLevel(AutoEnum):
|
23
25
|
READ_COMMITTED = AutoEnum.auto()
|
@@ -54,7 +56,7 @@ class Transaction(ContextModel):
|
|
54
56
|
)
|
55
57
|
overwrite: bool = False
|
56
58
|
_staged_value: Any = None
|
57
|
-
__var__ = ContextVar("transaction")
|
59
|
+
__var__: ContextVar = ContextVar("transaction")
|
58
60
|
|
59
61
|
def is_committed(self) -> bool:
|
60
62
|
return self.state == TransactionState.COMMITTED
|
@@ -92,7 +94,8 @@ class Transaction(ContextModel):
|
|
92
94
|
self._token = self.__var__.set(self)
|
93
95
|
return self
|
94
96
|
|
95
|
-
def __exit__(self,
|
97
|
+
def __exit__(self, *exc_info):
|
98
|
+
exc_type, exc_val, _ = exc_info
|
96
99
|
if not self._token:
|
97
100
|
raise RuntimeError(
|
98
101
|
"Asymmetric use of context. Context exit called without an enter."
|
@@ -123,11 +126,19 @@ class Transaction(ContextModel):
|
|
123
126
|
def begin(self):
|
124
127
|
# currently we only support READ_COMMITTED isolation
|
125
128
|
# i.e., no locking behavior
|
126
|
-
if
|
129
|
+
if (
|
130
|
+
not self.overwrite
|
131
|
+
and self.store
|
132
|
+
and self.key
|
133
|
+
and self.store.exists(key=self.key)
|
134
|
+
):
|
127
135
|
self.state = TransactionState.COMMITTED
|
128
136
|
|
129
|
-
def read(self) ->
|
130
|
-
|
137
|
+
def read(self) -> BaseResult:
|
138
|
+
if self.store and self.key:
|
139
|
+
return self.store.read(key=self.key)
|
140
|
+
else:
|
141
|
+
return {} # TODO: Determine what this should be
|
131
142
|
|
132
143
|
def reset(self) -> None:
|
133
144
|
parent = self.get_parent()
|
@@ -136,8 +147,9 @@ class Transaction(ContextModel):
|
|
136
147
|
# parent takes responsibility
|
137
148
|
parent.add_child(self)
|
138
149
|
|
139
|
-
self.
|
140
|
-
|
150
|
+
if self._token:
|
151
|
+
self.__var__.reset(self._token)
|
152
|
+
self._token = None
|
141
153
|
|
142
154
|
# do this below reset so that get_transaction() returns the relevant txn
|
143
155
|
if parent and self.state == TransactionState.ROLLED_BACK:
|
@@ -165,7 +177,7 @@ class Transaction(ContextModel):
|
|
165
177
|
for hook in self.on_commit_hooks:
|
166
178
|
hook(self)
|
167
179
|
|
168
|
-
if self.store:
|
180
|
+
if self.store and self.key:
|
169
181
|
self.store.write(key=self.key, value=self._staged_value)
|
170
182
|
self.state = TransactionState.COMMITTED
|
171
183
|
return True
|
@@ -174,11 +186,17 @@ class Transaction(ContextModel):
|
|
174
186
|
return False
|
175
187
|
|
176
188
|
def stage(
|
177
|
-
self,
|
189
|
+
self,
|
190
|
+
value: BaseResult,
|
191
|
+
on_rollback_hooks: Optional[List] = None,
|
192
|
+
on_commit_hooks: Optional[List] = None,
|
178
193
|
) -> None:
|
179
194
|
"""
|
180
195
|
Stage a value to be committed later.
|
181
196
|
"""
|
197
|
+
on_commit_hooks = on_commit_hooks or []
|
198
|
+
on_rollback_hooks = on_rollback_hooks or []
|
199
|
+
|
182
200
|
if self.state != TransactionState.COMMITTED:
|
183
201
|
self._staged_value = value
|
184
202
|
self.on_rollback_hooks += on_rollback_hooks
|
@@ -203,11 +221,11 @@ class Transaction(ContextModel):
|
|
203
221
|
return False
|
204
222
|
|
205
223
|
@classmethod
|
206
|
-
def get_active(cls: Type[
|
224
|
+
def get_active(cls: Type[Self]) -> Optional[Self]:
|
207
225
|
return cls.__var__.get(None)
|
208
226
|
|
209
227
|
|
210
|
-
def get_transaction() -> Transaction:
|
228
|
+
def get_transaction() -> Optional[Transaction]:
|
211
229
|
return Transaction.get_active()
|
212
230
|
|
213
231
|
|
@@ -218,6 +236,60 @@ def transaction(
|
|
218
236
|
commit_mode: CommitMode = CommitMode.LAZY,
|
219
237
|
overwrite: bool = False,
|
220
238
|
) -> Generator[Transaction, None, None]:
|
239
|
+
"""
|
240
|
+
A context manager for opening and managing a transaction.
|
241
|
+
|
242
|
+
Args:
|
243
|
+
- key: An identifier to use for the transaction
|
244
|
+
- store: The store to use for persisting the transaction result. If not provided,
|
245
|
+
a default store will be used based on the current run context.
|
246
|
+
- commit_mode: The commit mode controlling when the transaction and
|
247
|
+
child transactions are committed
|
248
|
+
- overwrite: Whether to overwrite an existing transaction record in the store
|
249
|
+
|
250
|
+
Yields:
|
251
|
+
- Transaction: An object representing the transaction state
|
252
|
+
"""
|
253
|
+
# if there is no key, we won't persist a record
|
254
|
+
if key and not store:
|
255
|
+
flow_run_context = FlowRunContext.get()
|
256
|
+
task_run_context = TaskRunContext.get()
|
257
|
+
existing_factory = getattr(task_run_context, "result_factory", None) or getattr(
|
258
|
+
flow_run_context, "result_factory", None
|
259
|
+
)
|
260
|
+
|
261
|
+
if existing_factory and existing_factory.storage_block_id:
|
262
|
+
new_factory = existing_factory.model_copy(
|
263
|
+
update={
|
264
|
+
"persist_result": True,
|
265
|
+
}
|
266
|
+
)
|
267
|
+
else:
|
268
|
+
default_storage = get_default_result_storage(_sync=True)
|
269
|
+
if not default_storage._block_document_id:
|
270
|
+
default_name = PREFECT_DEFAULT_RESULT_STORAGE_BLOCK.value().split("/")[
|
271
|
+
-1
|
272
|
+
]
|
273
|
+
default_storage.save(default_name, overwrite=True, _sync=True)
|
274
|
+
if existing_factory:
|
275
|
+
new_factory = existing_factory.model_copy(
|
276
|
+
update={
|
277
|
+
"persist_result": True,
|
278
|
+
"storage_block": default_storage,
|
279
|
+
"storage_block_id": default_storage._block_document_id,
|
280
|
+
}
|
281
|
+
)
|
282
|
+
else:
|
283
|
+
new_factory = run_coro_as_sync(
|
284
|
+
ResultFactory.default_factory(
|
285
|
+
persist_result=True,
|
286
|
+
result_storage=default_storage,
|
287
|
+
)
|
288
|
+
)
|
289
|
+
store = ResultFactoryStore(
|
290
|
+
result_factory=new_factory,
|
291
|
+
)
|
292
|
+
|
221
293
|
with Transaction(
|
222
294
|
key=key, store=store, commit_mode=commit_mode, overwrite=overwrite
|
223
295
|
) as txn:
|
prefect/types/__init__.py
CHANGED
@@ -20,7 +20,7 @@ timezone_set = available_timezones()
|
|
20
20
|
NonNegativeInteger = Annotated[int, Field(ge=0)]
|
21
21
|
PositiveInteger = Annotated[int, Field(gt=0)]
|
22
22
|
NonNegativeFloat = Annotated[float, Field(ge=0.0)]
|
23
|
-
TimeZone = Annotated[str, Field(default="UTC", pattern="|".join(timezone_set))]
|
23
|
+
TimeZone = Annotated[str, Field(default="UTC", pattern="|".join(sorted(timezone_set)))]
|
24
24
|
|
25
25
|
|
26
26
|
BANNED_CHARACTERS = ["/", "%", "&", ">", "<"]
|
prefect/utilities/asyncutils.py
CHANGED
@@ -314,7 +314,7 @@ def sync_compatible(async_fn: T, force_sync: bool = False) -> T:
|
|
314
314
|
"""
|
315
315
|
|
316
316
|
@wraps(async_fn)
|
317
|
-
def coroutine_wrapper(*args, _sync: bool = None, **kwargs):
|
317
|
+
def coroutine_wrapper(*args, _sync: Optional[bool] = None, **kwargs):
|
318
318
|
from prefect.context import MissingContextError, get_run_context
|
319
319
|
from prefect.settings import (
|
320
320
|
PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT,
|
@@ -376,8 +376,8 @@ def sync_compatible(async_fn: T, force_sync: bool = False) -> T:
|
|
376
376
|
|
377
377
|
|
378
378
|
@asynccontextmanager
|
379
|
-
async def asyncnullcontext():
|
380
|
-
yield
|
379
|
+
async def asyncnullcontext(value=None):
|
380
|
+
yield value
|
381
381
|
|
382
382
|
|
383
383
|
def sync(__async_fn: Callable[P, Awaitable[T]], *args: P.args, **kwargs: P.kwargs) -> T:
|
prefect/utilities/callables.py
CHANGED
@@ -44,11 +44,23 @@ def get_call_parameters(
|
|
44
44
|
apply_defaults: bool = True,
|
45
45
|
) -> Dict[str, Any]:
|
46
46
|
"""
|
47
|
-
Bind a call to a function to get parameter/value mapping. Default values on
|
48
|
-
signature will be included if not overridden.
|
49
|
-
|
50
|
-
|
47
|
+
Bind a call to a function to get parameter/value mapping. Default values on
|
48
|
+
the signature will be included if not overridden.
|
49
|
+
|
50
|
+
If the function has a `__prefect_self__` attribute, it will be included as
|
51
|
+
the first parameter. This attribute is set when Prefect decorates a bound
|
52
|
+
method, so this approach allows Prefect to work with bound methods in a way
|
53
|
+
that is consistent with how Python handles them (i.e. users don't have to
|
54
|
+
pass the instance argument to the method) while still making the implicit self
|
55
|
+
argument visible to all of Prefect's parameter machinery (such as cache key
|
56
|
+
functions).
|
57
|
+
|
58
|
+
Raises a ParameterBindError if the arguments/kwargs are not valid for the
|
59
|
+
function
|
51
60
|
"""
|
61
|
+
if hasattr(fn, "__prefect_self__"):
|
62
|
+
call_args = (fn.__prefect_self__,) + call_args
|
63
|
+
|
52
64
|
try:
|
53
65
|
bound_signature = inspect.signature(fn).bind(*call_args, **call_kwargs)
|
54
66
|
except TypeError as exc:
|
prefect/utilities/dockerutils.py
CHANGED
@@ -41,7 +41,9 @@ def python_version_micro() -> str:
|
|
41
41
|
|
42
42
|
|
43
43
|
def get_prefect_image_name(
|
44
|
-
prefect_version: str = None,
|
44
|
+
prefect_version: Optional[str] = None,
|
45
|
+
python_version: Optional[str] = None,
|
46
|
+
flavor: Optional[str] = None,
|
45
47
|
) -> str:
|
46
48
|
"""
|
47
49
|
Get the Prefect image name matching the current Prefect and Python versions.
|
@@ -138,7 +140,7 @@ def build_image(
|
|
138
140
|
dockerfile: str = "Dockerfile",
|
139
141
|
tag: Optional[str] = None,
|
140
142
|
pull: bool = False,
|
141
|
-
platform: str = None,
|
143
|
+
platform: Optional[str] = None,
|
142
144
|
stream_progress_to: Optional[TextIO] = None,
|
143
145
|
**kwargs,
|
144
146
|
) -> str:
|
@@ -209,7 +211,7 @@ class ImageBuilder:
|
|
209
211
|
self,
|
210
212
|
base_image: str,
|
211
213
|
base_directory: Path = None,
|
212
|
-
platform: str = None,
|
214
|
+
platform: Optional[str] = None,
|
213
215
|
context: Path = None,
|
214
216
|
):
|
215
217
|
"""Create an ImageBuilder
|
prefect/utilities/engine.py
CHANGED
@@ -786,6 +786,17 @@ def resolve_to_final_result(expr, context):
|
|
786
786
|
raise StopVisiting()
|
787
787
|
|
788
788
|
if isinstance(expr, NewPrefectFuture):
|
789
|
+
upstream_task_run = context.get("current_task_run")
|
790
|
+
upstream_task = context.get("current_task")
|
791
|
+
if (
|
792
|
+
upstream_task
|
793
|
+
and upstream_task_run
|
794
|
+
and expr.task_run_id == upstream_task_run.id
|
795
|
+
):
|
796
|
+
raise ValueError(
|
797
|
+
f"Discovered a task depending on itself. Raising to avoid a deadlock. Please inspect the inputs and dependencies of {upstream_task.name}."
|
798
|
+
)
|
799
|
+
|
789
800
|
expr.wait()
|
790
801
|
state = expr.state
|
791
802
|
elif isinstance(expr, State):
|
prefect/utilities/filesystem.py
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
"""
|
2
2
|
Utilities for working with file systems
|
3
3
|
"""
|
4
|
+
|
4
5
|
import os
|
5
6
|
import pathlib
|
6
7
|
import threading
|
7
8
|
from contextlib import contextmanager
|
8
9
|
from pathlib import Path, PureWindowsPath
|
9
|
-
from typing import Union
|
10
|
+
from typing import Optional, Union
|
10
11
|
|
11
12
|
import fsspec
|
12
13
|
import pathspec
|
@@ -32,7 +33,7 @@ def create_default_ignore_file(path: str) -> bool:
|
|
32
33
|
|
33
34
|
|
34
35
|
def filter_files(
|
35
|
-
root: str = ".", ignore_patterns: list = None, include_dirs: bool = True
|
36
|
+
root: str = ".", ignore_patterns: Optional[list] = None, include_dirs: bool = True
|
36
37
|
) -> set:
|
37
38
|
"""
|
38
39
|
This function accepts a root directory path and a list of file patterns to ignore, and returns
|
@@ -40,9 +41,7 @@ def filter_files(
|
|
40
41
|
|
41
42
|
The specification matches that of [.gitignore files](https://git-scm.com/docs/gitignore).
|
42
43
|
"""
|
43
|
-
|
44
|
-
ignore_patterns = []
|
45
|
-
spec = pathspec.PathSpec.from_lines("gitwildmatch", ignore_patterns)
|
44
|
+
spec = pathspec.PathSpec.from_lines("gitwildmatch", ignore_patterns or [])
|
46
45
|
ignored_files = {p.path for p in spec.match_tree_entries(root)}
|
47
46
|
if include_dirs:
|
48
47
|
all_files = {p.path for p in pathspec.util.iter_tree_entries(root)}
|
prefect/utilities/importtools.py
CHANGED
@@ -380,6 +380,15 @@ def safe_load_namespace(source_code: str):
|
|
380
380
|
|
381
381
|
namespace = {"__name__": "prefect_safe_namespace_loader"}
|
382
382
|
|
383
|
+
# Remove the body of the if __name__ == "__main__": block from the AST to prevent
|
384
|
+
# execution of guarded code
|
385
|
+
new_body = []
|
386
|
+
for node in parsed_code.body:
|
387
|
+
if _is_main_block(node):
|
388
|
+
continue
|
389
|
+
new_body.append(node)
|
390
|
+
parsed_code.body = new_body
|
391
|
+
|
383
392
|
# Walk through the AST and find all import statements
|
384
393
|
for node in ast.walk(parsed_code):
|
385
394
|
if isinstance(node, ast.Import):
|
@@ -426,3 +435,23 @@ def safe_load_namespace(source_code: str):
|
|
426
435
|
except Exception as e:
|
427
436
|
logger.debug("Failed to compile: %s", e)
|
428
437
|
return namespace
|
438
|
+
|
439
|
+
|
440
|
+
def _is_main_block(node: ast.AST):
|
441
|
+
"""
|
442
|
+
Check if the node is an `if __name__ == "__main__":` block.
|
443
|
+
"""
|
444
|
+
if isinstance(node, ast.If):
|
445
|
+
try:
|
446
|
+
# Check if the condition is `if __name__ == "__main__":`
|
447
|
+
if (
|
448
|
+
isinstance(node.test, ast.Compare)
|
449
|
+
and isinstance(node.test.left, ast.Name)
|
450
|
+
and node.test.left.id == "__name__"
|
451
|
+
and isinstance(node.test.comparators[0], ast.Constant)
|
452
|
+
and node.test.comparators[0].value == "__main__"
|
453
|
+
):
|
454
|
+
return True
|
455
|
+
except AttributeError:
|
456
|
+
pass
|
457
|
+
return False
|
prefect/utilities/services.py
CHANGED
@@ -2,7 +2,7 @@ import sys
|
|
2
2
|
from collections import deque
|
3
3
|
from traceback import format_exception
|
4
4
|
from types import TracebackType
|
5
|
-
from typing import Callable, Coroutine, Deque, Tuple
|
5
|
+
from typing import Callable, Coroutine, Deque, Optional, Tuple
|
6
6
|
|
7
7
|
import anyio
|
8
8
|
import httpx
|
@@ -22,7 +22,7 @@ async def critical_service_loop(
|
|
22
22
|
backoff: int = 1,
|
23
23
|
printer: Callable[..., None] = print,
|
24
24
|
run_once: bool = False,
|
25
|
-
jitter_range: float = None,
|
25
|
+
jitter_range: Optional[float] = None,
|
26
26
|
):
|
27
27
|
"""
|
28
28
|
Runs the given `workload` function on the specified `interval`, while being
|