prefect-client 3.0.0rc17__py3-none-any.whl → 3.0.0rc19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/concurrency/services.py +14 -0
- prefect/_internal/schemas/bases.py +1 -0
- prefect/blocks/core.py +36 -29
- prefect/client/orchestration.py +97 -2
- prefect/client/schemas/actions.py +14 -4
- prefect/client/schemas/filters.py +20 -0
- prefect/client/schemas/objects.py +3 -0
- prefect/client/schemas/responses.py +3 -0
- prefect/client/schemas/sorting.py +2 -0
- prefect/concurrency/v1/__init__.py +0 -0
- prefect/concurrency/v1/asyncio.py +143 -0
- prefect/concurrency/v1/context.py +27 -0
- prefect/concurrency/v1/events.py +61 -0
- prefect/concurrency/v1/services.py +116 -0
- prefect/concurrency/v1/sync.py +92 -0
- prefect/context.py +2 -2
- prefect/deployments/flow_runs.py +0 -7
- prefect/deployments/runner.py +11 -0
- prefect/events/clients.py +41 -0
- prefect/events/related.py +72 -73
- prefect/events/utilities.py +2 -0
- prefect/events/worker.py +12 -3
- prefect/flow_engine.py +2 -0
- prefect/flows.py +7 -0
- prefect/records/__init__.py +1 -1
- prefect/records/base.py +223 -0
- prefect/records/filesystem.py +207 -0
- prefect/records/memory.py +178 -0
- prefect/records/result_store.py +19 -14
- prefect/results.py +11 -0
- prefect/runner/runner.py +7 -4
- prefect/settings.py +0 -8
- prefect/task_engine.py +98 -209
- prefect/task_worker.py +7 -39
- prefect/tasks.py +2 -9
- prefect/transactions.py +67 -19
- prefect/utilities/asyncutils.py +3 -3
- prefect/utilities/callables.py +1 -3
- prefect/utilities/engine.py +7 -6
- {prefect_client-3.0.0rc17.dist-info → prefect_client-3.0.0rc19.dist-info}/METADATA +3 -4
- {prefect_client-3.0.0rc17.dist-info → prefect_client-3.0.0rc19.dist-info}/RECORD +44 -36
- prefect/records/store.py +0 -9
- {prefect_client-3.0.0rc17.dist-info → prefect_client-3.0.0rc19.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc17.dist-info → prefect_client-3.0.0rc19.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc17.dist-info → prefect_client-3.0.0rc19.dist-info}/top_level.txt +0 -0
prefect/runner/runner.py
CHANGED
@@ -35,7 +35,6 @@ import datetime
|
|
35
35
|
import inspect
|
36
36
|
import logging
|
37
37
|
import os
|
38
|
-
import shlex
|
39
38
|
import shutil
|
40
39
|
import signal
|
41
40
|
import subprocess
|
@@ -90,7 +89,11 @@ from prefect.utilities.asyncutils import (
|
|
90
89
|
sync_compatible,
|
91
90
|
)
|
92
91
|
from prefect.utilities.engine import propose_state
|
93
|
-
from prefect.utilities.processutils import
|
92
|
+
from prefect.utilities.processutils import (
|
93
|
+
_register_signal,
|
94
|
+
get_sys_executable,
|
95
|
+
run_process,
|
96
|
+
)
|
94
97
|
from prefect.utilities.services import (
|
95
98
|
critical_service_loop,
|
96
99
|
start_client_metrics_server,
|
@@ -527,7 +530,7 @@ class Runner:
|
|
527
530
|
task_status: anyio task status used to send a message to the caller
|
528
531
|
than the flow run process has started.
|
529
532
|
"""
|
530
|
-
command =
|
533
|
+
command = [get_sys_executable(), "-m", "prefect.engine"]
|
531
534
|
|
532
535
|
flow_run_logger = self._get_flow_run_logger(flow_run)
|
533
536
|
|
@@ -574,7 +577,7 @@ class Runner:
|
|
574
577
|
setattr(storage, "last_adhoc_pull", datetime.datetime.now())
|
575
578
|
|
576
579
|
process = await run_process(
|
577
|
-
|
580
|
+
command=command,
|
578
581
|
stream_output=True,
|
579
582
|
task_status=task_status,
|
580
583
|
env=env,
|
prefect/settings.py
CHANGED
@@ -1379,14 +1379,6 @@ PREFECT_API_MAX_FLOW_RUN_GRAPH_ARTIFACTS = Setting(int, default=10000)
|
|
1379
1379
|
The maximum number of artifacts to show on a flow run graph on the v2 API
|
1380
1380
|
"""
|
1381
1381
|
|
1382
|
-
|
1383
|
-
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION = Setting(
|
1384
|
-
bool, default=False
|
1385
|
-
)
|
1386
|
-
"""
|
1387
|
-
Whether or not to enable experimental client side task run orchestration.
|
1388
|
-
"""
|
1389
|
-
|
1390
1382
|
# Prefect Events feature flags
|
1391
1383
|
|
1392
1384
|
PREFECT_RUNNER_PROCESS_LIMIT = Setting(int, default=5)
|
prefect/task_engine.py
CHANGED
@@ -33,9 +33,10 @@ from prefect import Task
|
|
33
33
|
from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
|
34
34
|
from prefect.client.schemas import TaskRun
|
35
35
|
from prefect.client.schemas.objects import State, TaskRunInput
|
36
|
-
from prefect.concurrency.asyncio import concurrency as aconcurrency
|
37
36
|
from prefect.concurrency.context import ConcurrencyContext
|
38
|
-
from prefect.concurrency.
|
37
|
+
from prefect.concurrency.v1.asyncio import concurrency as aconcurrency
|
38
|
+
from prefect.concurrency.v1.context import ConcurrencyContext as ConcurrencyContextV1
|
39
|
+
from prefect.concurrency.v1.sync import concurrency
|
39
40
|
from prefect.context import (
|
40
41
|
AsyncClientContext,
|
41
42
|
FlowRunContext,
|
@@ -57,14 +58,12 @@ from prefect.records.result_store import ResultFactoryStore
|
|
57
58
|
from prefect.results import BaseResult, ResultFactory, _format_user_supplied_storage_key
|
58
59
|
from prefect.settings import (
|
59
60
|
PREFECT_DEBUG_MODE,
|
60
|
-
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION,
|
61
61
|
PREFECT_TASKS_REFRESH_CACHE,
|
62
62
|
)
|
63
63
|
from prefect.states import (
|
64
64
|
AwaitingRetry,
|
65
65
|
Completed,
|
66
66
|
Failed,
|
67
|
-
Paused,
|
68
67
|
Pending,
|
69
68
|
Retrying,
|
70
69
|
Running,
|
@@ -81,8 +80,6 @@ from prefect.utilities.engine import (
|
|
81
80
|
_get_hook_name,
|
82
81
|
emit_task_run_state_change_event,
|
83
82
|
link_state_to_result,
|
84
|
-
propose_state,
|
85
|
-
propose_state_sync,
|
86
83
|
resolve_to_final_result,
|
87
84
|
)
|
88
85
|
from prefect.utilities.math import clamped_poisson_interval
|
@@ -195,14 +192,13 @@ class BaseTaskRunEngine(Generic[P, R]):
|
|
195
192
|
)
|
196
193
|
|
197
194
|
def record_terminal_state_timing(self, state: State) -> None:
|
198
|
-
if
|
199
|
-
|
200
|
-
self.task_run.end_time = state.timestamp
|
195
|
+
if self.task_run and self.task_run.start_time and not self.task_run.end_time:
|
196
|
+
self.task_run.end_time = state.timestamp
|
201
197
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
198
|
+
if self.task_run.state.is_running():
|
199
|
+
self.task_run.total_run_time += (
|
200
|
+
state.timestamp - self.task_run.state.timestamp
|
201
|
+
)
|
206
202
|
|
207
203
|
def is_running(self) -> bool:
|
208
204
|
"""Whether or not the engine is currently running a task."""
|
@@ -361,15 +357,14 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
361
357
|
|
362
358
|
new_state = Running()
|
363
359
|
|
364
|
-
|
365
|
-
|
366
|
-
self.task_run.run_count += 1
|
360
|
+
self.task_run.start_time = new_state.timestamp
|
361
|
+
self.task_run.run_count += 1
|
367
362
|
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
363
|
+
flow_run_context = FlowRunContext.get()
|
364
|
+
if flow_run_context and flow_run_context.flow_run:
|
365
|
+
# Carry forward any task run information from the flow run
|
366
|
+
flow_run = flow_run_context.flow_run
|
367
|
+
self.task_run.flow_run_run_count = flow_run.run_count
|
373
368
|
|
374
369
|
state = self.set_state(new_state)
|
375
370
|
|
@@ -400,51 +395,28 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
400
395
|
if not self.task_run:
|
401
396
|
raise ValueError("Task run is not set")
|
402
397
|
|
403
|
-
|
404
|
-
self.task_run.state = new_state = state
|
405
|
-
|
406
|
-
# Ensure that the state_details are populated with the current run IDs
|
407
|
-
new_state.state_details.task_run_id = self.task_run.id
|
408
|
-
new_state.state_details.flow_run_id = self.task_run.flow_run_id
|
398
|
+
self.task_run.state = new_state = state
|
409
399
|
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
self.task_run.state_name = new_state.name
|
400
|
+
# Ensure that the state_details are populated with the current run IDs
|
401
|
+
new_state.state_details.task_run_id = self.task_run.id
|
402
|
+
new_state.state_details.flow_run_id = self.task_run.flow_run_id
|
414
403
|
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
):
|
420
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
421
|
-
# the purpose of disabling `cache_result_in_memory`
|
422
|
-
result = state.result(raise_on_failure=False, fetch=True)
|
423
|
-
if inspect.isawaitable(result):
|
424
|
-
result = run_coro_as_sync(result)
|
425
|
-
else:
|
426
|
-
result = state.data
|
404
|
+
# Predictively update the de-normalized task_run.state_* attributes
|
405
|
+
self.task_run.state_id = new_state.id
|
406
|
+
self.task_run.state_type = new_state.type
|
407
|
+
self.task_run.state_name = new_state.name
|
427
408
|
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
# just use a Paused state to assume an in-process pause.
|
438
|
-
new_state = exc.state if exc.state else Paused()
|
439
|
-
if new_state.state_details.pause_reschedule:
|
440
|
-
# If we're being asked to pause and reschedule, we should exit the
|
441
|
-
# task and expect to be resumed later.
|
442
|
-
raise
|
409
|
+
if new_state.is_final():
|
410
|
+
if isinstance(state.data, BaseResult) and state.data.has_cached_object():
|
411
|
+
# Avoid fetching the result unless it is cached, otherwise we defeat
|
412
|
+
# the purpose of disabling `cache_result_in_memory`
|
413
|
+
result = state.result(raise_on_failure=False, fetch=True)
|
414
|
+
if inspect.isawaitable(result):
|
415
|
+
result = run_coro_as_sync(result)
|
416
|
+
else:
|
417
|
+
result = state.data
|
443
418
|
|
444
|
-
|
445
|
-
# that has an in-memory result attached to it; using the API state
|
446
|
-
# could result in losing that reference
|
447
|
-
self.task_run.state = new_state
|
419
|
+
link_state_to_result(state, result)
|
448
420
|
|
449
421
|
# emit a state change event
|
450
422
|
self._last_event = emit_task_run_state_change_event(
|
@@ -531,8 +503,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
531
503
|
else:
|
532
504
|
delay = None
|
533
505
|
new_state = Retrying()
|
534
|
-
|
535
|
-
self.task_run.run_count += 1
|
506
|
+
self.task_run.run_count += 1
|
536
507
|
|
537
508
|
self.logger.info(
|
538
509
|
"Task run failed with exception: %r - " "Retry %s/%s will start %s",
|
@@ -606,8 +577,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
606
577
|
if not self.task_run:
|
607
578
|
raise ValueError("Task run is not set")
|
608
579
|
|
609
|
-
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
610
|
-
self.task_run = client.read_task_run(self.task_run.id)
|
611
580
|
with ExitStack() as stack:
|
612
581
|
if log_prints := should_log_prints(self.task):
|
613
582
|
stack.enter_context(patch_print())
|
@@ -621,6 +590,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
621
590
|
client=client,
|
622
591
|
)
|
623
592
|
)
|
593
|
+
stack.enter_context(ConcurrencyContextV1())
|
624
594
|
stack.enter_context(ConcurrencyContext())
|
625
595
|
|
626
596
|
self.logger = task_run_logger(task_run=self.task_run, task=self.task) # type: ignore
|
@@ -631,12 +601,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
631
601
|
task=self.task, parameters=self.parameters
|
632
602
|
)
|
633
603
|
|
634
|
-
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
635
|
-
# update the task run name if necessary
|
636
|
-
self.client.set_task_run_name(
|
637
|
-
task_run_id=self.task_run.id, name=task_run_name
|
638
|
-
)
|
639
|
-
|
640
604
|
self.logger.extra["task_run_name"] = task_run_name
|
641
605
|
self.logger.debug(
|
642
606
|
f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
|
@@ -660,42 +624,23 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
660
624
|
self._client = client_ctx.client
|
661
625
|
self._is_started = True
|
662
626
|
try:
|
663
|
-
if
|
664
|
-
|
665
|
-
self.
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
extra_task_inputs=dependencies,
|
673
|
-
)
|
674
|
-
)
|
675
|
-
# Emit an event to capture that the task run was in the `PENDING` state.
|
676
|
-
self._last_event = emit_task_run_state_change_event(
|
677
|
-
task_run=self.task_run,
|
678
|
-
initial_state=None,
|
679
|
-
validated_state=self.task_run.state,
|
680
|
-
)
|
681
|
-
else:
|
682
|
-
if not self.task_run:
|
683
|
-
self.task_run = run_coro_as_sync(
|
684
|
-
self.task.create_run(
|
685
|
-
id=task_run_id,
|
686
|
-
parameters=self.parameters,
|
687
|
-
flow_run_context=FlowRunContext.get(),
|
688
|
-
parent_task_run_context=TaskRunContext.get(),
|
689
|
-
wait_for=self.wait_for,
|
690
|
-
extra_task_inputs=dependencies,
|
691
|
-
)
|
692
|
-
)
|
693
|
-
# Emit an event to capture that the task run was in the `PENDING` state.
|
694
|
-
self._last_event = emit_task_run_state_change_event(
|
695
|
-
task_run=self.task_run,
|
696
|
-
initial_state=None,
|
697
|
-
validated_state=self.task_run.state,
|
627
|
+
if not self.task_run:
|
628
|
+
self.task_run = run_coro_as_sync(
|
629
|
+
self.task.create_local_run(
|
630
|
+
id=task_run_id,
|
631
|
+
parameters=self.parameters,
|
632
|
+
flow_run_context=FlowRunContext.get(),
|
633
|
+
parent_task_run_context=TaskRunContext.get(),
|
634
|
+
wait_for=self.wait_for,
|
635
|
+
extra_task_inputs=dependencies,
|
698
636
|
)
|
637
|
+
)
|
638
|
+
# Emit an event to capture that the task run was in the `PENDING` state.
|
639
|
+
self._last_event = emit_task_run_state_change_event(
|
640
|
+
task_run=self.task_run,
|
641
|
+
initial_state=None,
|
642
|
+
validated_state=self.task_run.state,
|
643
|
+
)
|
699
644
|
|
700
645
|
with self.setup_run_context():
|
701
646
|
# setup_run_context might update the task run name, so log creation here
|
@@ -808,15 +753,10 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
808
753
|
if transaction.is_committed():
|
809
754
|
result = transaction.read()
|
810
755
|
else:
|
811
|
-
if
|
812
|
-
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION.value()
|
813
|
-
and self.task.tags
|
814
|
-
):
|
756
|
+
if self.task.tags:
|
815
757
|
# Acquire a concurrency slot for each tag, but only if a limit
|
816
758
|
# matching the tag already exists.
|
817
|
-
with concurrency(
|
818
|
-
list(self.task.tags), occupy=1, create_if_missing=False
|
819
|
-
):
|
759
|
+
with concurrency(list(self.task.tags), self.task_run.id):
|
820
760
|
result = call_with_parameters(self.task.fn, parameters)
|
821
761
|
else:
|
822
762
|
result = call_with_parameters(self.task.fn, parameters)
|
@@ -920,15 +860,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
920
860
|
|
921
861
|
new_state = Running()
|
922
862
|
|
923
|
-
|
924
|
-
|
925
|
-
self.task_run.run_count += 1
|
863
|
+
self.task_run.start_time = new_state.timestamp
|
864
|
+
self.task_run.run_count += 1
|
926
865
|
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
931
|
-
|
866
|
+
flow_run_context = FlowRunContext.get()
|
867
|
+
if flow_run_context:
|
868
|
+
# Carry forward any task run information from the flow run
|
869
|
+
flow_run = flow_run_context.flow_run
|
870
|
+
self.task_run.flow_run_run_count = flow_run.run_count
|
932
871
|
|
933
872
|
state = await self.set_state(new_state)
|
934
873
|
|
@@ -959,49 +898,29 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
959
898
|
if not self.task_run:
|
960
899
|
raise ValueError("Task run is not set")
|
961
900
|
|
962
|
-
|
963
|
-
self.task_run.state = new_state = state
|
964
|
-
|
965
|
-
# Ensure that the state_details are populated with the current run IDs
|
966
|
-
new_state.state_details.task_run_id = self.task_run.id
|
967
|
-
new_state.state_details.flow_run_id = self.task_run.flow_run_id
|
901
|
+
self.task_run.state = new_state = state
|
968
902
|
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
self.task_run.state_name = new_state.name
|
903
|
+
# Ensure that the state_details are populated with the current run IDs
|
904
|
+
new_state.state_details.task_run_id = self.task_run.id
|
905
|
+
new_state.state_details.flow_run_id = self.task_run.flow_run_id
|
973
906
|
|
974
|
-
|
975
|
-
|
976
|
-
|
977
|
-
|
978
|
-
):
|
979
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
980
|
-
# the purpose of disabling `cache_result_in_memory`
|
981
|
-
result = await new_state.result(raise_on_failure=False, fetch=True)
|
982
|
-
else:
|
983
|
-
result = new_state.data
|
907
|
+
# Predictively update the de-normalized task_run.state_* attributes
|
908
|
+
self.task_run.state_id = new_state.id
|
909
|
+
self.task_run.state_type = new_state.type
|
910
|
+
self.task_run.state_name = new_state.name
|
984
911
|
|
985
|
-
|
986
|
-
|
987
|
-
|
988
|
-
|
989
|
-
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
995
|
-
new_state = exc.state if exc.state else Paused()
|
996
|
-
if new_state.state_details.pause_reschedule:
|
997
|
-
# If we're being asked to pause and reschedule, we should exit the
|
998
|
-
# task and expect to be resumed later.
|
999
|
-
raise
|
912
|
+
if new_state.is_final():
|
913
|
+
if (
|
914
|
+
isinstance(new_state.data, BaseResult)
|
915
|
+
and new_state.data.has_cached_object()
|
916
|
+
):
|
917
|
+
# Avoid fetching the result unless it is cached, otherwise we defeat
|
918
|
+
# the purpose of disabling `cache_result_in_memory`
|
919
|
+
result = await new_state.result(raise_on_failure=False, fetch=True)
|
920
|
+
else:
|
921
|
+
result = new_state.data
|
1000
922
|
|
1001
|
-
|
1002
|
-
# that has an in-memory result attached to it; using the API state
|
1003
|
-
# could result in losing that reference
|
1004
|
-
self.task_run.state = new_state
|
923
|
+
link_state_to_result(new_state, result)
|
1005
924
|
|
1006
925
|
# emit a state change event
|
1007
926
|
self._last_event = emit_task_run_state_change_event(
|
@@ -1083,8 +1002,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1083
1002
|
else:
|
1084
1003
|
delay = None
|
1085
1004
|
new_state = Retrying()
|
1086
|
-
|
1087
|
-
self.task_run.run_count += 1
|
1005
|
+
self.task_run.run_count += 1
|
1088
1006
|
|
1089
1007
|
self.logger.info(
|
1090
1008
|
"Task run failed with exception: %r - " "Retry %s/%s will start %s",
|
@@ -1156,8 +1074,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1156
1074
|
if not self.task_run:
|
1157
1075
|
raise ValueError("Task run is not set")
|
1158
1076
|
|
1159
|
-
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
1160
|
-
self.task_run = await client.read_task_run(self.task_run.id)
|
1161
1077
|
with ExitStack() as stack:
|
1162
1078
|
if log_prints := should_log_prints(self.task):
|
1163
1079
|
stack.enter_context(patch_print())
|
@@ -1179,11 +1095,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1179
1095
|
task_run_name = _resolve_custom_task_run_name(
|
1180
1096
|
task=self.task, parameters=self.parameters
|
1181
1097
|
)
|
1182
|
-
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
1183
|
-
# update the task run name if necessary
|
1184
|
-
await self.client.set_task_run_name(
|
1185
|
-
task_run_id=self.task_run.id, name=task_run_name
|
1186
|
-
)
|
1187
1098
|
self.logger.extra["task_run_name"] = task_run_name
|
1188
1099
|
self.logger.debug(
|
1189
1100
|
f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
|
@@ -1207,38 +1118,21 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1207
1118
|
self._client = get_client()
|
1208
1119
|
self._is_started = True
|
1209
1120
|
try:
|
1210
|
-
if
|
1211
|
-
|
1212
|
-
|
1213
|
-
|
1214
|
-
|
1215
|
-
|
1216
|
-
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1220
|
-
|
1221
|
-
self.
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
)
|
1226
|
-
else:
|
1227
|
-
if not self.task_run:
|
1228
|
-
self.task_run = await self.task.create_run(
|
1229
|
-
id=task_run_id,
|
1230
|
-
parameters=self.parameters,
|
1231
|
-
flow_run_context=FlowRunContext.get(),
|
1232
|
-
parent_task_run_context=TaskRunContext.get(),
|
1233
|
-
wait_for=self.wait_for,
|
1234
|
-
extra_task_inputs=dependencies,
|
1235
|
-
)
|
1236
|
-
# Emit an event to capture that the task run was in the `PENDING` state.
|
1237
|
-
self._last_event = emit_task_run_state_change_event(
|
1238
|
-
task_run=self.task_run,
|
1239
|
-
initial_state=None,
|
1240
|
-
validated_state=self.task_run.state,
|
1241
|
-
)
|
1121
|
+
if not self.task_run:
|
1122
|
+
self.task_run = await self.task.create_local_run(
|
1123
|
+
id=task_run_id,
|
1124
|
+
parameters=self.parameters,
|
1125
|
+
flow_run_context=FlowRunContext.get(),
|
1126
|
+
parent_task_run_context=TaskRunContext.get(),
|
1127
|
+
wait_for=self.wait_for,
|
1128
|
+
extra_task_inputs=dependencies,
|
1129
|
+
)
|
1130
|
+
# Emit an event to capture that the task run was in the `PENDING` state.
|
1131
|
+
self._last_event = emit_task_run_state_change_event(
|
1132
|
+
task_run=self.task_run,
|
1133
|
+
initial_state=None,
|
1134
|
+
validated_state=self.task_run.state,
|
1135
|
+
)
|
1242
1136
|
|
1243
1137
|
async with self.setup_run_context():
|
1244
1138
|
# setup_run_context might update the task run name, so log creation here
|
@@ -1353,15 +1247,10 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1353
1247
|
if transaction.is_committed():
|
1354
1248
|
result = transaction.read()
|
1355
1249
|
else:
|
1356
|
-
if
|
1357
|
-
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION.value()
|
1358
|
-
and self.task.tags
|
1359
|
-
):
|
1250
|
+
if self.task.tags:
|
1360
1251
|
# Acquire a concurrency slot for each tag, but only if a limit
|
1361
1252
|
# matching the tag already exists.
|
1362
|
-
async with aconcurrency(
|
1363
|
-
list(self.task.tags), occupy=1, create_if_missing=False
|
1364
|
-
):
|
1253
|
+
async with aconcurrency(list(self.task.tags), self.task_run.id):
|
1365
1254
|
result = await call_with_parameters(self.task.fn, parameters)
|
1366
1255
|
else:
|
1367
1256
|
result = await call_with_parameters(self.task.fn, parameters)
|
prefect/task_worker.py
CHANGED
@@ -24,19 +24,17 @@ from prefect.cache_policies import DEFAULT, NONE
|
|
24
24
|
from prefect.client.orchestration import get_client
|
25
25
|
from prefect.client.schemas.objects import TaskRun
|
26
26
|
from prefect.client.subscriptions import Subscription
|
27
|
-
from prefect.exceptions import Abort, PrefectHTTPStatusError
|
28
27
|
from prefect.logging.loggers import get_logger
|
29
28
|
from prefect.results import ResultFactory
|
30
29
|
from prefect.settings import (
|
31
30
|
PREFECT_API_URL,
|
32
|
-
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION,
|
33
31
|
PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS,
|
34
32
|
)
|
35
33
|
from prefect.states import Pending
|
36
34
|
from prefect.task_engine import run_task_async, run_task_sync
|
37
35
|
from prefect.utilities.annotations import NotSet
|
38
36
|
from prefect.utilities.asyncutils import asyncnullcontext, sync_compatible
|
39
|
-
from prefect.utilities.engine import emit_task_run_state_change_event
|
37
|
+
from prefect.utilities.engine import emit_task_run_state_change_event
|
40
38
|
from prefect.utilities.processutils import _register_signal
|
41
39
|
from prefect.utilities.services import start_client_metrics_server
|
42
40
|
from prefect.utilities.urls import url_for
|
@@ -294,42 +292,12 @@ class TaskWorker:
|
|
294
292
|
return
|
295
293
|
|
296
294
|
initial_state = task_run.state
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
state = new_state
|
304
|
-
task_run.state = state
|
305
|
-
else:
|
306
|
-
try:
|
307
|
-
new_state = Pending()
|
308
|
-
new_state.state_details.deferred = True
|
309
|
-
state = await propose_state(
|
310
|
-
client=get_client(), # TODO prove that we cannot use self._client here
|
311
|
-
state=new_state,
|
312
|
-
task_run_id=task_run.id,
|
313
|
-
)
|
314
|
-
except Abort as exc:
|
315
|
-
logger.exception(
|
316
|
-
f"Failed to submit task run {task_run.id!r} to engine", exc_info=exc
|
317
|
-
)
|
318
|
-
return
|
319
|
-
except PrefectHTTPStatusError as exc:
|
320
|
-
if exc.response.status_code == 404:
|
321
|
-
logger.warning(
|
322
|
-
f"Task run {task_run.id!r} not found. It may have been deleted."
|
323
|
-
)
|
324
|
-
return
|
325
|
-
raise
|
326
|
-
|
327
|
-
if not state.is_pending():
|
328
|
-
logger.warning(
|
329
|
-
f"Cancelling submission of task run {task_run.id!r} -"
|
330
|
-
f" server returned a non-pending state {state.type.value!r}."
|
331
|
-
)
|
332
|
-
return
|
295
|
+
new_state = Pending()
|
296
|
+
new_state.state_details.deferred = True
|
297
|
+
new_state.state_details.task_run_id = task_run.id
|
298
|
+
new_state.state_details.flow_run_id = task_run.flow_run_id
|
299
|
+
state = new_state
|
300
|
+
task_run.state = state
|
333
301
|
|
334
302
|
emit_task_run_state_change_event(
|
335
303
|
task_run=task_run,
|
prefect/tasks.py
CHANGED
@@ -52,7 +52,6 @@ from prefect.futures import PrefectDistributedFuture, PrefectFuture, PrefectFutu
|
|
52
52
|
from prefect.logging.loggers import get_logger
|
53
53
|
from prefect.results import ResultFactory, ResultSerializer, ResultStorage
|
54
54
|
from prefect.settings import (
|
55
|
-
PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION,
|
56
55
|
PREFECT_TASK_DEFAULT_RETRIES,
|
57
56
|
PREFECT_TASK_DEFAULT_RETRY_DELAY_SECONDS,
|
58
57
|
)
|
@@ -815,12 +814,6 @@ class Task(Generic[P, R]):
|
|
815
814
|
extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
|
816
815
|
deferred: bool = False,
|
817
816
|
) -> TaskRun:
|
818
|
-
if not PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
|
819
|
-
raise RuntimeError(
|
820
|
-
"Cannot call `Task.create_local_run` unless "
|
821
|
-
"PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION is True"
|
822
|
-
)
|
823
|
-
|
824
817
|
from prefect.utilities.engine import (
|
825
818
|
_dynamic_key_for_task_run,
|
826
819
|
collect_task_run_inputs_sync,
|
@@ -841,9 +834,9 @@ class Task(Generic[P, R]):
|
|
841
834
|
task_run_name = self.name
|
842
835
|
else:
|
843
836
|
dynamic_key = _dynamic_key_for_task_run(
|
844
|
-
context=flow_run_context, task=self
|
837
|
+
context=flow_run_context, task=self, stable=False
|
845
838
|
)
|
846
|
-
task_run_name = f"{self.name}-{dynamic_key}"
|
839
|
+
task_run_name = f"{self.name}-{dynamic_key[:3]}"
|
847
840
|
|
848
841
|
if deferred:
|
849
842
|
state = Scheduled()
|