prefect-client 3.1.9__py3-none-any.whl → 3.1.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/lineage.py +7 -8
- prefect/_internal/_logging.py +15 -3
- prefect/_internal/compatibility/async_dispatch.py +22 -16
- prefect/_internal/compatibility/deprecated.py +42 -18
- prefect/_internal/compatibility/migration.py +2 -2
- prefect/_internal/concurrency/inspection.py +12 -14
- prefect/_internal/concurrency/primitives.py +2 -2
- prefect/_internal/concurrency/services.py +154 -80
- prefect/_internal/concurrency/waiters.py +13 -9
- prefect/_internal/pydantic/annotations/pendulum.py +7 -7
- prefect/_internal/pytz.py +4 -3
- prefect/_internal/retries.py +10 -5
- prefect/_internal/schemas/bases.py +19 -10
- prefect/_internal/schemas/validators.py +227 -388
- prefect/_version.py +3 -3
- prefect/artifacts.py +61 -74
- prefect/automations.py +27 -7
- prefect/blocks/core.py +3 -3
- prefect/client/{orchestration.py → orchestration/__init__.py} +38 -701
- prefect/client/orchestration/_artifacts/__init__.py +0 -0
- prefect/client/orchestration/_artifacts/client.py +239 -0
- prefect/client/orchestration/_concurrency_limits/__init__.py +0 -0
- prefect/client/orchestration/_concurrency_limits/client.py +762 -0
- prefect/client/orchestration/_logs/__init__.py +0 -0
- prefect/client/orchestration/_logs/client.py +95 -0
- prefect/client/orchestration/_variables/__init__.py +0 -0
- prefect/client/orchestration/_variables/client.py +157 -0
- prefect/client/orchestration/base.py +46 -0
- prefect/client/orchestration/routes.py +145 -0
- prefect/client/schemas/actions.py +2 -2
- prefect/client/schemas/filters.py +5 -0
- prefect/client/schemas/objects.py +3 -10
- prefect/client/schemas/schedules.py +22 -10
- prefect/concurrency/_asyncio.py +87 -0
- prefect/concurrency/{events.py → _events.py} +10 -10
- prefect/concurrency/asyncio.py +20 -104
- prefect/concurrency/context.py +6 -4
- prefect/concurrency/services.py +26 -74
- prefect/concurrency/sync.py +23 -44
- prefect/concurrency/v1/_asyncio.py +63 -0
- prefect/concurrency/v1/{events.py → _events.py} +13 -15
- prefect/concurrency/v1/asyncio.py +27 -80
- prefect/concurrency/v1/context.py +6 -4
- prefect/concurrency/v1/services.py +33 -79
- prefect/concurrency/v1/sync.py +18 -37
- prefect/context.py +66 -70
- prefect/deployments/base.py +4 -144
- prefect/deployments/flow_runs.py +12 -2
- prefect/deployments/runner.py +11 -3
- prefect/deployments/steps/pull.py +13 -0
- prefect/events/clients.py +7 -1
- prefect/events/schemas/events.py +3 -2
- prefect/flow_engine.py +54 -47
- prefect/flows.py +2 -1
- prefect/futures.py +42 -27
- prefect/input/run_input.py +2 -1
- prefect/locking/filesystem.py +8 -7
- prefect/locking/memory.py +5 -3
- prefect/locking/protocol.py +1 -1
- prefect/main.py +1 -3
- prefect/plugins.py +12 -10
- prefect/results.py +3 -308
- prefect/runner/storage.py +87 -21
- prefect/serializers.py +32 -25
- prefect/settings/legacy.py +4 -4
- prefect/settings/models/api.py +3 -3
- prefect/settings/models/cli.py +3 -3
- prefect/settings/models/client.py +5 -3
- prefect/settings/models/cloud.py +3 -3
- prefect/settings/models/deployments.py +3 -3
- prefect/settings/models/experiments.py +4 -2
- prefect/settings/models/flows.py +3 -3
- prefect/settings/models/internal.py +4 -2
- prefect/settings/models/logging.py +4 -3
- prefect/settings/models/results.py +3 -3
- prefect/settings/models/root.py +3 -2
- prefect/settings/models/runner.py +4 -4
- prefect/settings/models/server/api.py +3 -3
- prefect/settings/models/server/database.py +11 -4
- prefect/settings/models/server/deployments.py +6 -2
- prefect/settings/models/server/ephemeral.py +4 -2
- prefect/settings/models/server/events.py +3 -2
- prefect/settings/models/server/flow_run_graph.py +6 -2
- prefect/settings/models/server/root.py +3 -3
- prefect/settings/models/server/services.py +26 -11
- prefect/settings/models/server/tasks.py +6 -3
- prefect/settings/models/server/ui.py +3 -3
- prefect/settings/models/tasks.py +5 -5
- prefect/settings/models/testing.py +3 -3
- prefect/settings/models/worker.py +5 -3
- prefect/settings/profiles.py +15 -2
- prefect/states.py +4 -7
- prefect/task_engine.py +54 -75
- prefect/tasks.py +84 -32
- prefect/telemetry/processors.py +6 -6
- prefect/telemetry/run_telemetry.py +13 -8
- prefect/telemetry/services.py +32 -31
- prefect/transactions.py +4 -15
- prefect/utilities/_git.py +34 -0
- prefect/utilities/asyncutils.py +1 -1
- prefect/utilities/engine.py +3 -19
- prefect/utilities/generics.py +18 -0
- prefect/workers/__init__.py +2 -0
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/METADATA +1 -1
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/RECORD +108 -99
- prefect/records/__init__.py +0 -1
- prefect/records/base.py +0 -235
- prefect/records/filesystem.py +0 -213
- prefect/records/memory.py +0 -184
- prefect/records/result_store.py +0 -70
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/top_level.txt +0 -0
prefect/tasks.py
CHANGED
@@ -74,7 +74,7 @@ if TYPE_CHECKING:
|
|
74
74
|
from prefect.context import TaskRunContext
|
75
75
|
from prefect.transactions import Transaction
|
76
76
|
|
77
|
-
T = TypeVar("T")
|
77
|
+
T = TypeVar("T")
|
78
78
|
R = TypeVar("R") # The return type of the user's function
|
79
79
|
P = ParamSpec("P") # The parameters of the task
|
80
80
|
|
@@ -82,6 +82,11 @@ NUM_CHARS_DYNAMIC_KEY = 8
|
|
82
82
|
|
83
83
|
logger = get_logger("tasks")
|
84
84
|
|
85
|
+
FutureOrResult: TypeAlias = Union[PrefectFuture[T], T]
|
86
|
+
OneOrManyFutureOrResult: TypeAlias = Union[
|
87
|
+
FutureOrResult[T], Iterable[FutureOrResult[T]]
|
88
|
+
]
|
89
|
+
|
85
90
|
|
86
91
|
def task_input_hash(
|
87
92
|
context: "TaskRunContext", arguments: dict[str, Any]
|
@@ -737,7 +742,7 @@ class Task(Generic[P, R]):
|
|
737
742
|
parameters: Optional[dict[str, Any]] = None,
|
738
743
|
flow_run_context: Optional[FlowRunContext] = None,
|
739
744
|
parent_task_run_context: Optional[TaskRunContext] = None,
|
740
|
-
wait_for: Optional[
|
745
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
741
746
|
extra_task_inputs: Optional[dict[str, set[TaskRunInput]]] = None,
|
742
747
|
deferred: bool = False,
|
743
748
|
) -> TaskRun:
|
@@ -838,7 +843,7 @@ class Task(Generic[P, R]):
|
|
838
843
|
parameters: Optional[dict[str, Any]] = None,
|
839
844
|
flow_run_context: Optional[FlowRunContext] = None,
|
840
845
|
parent_task_run_context: Optional[TaskRunContext] = None,
|
841
|
-
wait_for: Optional[
|
846
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
842
847
|
extra_task_inputs: Optional[dict[str, set[TaskRunInput]]] = None,
|
843
848
|
deferred: bool = False,
|
844
849
|
) -> TaskRun:
|
@@ -952,6 +957,8 @@ class Task(Generic[P, R]):
|
|
952
957
|
def __call__(
|
953
958
|
self: "Task[P, NoReturn]",
|
954
959
|
*args: P.args,
|
960
|
+
return_state: Literal[False],
|
961
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
955
962
|
**kwargs: P.kwargs,
|
956
963
|
) -> None:
|
957
964
|
# `NoReturn` matches if a type can't be inferred for the function which stops a
|
@@ -960,28 +967,41 @@ class Task(Generic[P, R]):
|
|
960
967
|
|
961
968
|
@overload
|
962
969
|
def __call__(
|
963
|
-
self: "Task[P,
|
970
|
+
self: "Task[P, R]",
|
964
971
|
*args: P.args,
|
972
|
+
return_state: Literal[True],
|
973
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
965
974
|
**kwargs: P.kwargs,
|
966
|
-
) ->
|
975
|
+
) -> State[R]:
|
967
976
|
...
|
968
977
|
|
969
978
|
@overload
|
970
979
|
def __call__(
|
971
|
-
self: "Task[P,
|
980
|
+
self: "Task[P, R]",
|
972
981
|
*args: P.args,
|
973
|
-
return_state: Literal[
|
982
|
+
return_state: Literal[False],
|
983
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
974
984
|
**kwargs: P.kwargs,
|
975
|
-
) ->
|
985
|
+
) -> R:
|
976
986
|
...
|
977
987
|
|
988
|
+
@overload
|
978
989
|
def __call__(
|
979
|
-
self,
|
990
|
+
self: "Task[P, R]",
|
991
|
+
*args: P.args,
|
992
|
+
return_state: Literal[False] = False,
|
993
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
994
|
+
**kwargs: P.kwargs,
|
995
|
+
) -> R:
|
996
|
+
...
|
997
|
+
|
998
|
+
def __call__(
|
999
|
+
self: "Union[Task[P, R], Task[P, NoReturn]]",
|
980
1000
|
*args: P.args,
|
981
1001
|
return_state: bool = False,
|
982
|
-
wait_for: Optional[
|
1002
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
983
1003
|
**kwargs: P.kwargs,
|
984
|
-
):
|
1004
|
+
) -> Union[R, State[R], None]:
|
985
1005
|
"""
|
986
1006
|
Run the task and return the result. If `return_state` is True returns
|
987
1007
|
the result is wrapped in a Prefect State which provides error handling.
|
@@ -1013,53 +1033,57 @@ class Task(Generic[P, R]):
|
|
1013
1033
|
|
1014
1034
|
@overload
|
1015
1035
|
def submit(
|
1016
|
-
self: "Task[P,
|
1036
|
+
self: "Task[P, R]",
|
1017
1037
|
*args: P.args,
|
1018
1038
|
**kwargs: P.kwargs,
|
1019
|
-
) -> PrefectFuture[
|
1020
|
-
# `NoReturn` matches if a type can't be inferred for the function which stops a
|
1021
|
-
# sync function from matching the `Coroutine` overload
|
1039
|
+
) -> PrefectFuture[R]:
|
1022
1040
|
...
|
1023
1041
|
|
1024
1042
|
@overload
|
1025
1043
|
def submit(
|
1026
|
-
self: "Task[P, Coroutine[Any, Any,
|
1044
|
+
self: "Task[P, Coroutine[Any, Any, R]]",
|
1027
1045
|
*args: P.args,
|
1046
|
+
return_state: Literal[False],
|
1047
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
1028
1048
|
**kwargs: P.kwargs,
|
1029
|
-
) -> PrefectFuture[
|
1049
|
+
) -> PrefectFuture[R]:
|
1030
1050
|
...
|
1031
1051
|
|
1032
1052
|
@overload
|
1033
1053
|
def submit(
|
1034
|
-
self: "Task[P,
|
1054
|
+
self: "Task[P, R]",
|
1035
1055
|
*args: P.args,
|
1056
|
+
return_state: Literal[False],
|
1057
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
1036
1058
|
**kwargs: P.kwargs,
|
1037
|
-
) -> PrefectFuture[
|
1059
|
+
) -> PrefectFuture[R]:
|
1038
1060
|
...
|
1039
1061
|
|
1040
1062
|
@overload
|
1041
1063
|
def submit(
|
1042
|
-
self: "Task[P, Coroutine[Any, Any,
|
1064
|
+
self: "Task[P, Coroutine[Any, Any, R]]",
|
1043
1065
|
*args: P.args,
|
1044
1066
|
return_state: Literal[True],
|
1067
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
1045
1068
|
**kwargs: P.kwargs,
|
1046
|
-
) -> State[
|
1069
|
+
) -> State[R]:
|
1047
1070
|
...
|
1048
1071
|
|
1049
1072
|
@overload
|
1050
1073
|
def submit(
|
1051
|
-
self: "Task[P,
|
1074
|
+
self: "Task[P, R]",
|
1052
1075
|
*args: P.args,
|
1053
1076
|
return_state: Literal[True],
|
1077
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
1054
1078
|
**kwargs: P.kwargs,
|
1055
|
-
) -> State[
|
1079
|
+
) -> State[R]:
|
1056
1080
|
...
|
1057
1081
|
|
1058
1082
|
def submit(
|
1059
|
-
self,
|
1083
|
+
self: "Union[Task[P, R], Task[P, Coroutine[Any, Any, R]]]",
|
1060
1084
|
*args: Any,
|
1061
1085
|
return_state: bool = False,
|
1062
|
-
wait_for: Optional[
|
1086
|
+
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
1063
1087
|
**kwargs: Any,
|
1064
1088
|
):
|
1065
1089
|
"""
|
@@ -1584,11 +1608,39 @@ def task(__fn: Callable[P, R]) -> Task[P, R]:
|
|
1584
1608
|
@overload
|
1585
1609
|
def task(
|
1586
1610
|
__fn: Literal[None] = None,
|
1611
|
+
*,
|
1612
|
+
name: Optional[str] = None,
|
1613
|
+
description: Optional[str] = None,
|
1614
|
+
tags: Optional[Iterable[str]] = None,
|
1615
|
+
version: Optional[str] = None,
|
1616
|
+
cache_policy: Union[CachePolicy, type[NotSet]] = NotSet,
|
1617
|
+
cache_key_fn: Optional[
|
1618
|
+
Callable[["TaskRunContext", dict[str, Any]], Optional[str]]
|
1619
|
+
] = None,
|
1620
|
+
cache_expiration: Optional[datetime.timedelta] = None,
|
1621
|
+
task_run_name: Optional[TaskRunNameValueOrCallable] = None,
|
1622
|
+
retries: int = 0,
|
1623
|
+
retry_delay_seconds: Union[
|
1624
|
+
float, int, list[float], Callable[[int], list[float]], None
|
1625
|
+
] = None,
|
1626
|
+
retry_jitter_factor: Optional[float] = None,
|
1627
|
+
persist_result: Optional[bool] = None,
|
1628
|
+
result_storage: Optional[ResultStorage] = None,
|
1629
|
+
result_storage_key: Optional[str] = None,
|
1630
|
+
result_serializer: Optional[ResultSerializer] = None,
|
1631
|
+
cache_result_in_memory: bool = True,
|
1632
|
+
timeout_seconds: Union[int, float, None] = None,
|
1633
|
+
log_prints: Optional[bool] = None,
|
1634
|
+
refresh_cache: Optional[bool] = None,
|
1635
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
1636
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
1637
|
+
retry_condition_fn: Optional[Callable[[Task[P, Any], TaskRun, State], bool]] = None,
|
1638
|
+
viz_return_value: Any = None,
|
1587
1639
|
) -> Callable[[Callable[P, R]], Task[P, R]]:
|
1588
1640
|
...
|
1589
1641
|
|
1590
1642
|
|
1591
|
-
@overload
|
1643
|
+
@overload # TODO: do we need this overload?
|
1592
1644
|
def task(
|
1593
1645
|
*,
|
1594
1646
|
name: Optional[str] = None,
|
@@ -1619,7 +1671,7 @@ def task(
|
|
1619
1671
|
refresh_cache: Optional[bool] = None,
|
1620
1672
|
on_completion: Optional[list[StateHookCallable]] = None,
|
1621
1673
|
on_failure: Optional[list[StateHookCallable]] = None,
|
1622
|
-
retry_condition_fn: Optional[Callable[[
|
1674
|
+
retry_condition_fn: Optional[Callable[[Task[P, Any], TaskRun, State], bool]] = None,
|
1623
1675
|
viz_return_value: Any = None,
|
1624
1676
|
) -> Callable[[Callable[P, R]], Task[P, R]]:
|
1625
1677
|
...
|
@@ -1653,7 +1705,7 @@ def task(
|
|
1653
1705
|
refresh_cache: Optional[bool] = None,
|
1654
1706
|
on_completion: Optional[list[StateHookCallable]] = None,
|
1655
1707
|
on_failure: Optional[list[StateHookCallable]] = None,
|
1656
|
-
retry_condition_fn: Optional[Callable[[
|
1708
|
+
retry_condition_fn: Optional[Callable[[Task[P, Any], TaskRun, State], bool]] = None,
|
1657
1709
|
viz_return_value: Any = None,
|
1658
1710
|
):
|
1659
1711
|
"""
|
@@ -1685,10 +1737,10 @@ def task(
|
|
1685
1737
|
callable that, given the total number of retries, generates a list of retry
|
1686
1738
|
delays. If a number of seconds, that delay will be applied to all retries.
|
1687
1739
|
If a list, each retry will wait for the corresponding delay before retrying.
|
1688
|
-
When passing a callable or a list, the number of
|
1689
|
-
cannot exceed 50.
|
1690
|
-
retry_jitter_factor: An optional factor that defines the factor to which a
|
1691
|
-
can be jittered in order to avoid a "thundering herd".
|
1740
|
+
When passing a callable or a list, the number of
|
1741
|
+
configured retry delays cannot exceed 50.
|
1742
|
+
retry_jitter_factor: An optional factor that defines the factor to which a
|
1743
|
+
retry can be jittered in order to avoid a "thundering herd".
|
1692
1744
|
persist_result: A toggle indicating whether the result of this task
|
1693
1745
|
should be persisted to result storage. Defaults to `None`, which
|
1694
1746
|
indicates that the global default should be used (which is `True` by
|
prefect/telemetry/processors.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
import time
|
2
2
|
from threading import Event, Lock, Thread
|
3
|
-
from typing import TYPE_CHECKING,
|
3
|
+
from typing import TYPE_CHECKING, Optional
|
4
4
|
|
5
5
|
from opentelemetry.context import Context
|
6
6
|
from opentelemetry.sdk.trace import Span, SpanProcessor
|
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
|
|
13
13
|
class InFlightSpanProcessor(SpanProcessor):
|
14
14
|
def __init__(self, span_exporter: "SpanExporter"):
|
15
15
|
self.span_exporter = span_exporter
|
16
|
-
self._in_flight:
|
16
|
+
self._in_flight: dict[int, Span] = {}
|
17
17
|
self._lock = Lock()
|
18
18
|
self._stop_event = Event()
|
19
19
|
self._export_thread = Thread(target=self._export_periodically, daemon=True)
|
@@ -30,10 +30,10 @@ class InFlightSpanProcessor(SpanProcessor):
|
|
30
30
|
self.span_exporter.export(to_export)
|
31
31
|
|
32
32
|
def _readable_span(self, span: "Span") -> "ReadableSpan":
|
33
|
-
readable = span._readable_span()
|
34
|
-
readable._end_time = time.time_ns()
|
35
|
-
readable._attributes = {
|
36
|
-
**(readable._attributes or {}),
|
33
|
+
readable = span._readable_span() # pyright: ignore[reportPrivateUsage]
|
34
|
+
readable._end_time = time.time_ns() # pyright: ignore[reportPrivateUsage]
|
35
|
+
readable._attributes = { # pyright: ignore[reportPrivateUsage]
|
36
|
+
**(readable._attributes or {}), # pyright: ignore[reportPrivateUsage]
|
37
37
|
"prefect.in-flight": True,
|
38
38
|
}
|
39
39
|
return readable
|
@@ -53,10 +53,9 @@ class RunTelemetry:
|
|
53
53
|
self,
|
54
54
|
run: FlowOrTaskRun,
|
55
55
|
client: PrefectClient,
|
56
|
-
name: Optional[str] = None,
|
57
56
|
parameters: Optional[dict[str, Any]] = None,
|
58
57
|
):
|
59
|
-
traceparent, span = self._start_span(run,
|
58
|
+
traceparent, span = self._start_span(run, parameters)
|
60
59
|
|
61
60
|
if self._run_type(run) == "flow" and traceparent:
|
62
61
|
# Only explicitly update labels if the run is a flow as task runs
|
@@ -71,10 +70,9 @@ class RunTelemetry:
|
|
71
70
|
self,
|
72
71
|
run: FlowOrTaskRun,
|
73
72
|
client: SyncPrefectClient,
|
74
|
-
name: Optional[str] = None,
|
75
73
|
parameters: Optional[dict[str, Any]] = None,
|
76
74
|
):
|
77
|
-
traceparent, span = self._start_span(run,
|
75
|
+
traceparent, span = self._start_span(run, parameters)
|
78
76
|
|
79
77
|
if self._run_type(run) == "flow" and traceparent:
|
80
78
|
# Only explicitly update labels if the run is a flow as task runs
|
@@ -86,7 +84,6 @@ class RunTelemetry:
|
|
86
84
|
def _start_span(
|
87
85
|
self,
|
88
86
|
run: FlowOrTaskRun,
|
89
|
-
name: Optional[str] = None,
|
90
87
|
parameters: Optional[dict[str, Any]] = None,
|
91
88
|
) -> tuple[Optional[str], Span]:
|
92
89
|
"""
|
@@ -117,10 +114,10 @@ class RunTelemetry:
|
|
117
114
|
run_type = self._run_type(run)
|
118
115
|
|
119
116
|
self.span = self._tracer.start_span(
|
120
|
-
name=
|
117
|
+
name=run.name,
|
121
118
|
context=context,
|
122
119
|
attributes={
|
123
|
-
"prefect.run.name":
|
120
|
+
"prefect.run.name": run.name,
|
124
121
|
"prefect.run.type": run_type,
|
125
122
|
"prefect.run.id": str(run.id),
|
126
123
|
"prefect.tags": run.tags,
|
@@ -152,7 +149,7 @@ class RunTelemetry:
|
|
152
149
|
return propagate.extract(carrier)
|
153
150
|
|
154
151
|
def _traceparent_from_span(self, span: Span) -> Optional[str]:
|
155
|
-
carrier = {}
|
152
|
+
carrier: dict[str, Any] = {}
|
156
153
|
propagate.inject(carrier, context=trace.set_span_in_context(span))
|
157
154
|
return carrier.get(TRACEPARENT_KEY)
|
158
155
|
|
@@ -198,6 +195,14 @@ class RunTelemetry:
|
|
198
195
|
},
|
199
196
|
)
|
200
197
|
|
198
|
+
def update_run_name(self, name: str) -> None:
|
199
|
+
"""
|
200
|
+
Update the name of the run.
|
201
|
+
"""
|
202
|
+
if self.span:
|
203
|
+
self.span.update_name(name=name)
|
204
|
+
self.span.set_attribute("prefect.run.name", name)
|
205
|
+
|
201
206
|
def _parent_run(self) -> Union[FlowOrTaskRun, None]:
|
202
207
|
"""
|
203
208
|
Identify the "parent run" for the current execution context.
|
prefect/telemetry/services.py
CHANGED
@@ -1,32 +1,38 @@
|
|
1
|
-
from abc import
|
2
|
-
from typing import
|
1
|
+
from collections.abc import Sequence
|
2
|
+
from typing import Any, Protocol, TypeVar
|
3
3
|
|
4
4
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
5
5
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
6
6
|
from opentelemetry.sdk._logs import LogData
|
7
7
|
from opentelemetry.sdk._logs.export import LogExporter
|
8
8
|
from opentelemetry.sdk.trace import ReadableSpan
|
9
|
-
from opentelemetry.sdk.trace.export import SpanExporter
|
9
|
+
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
10
10
|
|
11
11
|
from prefect._internal.concurrency.services import BatchedQueueService
|
12
12
|
|
13
|
+
BatchItem = TypeVar("BatchItem", ReadableSpan, LogData)
|
14
|
+
T_contra = TypeVar("T_contra", contravariant=True)
|
13
15
|
|
14
|
-
|
16
|
+
|
17
|
+
class OTLPExporter(Protocol[T_contra]):
|
18
|
+
def export(self, __items: Sequence[T_contra]) -> Any:
|
19
|
+
...
|
20
|
+
|
21
|
+
def shutdown(self) -> Any:
|
22
|
+
...
|
23
|
+
|
24
|
+
|
25
|
+
class BaseQueueingExporter(BatchedQueueService[BatchItem]):
|
15
26
|
_max_batch_size = 512
|
16
27
|
_min_interval = 2.0
|
17
|
-
_otlp_exporter: Union[SpanExporter, LogExporter]
|
18
|
-
|
19
|
-
def export(self, batch: list[Union[ReadableSpan, LogData]]) -> None:
|
20
|
-
for item in batch:
|
21
|
-
self.send(item)
|
22
28
|
|
23
|
-
|
24
|
-
|
25
|
-
|
29
|
+
def __init__(self, otlp_exporter: OTLPExporter[BatchItem]) -> None:
|
30
|
+
super().__init__()
|
31
|
+
self._otlp_exporter = otlp_exporter
|
26
32
|
|
27
|
-
async def _handle_batch(self, items: list[
|
33
|
+
async def _handle_batch(self, items: list[BatchItem]) -> None:
|
28
34
|
try:
|
29
|
-
self.
|
35
|
+
self._otlp_exporter.export(items)
|
30
36
|
except Exception as e:
|
31
37
|
self._logger.exception(f"Failed to export batch: {e}")
|
32
38
|
raise
|
@@ -39,29 +45,24 @@ class BaseQueueingExporter(BatchedQueueService):
|
|
39
45
|
self._otlp_exporter.shutdown()
|
40
46
|
|
41
47
|
|
42
|
-
class QueueingSpanExporter(BaseQueueingExporter, SpanExporter):
|
48
|
+
class QueueingSpanExporter(BaseQueueingExporter[ReadableSpan], SpanExporter):
|
43
49
|
_otlp_exporter: OTLPSpanExporter
|
44
50
|
|
45
51
|
def __init__(self, endpoint: str, headers: tuple[tuple[str, str]]):
|
46
|
-
super().__init__()
|
47
|
-
self._otlp_exporter = OTLPSpanExporter(
|
48
|
-
endpoint=endpoint,
|
49
|
-
headers=dict(headers),
|
50
|
-
)
|
52
|
+
super().__init__(OTLPSpanExporter(endpoint=endpoint, headers=dict(headers)))
|
51
53
|
|
52
|
-
def
|
53
|
-
|
54
|
+
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
55
|
+
for item in spans:
|
56
|
+
self.send(item)
|
57
|
+
return SpanExportResult.SUCCESS
|
54
58
|
|
55
59
|
|
56
|
-
class QueueingLogExporter(BaseQueueingExporter, LogExporter):
|
60
|
+
class QueueingLogExporter(BaseQueueingExporter[LogData], LogExporter):
|
57
61
|
_otlp_exporter: OTLPLogExporter
|
58
62
|
|
59
|
-
def __init__(self, endpoint: str, headers: tuple[tuple[str, str]]):
|
60
|
-
super().__init__()
|
61
|
-
self._otlp_exporter = OTLPLogExporter(
|
62
|
-
endpoint=endpoint,
|
63
|
-
headers=dict(headers),
|
64
|
-
)
|
63
|
+
def __init__(self, endpoint: str, headers: tuple[tuple[str, str]]) -> None:
|
64
|
+
super().__init__(OTLPLogExporter(endpoint=endpoint, headers=dict(headers)))
|
65
65
|
|
66
|
-
def
|
67
|
-
|
66
|
+
def export(self, batch: Sequence[LogData]) -> None:
|
67
|
+
for item in batch:
|
68
|
+
self.send(item)
|
prefect/transactions.py
CHANGED
@@ -24,10 +24,7 @@ from prefect.exceptions import (
|
|
24
24
|
SerializationError,
|
25
25
|
)
|
26
26
|
from prefect.logging.loggers import LoggingAdapter, get_logger, get_run_logger
|
27
|
-
from prefect.records import RecordStore
|
28
|
-
from prefect.records.base import TransactionRecord
|
29
27
|
from prefect.results import (
|
30
|
-
BaseResult,
|
31
28
|
ResultRecord,
|
32
29
|
ResultStore,
|
33
30
|
get_result_store,
|
@@ -61,7 +58,7 @@ class Transaction(ContextModel):
|
|
61
58
|
A base model for transaction state.
|
62
59
|
"""
|
63
60
|
|
64
|
-
store:
|
61
|
+
store: Optional[ResultStore] = None
|
65
62
|
key: Optional[str] = None
|
66
63
|
children: List["Transaction"] = Field(default_factory=list)
|
67
64
|
commit_mode: Optional[CommitMode] = None
|
@@ -254,15 +251,11 @@ class Transaction(ContextModel):
|
|
254
251
|
):
|
255
252
|
self.state = TransactionState.COMMITTED
|
256
253
|
|
257
|
-
def read(self) ->
|
254
|
+
def read(self) -> Optional[ResultRecord[Any]]:
|
258
255
|
if self.store and self.key:
|
259
256
|
record = self.store.read(key=self.key)
|
260
257
|
if isinstance(record, ResultRecord):
|
261
258
|
return record
|
262
|
-
# for backwards compatibility, if we encounter a transaction record, return the result
|
263
|
-
# This happens when the transaction is using a `ResultStore`
|
264
|
-
if isinstance(record, TransactionRecord):
|
265
|
-
return record.result
|
266
259
|
return None
|
267
260
|
|
268
261
|
def reset(self) -> None:
|
@@ -315,11 +308,7 @@ class Transaction(ContextModel):
|
|
315
308
|
|
316
309
|
if self.store and self.key and self.write_on_commit:
|
317
310
|
if isinstance(self.store, ResultStore):
|
318
|
-
if isinstance(self._staged_value,
|
319
|
-
self.store.write(
|
320
|
-
key=self.key, obj=self._staged_value.get(_sync=True)
|
321
|
-
)
|
322
|
-
elif isinstance(self._staged_value, ResultRecord):
|
311
|
+
if isinstance(self._staged_value, ResultRecord):
|
323
312
|
self.store.persist_result_record(
|
324
313
|
result_record=self._staged_value
|
325
314
|
)
|
@@ -436,7 +425,7 @@ def get_transaction() -> Optional[Transaction]:
|
|
436
425
|
@contextmanager
|
437
426
|
def transaction(
|
438
427
|
key: Optional[str] = None,
|
439
|
-
store:
|
428
|
+
store: Optional[ResultStore] = None,
|
440
429
|
commit_mode: Optional[CommitMode] = None,
|
441
430
|
isolation_level: Optional[IsolationLevel] = None,
|
442
431
|
overwrite: bool = False,
|
@@ -0,0 +1,34 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import subprocess
|
4
|
+
import sys
|
5
|
+
|
6
|
+
|
7
|
+
def get_git_remote_origin_url() -> str | None:
|
8
|
+
"""
|
9
|
+
Returns the git remote origin URL for the current directory.
|
10
|
+
"""
|
11
|
+
try:
|
12
|
+
origin_url = subprocess.check_output(
|
13
|
+
["git", "config", "--get", "remote.origin.url"],
|
14
|
+
shell=sys.platform == "win32",
|
15
|
+
stderr=subprocess.DEVNULL,
|
16
|
+
)
|
17
|
+
origin_url = origin_url.decode().strip()
|
18
|
+
except subprocess.CalledProcessError:
|
19
|
+
return None
|
20
|
+
|
21
|
+
return origin_url
|
22
|
+
|
23
|
+
|
24
|
+
def get_git_branch() -> str | None:
|
25
|
+
"""
|
26
|
+
Returns the git branch for the current directory.
|
27
|
+
"""
|
28
|
+
try:
|
29
|
+
branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"])
|
30
|
+
branch = branch.decode().strip()
|
31
|
+
except subprocess.CalledProcessError:
|
32
|
+
return None
|
33
|
+
|
34
|
+
return branch
|
prefect/utilities/asyncutils.py
CHANGED
@@ -77,7 +77,7 @@ def get_thread_limiter() -> anyio.CapacityLimiter:
|
|
77
77
|
|
78
78
|
def is_async_fn(
|
79
79
|
func: _SyncOrAsyncCallable[P, R],
|
80
|
-
) -> TypeGuard[Callable[P,
|
80
|
+
) -> TypeGuard[Callable[P, Coroutine[Any, Any, Any]]]:
|
81
81
|
"""
|
82
82
|
Returns `True` if a function returns a coroutine.
|
83
83
|
|
prefect/utilities/engine.py
CHANGED
@@ -45,7 +45,7 @@ from prefect.exceptions import (
|
|
45
45
|
from prefect.flows import Flow
|
46
46
|
from prefect.futures import PrefectFuture
|
47
47
|
from prefect.logging.loggers import get_logger
|
48
|
-
from prefect.results import
|
48
|
+
from prefect.results import ResultRecord, should_persist_result
|
49
49
|
from prefect.settings import PREFECT_LOGGING_LOG_PRINTS
|
50
50
|
from prefect.states import State
|
51
51
|
from prefect.tasks import Task
|
@@ -283,10 +283,6 @@ async def resolve_inputs(
|
|
283
283
|
return resolved_parameters
|
284
284
|
|
285
285
|
|
286
|
-
def _is_base_result(data: Any) -> TypeIs[BaseResult[Any]]:
|
287
|
-
return isinstance(data, BaseResult)
|
288
|
-
|
289
|
-
|
290
286
|
def _is_result_record(data: Any) -> TypeIs[ResultRecord[Any]]:
|
291
287
|
return isinstance(data, ResultRecord)
|
292
288
|
|
@@ -335,11 +331,7 @@ async def propose_state(
|
|
335
331
|
# Handle task and sub-flow tracing
|
336
332
|
if state.is_final():
|
337
333
|
result: Any
|
338
|
-
if
|
339
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
340
|
-
# the purpose of disabling `cache_result_in_memory`
|
341
|
-
result = state.result(raise_on_failure=False, fetch=True)
|
342
|
-
elif _is_result_record(state.data):
|
334
|
+
if _is_result_record(state.data):
|
343
335
|
result = state.data.result
|
344
336
|
else:
|
345
337
|
result = state.data
|
@@ -451,13 +443,7 @@ def propose_state_sync(
|
|
451
443
|
|
452
444
|
# Handle task and sub-flow tracing
|
453
445
|
if state.is_final():
|
454
|
-
if
|
455
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
456
|
-
# the purpose of disabling `cache_result_in_memory`
|
457
|
-
result = state.result(raise_on_failure=False, fetch=True)
|
458
|
-
if asyncio.iscoroutine(result):
|
459
|
-
result = run_coro_as_sync(result)
|
460
|
-
elif _is_result_record(state.data):
|
446
|
+
if _is_result_record(state.data):
|
461
447
|
result = state.data.result
|
462
448
|
else:
|
463
449
|
result = state.data
|
@@ -636,8 +622,6 @@ def emit_task_run_state_change_event(
|
|
636
622
|
|
637
623
|
if _is_result_record(validated_state.data) and should_persist_result():
|
638
624
|
data = validated_state.data.metadata.model_dump(mode="json")
|
639
|
-
elif _is_base_result(validated_state.data):
|
640
|
-
data = validated_state.data.model_dump(mode="json")
|
641
625
|
else:
|
642
626
|
data = None
|
643
627
|
|
@@ -0,0 +1,18 @@
|
|
1
|
+
from typing import Any, TypeVar
|
2
|
+
|
3
|
+
from pydantic import BaseModel
|
4
|
+
from pydantic_core import SchemaValidator, core_schema
|
5
|
+
|
6
|
+
T = TypeVar("T", bound=BaseModel)
|
7
|
+
|
8
|
+
ListValidator = SchemaValidator(
|
9
|
+
schema=core_schema.list_schema(
|
10
|
+
items_schema=core_schema.dict_schema(
|
11
|
+
keys_schema=core_schema.str_schema(), values_schema=core_schema.any_schema()
|
12
|
+
)
|
13
|
+
)
|
14
|
+
)
|
15
|
+
|
16
|
+
|
17
|
+
def validate_list(model: type[T], input: Any) -> list[T]:
|
18
|
+
return [model.model_validate(item) for item in ListValidator.validate_python(input)]
|
prefect/workers/__init__.py
CHANGED