prefect-client 2.14.21__py3-none-any.whl → 2.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/concurrency/api.py +37 -2
- prefect/_internal/concurrency/calls.py +9 -0
- prefect/_internal/concurrency/cancellation.py +3 -1
- prefect/_internal/concurrency/event_loop.py +2 -2
- prefect/_internal/concurrency/threads.py +3 -2
- prefect/_internal/pydantic/annotations/pendulum.py +4 -4
- prefect/_internal/pydantic/v2_schema.py +2 -2
- prefect/_vendor/fastapi/__init__.py +1 -1
- prefect/_vendor/fastapi/applications.py +13 -13
- prefect/_vendor/fastapi/background.py +3 -1
- prefect/_vendor/fastapi/concurrency.py +7 -3
- prefect/_vendor/fastapi/datastructures.py +9 -7
- prefect/_vendor/fastapi/dependencies/utils.py +12 -7
- prefect/_vendor/fastapi/encoders.py +1 -1
- prefect/_vendor/fastapi/exception_handlers.py +7 -4
- prefect/_vendor/fastapi/exceptions.py +4 -2
- prefect/_vendor/fastapi/middleware/__init__.py +1 -1
- prefect/_vendor/fastapi/middleware/asyncexitstack.py +1 -1
- prefect/_vendor/fastapi/middleware/cors.py +3 -1
- prefect/_vendor/fastapi/middleware/gzip.py +3 -1
- prefect/_vendor/fastapi/middleware/httpsredirect.py +1 -1
- prefect/_vendor/fastapi/middleware/trustedhost.py +1 -1
- prefect/_vendor/fastapi/middleware/wsgi.py +3 -1
- prefect/_vendor/fastapi/openapi/docs.py +1 -1
- prefect/_vendor/fastapi/openapi/utils.py +3 -3
- prefect/_vendor/fastapi/requests.py +4 -2
- prefect/_vendor/fastapi/responses.py +13 -7
- prefect/_vendor/fastapi/routing.py +15 -15
- prefect/_vendor/fastapi/security/api_key.py +3 -3
- prefect/_vendor/fastapi/security/http.py +2 -2
- prefect/_vendor/fastapi/security/oauth2.py +2 -2
- prefect/_vendor/fastapi/security/open_id_connect_url.py +3 -3
- prefect/_vendor/fastapi/staticfiles.py +1 -1
- prefect/_vendor/fastapi/templating.py +3 -1
- prefect/_vendor/fastapi/testclient.py +1 -1
- prefect/_vendor/fastapi/utils.py +3 -3
- prefect/_vendor/fastapi/websockets.py +7 -3
- prefect/_vendor/starlette/__init__.py +1 -0
- prefect/_vendor/starlette/_compat.py +28 -0
- prefect/_vendor/starlette/_exception_handler.py +80 -0
- prefect/_vendor/starlette/_utils.py +88 -0
- prefect/_vendor/starlette/applications.py +261 -0
- prefect/_vendor/starlette/authentication.py +159 -0
- prefect/_vendor/starlette/background.py +43 -0
- prefect/_vendor/starlette/concurrency.py +59 -0
- prefect/_vendor/starlette/config.py +151 -0
- prefect/_vendor/starlette/convertors.py +87 -0
- prefect/_vendor/starlette/datastructures.py +707 -0
- prefect/_vendor/starlette/endpoints.py +130 -0
- prefect/_vendor/starlette/exceptions.py +60 -0
- prefect/_vendor/starlette/formparsers.py +276 -0
- prefect/_vendor/starlette/middleware/__init__.py +17 -0
- prefect/_vendor/starlette/middleware/authentication.py +52 -0
- prefect/_vendor/starlette/middleware/base.py +220 -0
- prefect/_vendor/starlette/middleware/cors.py +176 -0
- prefect/_vendor/starlette/middleware/errors.py +265 -0
- prefect/_vendor/starlette/middleware/exceptions.py +74 -0
- prefect/_vendor/starlette/middleware/gzip.py +113 -0
- prefect/_vendor/starlette/middleware/httpsredirect.py +19 -0
- prefect/_vendor/starlette/middleware/sessions.py +82 -0
- prefect/_vendor/starlette/middleware/trustedhost.py +64 -0
- prefect/_vendor/starlette/middleware/wsgi.py +147 -0
- prefect/_vendor/starlette/requests.py +328 -0
- prefect/_vendor/starlette/responses.py +347 -0
- prefect/_vendor/starlette/routing.py +933 -0
- prefect/_vendor/starlette/schemas.py +154 -0
- prefect/_vendor/starlette/staticfiles.py +248 -0
- prefect/_vendor/starlette/status.py +199 -0
- prefect/_vendor/starlette/templating.py +231 -0
- prefect/_vendor/starlette/testclient.py +805 -0
- prefect/_vendor/starlette/types.py +30 -0
- prefect/_vendor/starlette/websockets.py +193 -0
- prefect/blocks/core.py +3 -3
- prefect/blocks/notifications.py +8 -8
- prefect/client/base.py +1 -1
- prefect/client/cloud.py +1 -1
- prefect/client/orchestration.py +1 -1
- prefect/client/subscriptions.py +2 -6
- prefect/concurrency/services.py +1 -1
- prefect/context.py +3 -3
- prefect/deployments/deployments.py +3 -3
- prefect/engine.py +69 -9
- prefect/events/clients.py +1 -1
- prefect/filesystems.py +9 -9
- prefect/flow_runs.py +5 -1
- prefect/futures.py +1 -1
- prefect/infrastructure/container.py +3 -3
- prefect/infrastructure/kubernetes.py +4 -6
- prefect/infrastructure/process.py +3 -3
- prefect/input/run_input.py +1 -1
- prefect/logging/formatters.py +1 -1
- prefect/runner/server.py +3 -3
- prefect/settings.py +3 -4
- prefect/software/pip.py +1 -1
- prefect/task_engine.py +4 -0
- prefect/task_server.py +35 -17
- prefect/utilities/asyncutils.py +1 -1
- prefect/utilities/collections.py +1 -1
- {prefect_client-2.14.21.dist-info → prefect_client-2.15.0.dist-info}/METADATA +4 -2
- {prefect_client-2.14.21.dist-info → prefect_client-2.15.0.dist-info}/RECORD +103 -68
- {prefect_client-2.14.21.dist-info → prefect_client-2.15.0.dist-info}/LICENSE +0 -0
- {prefect_client-2.14.21.dist-info → prefect_client-2.15.0.dist-info}/WHEEL +0 -0
- {prefect_client-2.14.21.dist-info → prefect_client-2.15.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,30 @@
|
|
1
|
+
import typing
|
2
|
+
|
3
|
+
if typing.TYPE_CHECKING:
|
4
|
+
from prefect._vendor.starlette.requests import Request
|
5
|
+
from prefect._vendor.starlette.responses import Response
|
6
|
+
from prefect._vendor.starlette.websockets import WebSocket
|
7
|
+
|
8
|
+
AppType = typing.TypeVar("AppType")
|
9
|
+
|
10
|
+
Scope = typing.MutableMapping[str, typing.Any]
|
11
|
+
Message = typing.MutableMapping[str, typing.Any]
|
12
|
+
|
13
|
+
Receive = typing.Callable[[], typing.Awaitable[Message]]
|
14
|
+
Send = typing.Callable[[Message], typing.Awaitable[None]]
|
15
|
+
|
16
|
+
ASGIApp = typing.Callable[[Scope, Receive, Send], typing.Awaitable[None]]
|
17
|
+
|
18
|
+
StatelessLifespan = typing.Callable[[AppType], typing.AsyncContextManager[None]]
|
19
|
+
StatefulLifespan = typing.Callable[
|
20
|
+
[AppType], typing.AsyncContextManager[typing.Mapping[str, typing.Any]]
|
21
|
+
]
|
22
|
+
Lifespan = typing.Union[StatelessLifespan[AppType], StatefulLifespan[AppType]]
|
23
|
+
|
24
|
+
HTTPExceptionHandler = typing.Callable[
|
25
|
+
["Request", Exception], typing.Union["Response", typing.Awaitable["Response"]]
|
26
|
+
]
|
27
|
+
WebSocketExceptionHandler = typing.Callable[
|
28
|
+
["WebSocket", Exception], typing.Awaitable[None]
|
29
|
+
]
|
30
|
+
ExceptionHandler = typing.Union[HTTPExceptionHandler, WebSocketExceptionHandler]
|
@@ -0,0 +1,193 @@
|
|
1
|
+
import enum
|
2
|
+
import json
|
3
|
+
import typing
|
4
|
+
|
5
|
+
from prefect._vendor.starlette.requests import HTTPConnection
|
6
|
+
from prefect._vendor.starlette.types import Message, Receive, Scope, Send
|
7
|
+
|
8
|
+
|
9
|
+
class WebSocketState(enum.Enum):
|
10
|
+
CONNECTING = 0
|
11
|
+
CONNECTED = 1
|
12
|
+
DISCONNECTED = 2
|
13
|
+
|
14
|
+
|
15
|
+
class WebSocketDisconnect(Exception):
|
16
|
+
def __init__(self, code: int = 1000, reason: typing.Optional[str] = None) -> None:
|
17
|
+
self.code = code
|
18
|
+
self.reason = reason or ""
|
19
|
+
|
20
|
+
|
21
|
+
class WebSocket(HTTPConnection):
|
22
|
+
def __init__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
23
|
+
super().__init__(scope)
|
24
|
+
assert scope["type"] == "websocket"
|
25
|
+
self._receive = receive
|
26
|
+
self._send = send
|
27
|
+
self.client_state = WebSocketState.CONNECTING
|
28
|
+
self.application_state = WebSocketState.CONNECTING
|
29
|
+
|
30
|
+
async def receive(self) -> Message:
|
31
|
+
"""
|
32
|
+
Receive ASGI websocket messages, ensuring valid state transitions.
|
33
|
+
"""
|
34
|
+
if self.client_state == WebSocketState.CONNECTING:
|
35
|
+
message = await self._receive()
|
36
|
+
message_type = message["type"]
|
37
|
+
if message_type != "websocket.connect":
|
38
|
+
raise RuntimeError(
|
39
|
+
'Expected ASGI message "websocket.connect", '
|
40
|
+
f"but got {message_type!r}"
|
41
|
+
)
|
42
|
+
self.client_state = WebSocketState.CONNECTED
|
43
|
+
return message
|
44
|
+
elif self.client_state == WebSocketState.CONNECTED:
|
45
|
+
message = await self._receive()
|
46
|
+
message_type = message["type"]
|
47
|
+
if message_type not in {"websocket.receive", "websocket.disconnect"}:
|
48
|
+
raise RuntimeError(
|
49
|
+
'Expected ASGI message "websocket.receive" or '
|
50
|
+
f'"websocket.disconnect", but got {message_type!r}'
|
51
|
+
)
|
52
|
+
if message_type == "websocket.disconnect":
|
53
|
+
self.client_state = WebSocketState.DISCONNECTED
|
54
|
+
return message
|
55
|
+
else:
|
56
|
+
raise RuntimeError(
|
57
|
+
'Cannot call "receive" once a disconnect message has been received.'
|
58
|
+
)
|
59
|
+
|
60
|
+
async def send(self, message: Message) -> None:
|
61
|
+
"""
|
62
|
+
Send ASGI websocket messages, ensuring valid state transitions.
|
63
|
+
"""
|
64
|
+
if self.application_state == WebSocketState.CONNECTING:
|
65
|
+
message_type = message["type"]
|
66
|
+
if message_type not in {"websocket.accept", "websocket.close"}:
|
67
|
+
raise RuntimeError(
|
68
|
+
'Expected ASGI message "websocket.accept" or '
|
69
|
+
f'"websocket.close", but got {message_type!r}'
|
70
|
+
)
|
71
|
+
if message_type == "websocket.close":
|
72
|
+
self.application_state = WebSocketState.DISCONNECTED
|
73
|
+
else:
|
74
|
+
self.application_state = WebSocketState.CONNECTED
|
75
|
+
await self._send(message)
|
76
|
+
elif self.application_state == WebSocketState.CONNECTED:
|
77
|
+
message_type = message["type"]
|
78
|
+
if message_type not in {"websocket.send", "websocket.close"}:
|
79
|
+
raise RuntimeError(
|
80
|
+
'Expected ASGI message "websocket.send" or "websocket.close", '
|
81
|
+
f"but got {message_type!r}"
|
82
|
+
)
|
83
|
+
if message_type == "websocket.close":
|
84
|
+
self.application_state = WebSocketState.DISCONNECTED
|
85
|
+
await self._send(message)
|
86
|
+
else:
|
87
|
+
raise RuntimeError('Cannot call "send" once a close message has been sent.')
|
88
|
+
|
89
|
+
async def accept(
|
90
|
+
self,
|
91
|
+
subprotocol: typing.Optional[str] = None,
|
92
|
+
headers: typing.Optional[typing.Iterable[typing.Tuple[bytes, bytes]]] = None,
|
93
|
+
) -> None:
|
94
|
+
headers = headers or []
|
95
|
+
|
96
|
+
if self.client_state == WebSocketState.CONNECTING:
|
97
|
+
# If we haven't yet seen the 'connect' message, then wait for it first.
|
98
|
+
await self.receive()
|
99
|
+
await self.send(
|
100
|
+
{"type": "websocket.accept", "subprotocol": subprotocol, "headers": headers}
|
101
|
+
)
|
102
|
+
|
103
|
+
def _raise_on_disconnect(self, message: Message) -> None:
|
104
|
+
if message["type"] == "websocket.disconnect":
|
105
|
+
raise WebSocketDisconnect(message["code"], message.get("reason"))
|
106
|
+
|
107
|
+
async def receive_text(self) -> str:
|
108
|
+
if self.application_state != WebSocketState.CONNECTED:
|
109
|
+
raise RuntimeError(
|
110
|
+
'WebSocket is not connected. Need to call "accept" first.'
|
111
|
+
)
|
112
|
+
message = await self.receive()
|
113
|
+
self._raise_on_disconnect(message)
|
114
|
+
return typing.cast(str, message["text"])
|
115
|
+
|
116
|
+
async def receive_bytes(self) -> bytes:
|
117
|
+
if self.application_state != WebSocketState.CONNECTED:
|
118
|
+
raise RuntimeError(
|
119
|
+
'WebSocket is not connected. Need to call "accept" first.'
|
120
|
+
)
|
121
|
+
message = await self.receive()
|
122
|
+
self._raise_on_disconnect(message)
|
123
|
+
return typing.cast(bytes, message["bytes"])
|
124
|
+
|
125
|
+
async def receive_json(self, mode: str = "text") -> typing.Any:
|
126
|
+
if mode not in {"text", "binary"}:
|
127
|
+
raise RuntimeError('The "mode" argument should be "text" or "binary".')
|
128
|
+
if self.application_state != WebSocketState.CONNECTED:
|
129
|
+
raise RuntimeError(
|
130
|
+
'WebSocket is not connected. Need to call "accept" first.'
|
131
|
+
)
|
132
|
+
message = await self.receive()
|
133
|
+
self._raise_on_disconnect(message)
|
134
|
+
|
135
|
+
if mode == "text":
|
136
|
+
text = message["text"]
|
137
|
+
else:
|
138
|
+
text = message["bytes"].decode("utf-8")
|
139
|
+
return json.loads(text)
|
140
|
+
|
141
|
+
async def iter_text(self) -> typing.AsyncIterator[str]:
|
142
|
+
try:
|
143
|
+
while True:
|
144
|
+
yield await self.receive_text()
|
145
|
+
except WebSocketDisconnect:
|
146
|
+
pass
|
147
|
+
|
148
|
+
async def iter_bytes(self) -> typing.AsyncIterator[bytes]:
|
149
|
+
try:
|
150
|
+
while True:
|
151
|
+
yield await self.receive_bytes()
|
152
|
+
except WebSocketDisconnect:
|
153
|
+
pass
|
154
|
+
|
155
|
+
async def iter_json(self) -> typing.AsyncIterator[typing.Any]:
|
156
|
+
try:
|
157
|
+
while True:
|
158
|
+
yield await self.receive_json()
|
159
|
+
except WebSocketDisconnect:
|
160
|
+
pass
|
161
|
+
|
162
|
+
async def send_text(self, data: str) -> None:
|
163
|
+
await self.send({"type": "websocket.send", "text": data})
|
164
|
+
|
165
|
+
async def send_bytes(self, data: bytes) -> None:
|
166
|
+
await self.send({"type": "websocket.send", "bytes": data})
|
167
|
+
|
168
|
+
async def send_json(self, data: typing.Any, mode: str = "text") -> None:
|
169
|
+
if mode not in {"text", "binary"}:
|
170
|
+
raise RuntimeError('The "mode" argument should be "text" or "binary".')
|
171
|
+
text = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
172
|
+
if mode == "text":
|
173
|
+
await self.send({"type": "websocket.send", "text": text})
|
174
|
+
else:
|
175
|
+
await self.send({"type": "websocket.send", "bytes": text.encode("utf-8")})
|
176
|
+
|
177
|
+
async def close(
|
178
|
+
self, code: int = 1000, reason: typing.Optional[str] = None
|
179
|
+
) -> None:
|
180
|
+
await self.send(
|
181
|
+
{"type": "websocket.close", "code": code, "reason": reason or ""}
|
182
|
+
)
|
183
|
+
|
184
|
+
|
185
|
+
class WebSocketClose:
|
186
|
+
def __init__(self, code: int = 1000, reason: typing.Optional[str] = None) -> None:
|
187
|
+
self.code = code
|
188
|
+
self.reason = reason or ""
|
189
|
+
|
190
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
191
|
+
await send(
|
192
|
+
{"type": "websocket.close", "code": self.code, "reason": self.reason}
|
193
|
+
)
|
prefect/blocks/core.py
CHANGED
@@ -257,9 +257,9 @@ class Block(BaseModel, ABC):
|
|
257
257
|
type_._to_block_schema_reference_dict(),
|
258
258
|
]
|
259
259
|
else:
|
260
|
-
refs[
|
261
|
-
|
262
|
-
)
|
260
|
+
refs[
|
261
|
+
field.name
|
262
|
+
] = type_._to_block_schema_reference_dict()
|
263
263
|
|
264
264
|
def __init__(self, *args, **kwargs):
|
265
265
|
super().__init__(*args, **kwargs)
|
prefect/blocks/notifications.py
CHANGED
@@ -24,14 +24,14 @@ class AbstractAppriseNotificationBlock(NotificationBlock, ABC):
|
|
24
24
|
An abstract class for sending notifications using Apprise.
|
25
25
|
"""
|
26
26
|
|
27
|
-
notify_type: Literal[
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
)
|
27
|
+
notify_type: Literal[
|
28
|
+
"prefect_default", "info", "success", "warning", "failure"
|
29
|
+
] = Field(
|
30
|
+
default=PREFECT_NOTIFY_TYPE_DEFAULT,
|
31
|
+
description=(
|
32
|
+
"The type of notification being performed; the prefect_default "
|
33
|
+
"is a plain notification that does not attach an image."
|
34
|
+
),
|
35
35
|
)
|
36
36
|
|
37
37
|
def __init__(self, *args, **kwargs):
|
prefect/client/base.py
CHANGED
@@ -22,7 +22,7 @@ import anyio
|
|
22
22
|
import httpx
|
23
23
|
from asgi_lifespan import LifespanManager
|
24
24
|
from httpx import HTTPStatusError, Response
|
25
|
-
from starlette import status
|
25
|
+
from prefect._vendor.starlette import status
|
26
26
|
from typing_extensions import Self
|
27
27
|
|
28
28
|
from prefect.exceptions import PrefectHTTPStatusError
|
prefect/client/cloud.py
CHANGED
prefect/client/orchestration.py
CHANGED
prefect/client/subscriptions.py
CHANGED
@@ -4,7 +4,7 @@ from typing import Generic, List, Type, TypeVar
|
|
4
4
|
import orjson
|
5
5
|
import websockets
|
6
6
|
import websockets.exceptions
|
7
|
-
from starlette.status import WS_1008_POLICY_VIOLATION
|
7
|
+
from prefect._vendor.starlette.status import WS_1008_POLICY_VIOLATION
|
8
8
|
from typing_extensions import Self
|
9
9
|
|
10
10
|
from prefect._internal.schemas.bases import IDBaseModel
|
@@ -40,11 +40,7 @@ class Subscription(Generic[S]):
|
|
40
40
|
await self._ensure_connected()
|
41
41
|
message = await self._websocket.recv()
|
42
42
|
|
43
|
-
|
44
|
-
|
45
|
-
if message_data.get("type") == "ping":
|
46
|
-
await self._websocket.send(orjson.dumps({"type": "pong"}).decode())
|
47
|
-
continue
|
43
|
+
await self._websocket.send(orjson.dumps({"type": "ack"}).decode())
|
48
44
|
|
49
45
|
return self.model.parse_raw(message)
|
50
46
|
except (
|
prefect/concurrency/services.py
CHANGED
prefect/context.py
CHANGED
@@ -137,9 +137,9 @@ class PrefectObjectRegistry(ContextModel):
|
|
137
137
|
)
|
138
138
|
|
139
139
|
# Failures will be a tuple of (exception, instance, args, kwargs)
|
140
|
-
_instance_init_failures: Dict[
|
141
|
-
|
142
|
-
)
|
140
|
+
_instance_init_failures: Dict[
|
141
|
+
Type[T], List[Tuple[Exception, T, Tuple, Dict]]
|
142
|
+
] = PrivateAttr(default_factory=lambda: defaultdict(list))
|
143
143
|
|
144
144
|
block_code_execution: bool = False
|
145
145
|
capture_failures: bool = False
|
@@ -93,13 +93,13 @@ async def run_deployment(
|
|
93
93
|
run metadata immediately. Setting `timeout` to None will allow this
|
94
94
|
function to poll indefinitely. Defaults to None.
|
95
95
|
poll_interval: The number of seconds between polls
|
96
|
-
tags: A list of tags to associate with this flow run;
|
97
|
-
|
96
|
+
tags: A list of tags to associate with this flow run; tags can be used in
|
97
|
+
automations and for organizational purposes.
|
98
98
|
idempotency_key: A unique value to recognize retries of the same run, and
|
99
99
|
prevent creating multiple flow runs.
|
100
100
|
work_queue_name: The name of a work queue to use for this run. Defaults to
|
101
101
|
the default work queue for the deployment.
|
102
|
-
as_subflow: Whether
|
102
|
+
as_subflow: Whether to link the flow run as a subflow of the current
|
103
103
|
flow or task run.
|
104
104
|
"""
|
105
105
|
if timeout is not None and timeout < 0:
|
prefect/engine.py
CHANGED
@@ -192,6 +192,8 @@ from prefect.states import (
|
|
192
192
|
from prefect.task_runners import (
|
193
193
|
CONCURRENCY_MESSAGES,
|
194
194
|
BaseTaskRunner,
|
195
|
+
ConcurrentTaskRunner,
|
196
|
+
SequentialTaskRunner,
|
195
197
|
TaskConcurrencyType,
|
196
198
|
)
|
197
199
|
from prefect.tasks import Task
|
@@ -850,7 +852,11 @@ async def orchestrate_flow_run(
|
|
850
852
|
not parent_flow_run_context
|
851
853
|
or (
|
852
854
|
parent_flow_run_context
|
853
|
-
and
|
855
|
+
and
|
856
|
+
# Unless the parent is async and the child is sync, run the
|
857
|
+
# child flow in the parent thread; running a sync child in
|
858
|
+
# an async parent could be bad for async performance.
|
859
|
+
not (parent_flow_run_context.flow.isasync and not flow.isasync)
|
854
860
|
)
|
855
861
|
):
|
856
862
|
from_async.call_soon_in_waiting_thread(
|
@@ -1405,6 +1411,7 @@ def enter_task_run_engine(
|
|
1405
1411
|
wait_for=wait_for,
|
1406
1412
|
return_type=return_type,
|
1407
1413
|
task_runner=task_runner,
|
1414
|
+
user_thread=threading.current_thread(),
|
1408
1415
|
)
|
1409
1416
|
|
1410
1417
|
if task.isasync and flow_run_context.flow.isasync:
|
@@ -1421,6 +1428,7 @@ async def begin_task_map(
|
|
1421
1428
|
wait_for: Optional[Iterable[PrefectFuture]],
|
1422
1429
|
return_type: EngineReturnType,
|
1423
1430
|
task_runner: Optional[BaseTaskRunner],
|
1431
|
+
user_thread: threading.Thread,
|
1424
1432
|
) -> List[Union[PrefectFuture, Awaitable[PrefectFuture]]]:
|
1425
1433
|
"""Async entrypoint for task mapping"""
|
1426
1434
|
# We need to resolve some futures to map over their data, collect the upstream
|
@@ -1498,6 +1506,7 @@ async def begin_task_map(
|
|
1498
1506
|
return_type=return_type,
|
1499
1507
|
task_runner=task_runner,
|
1500
1508
|
extra_task_inputs=task_inputs,
|
1509
|
+
user_thread=user_thread,
|
1501
1510
|
)
|
1502
1511
|
)
|
1503
1512
|
|
@@ -1562,6 +1571,7 @@ async def get_task_call_return_value(
|
|
1562
1571
|
wait_for: Optional[Iterable[PrefectFuture]],
|
1563
1572
|
return_type: EngineReturnType,
|
1564
1573
|
task_runner: Optional[BaseTaskRunner],
|
1574
|
+
user_thread: threading.Thread,
|
1565
1575
|
extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
|
1566
1576
|
):
|
1567
1577
|
extra_task_inputs = extra_task_inputs or {}
|
@@ -1573,6 +1583,7 @@ async def get_task_call_return_value(
|
|
1573
1583
|
wait_for=wait_for,
|
1574
1584
|
task_runner=task_runner,
|
1575
1585
|
extra_task_inputs=extra_task_inputs,
|
1586
|
+
user_thread=user_thread,
|
1576
1587
|
)
|
1577
1588
|
if return_type == "future":
|
1578
1589
|
return future
|
@@ -1591,6 +1602,7 @@ async def create_task_run_future(
|
|
1591
1602
|
wait_for: Optional[Iterable[PrefectFuture]],
|
1592
1603
|
task_runner: Optional[BaseTaskRunner],
|
1593
1604
|
extra_task_inputs: Dict[str, Set[TaskRunInput]],
|
1605
|
+
user_thread: threading.Thread,
|
1594
1606
|
) -> PrefectFuture:
|
1595
1607
|
# Default to the flow run's task runner
|
1596
1608
|
task_runner = task_runner or flow_run_context.task_runner
|
@@ -1628,6 +1640,7 @@ async def create_task_run_future(
|
|
1628
1640
|
wait_for=wait_for,
|
1629
1641
|
task_runner=task_runner,
|
1630
1642
|
extra_task_inputs=extra_task_inputs,
|
1643
|
+
user_thread=user_thread,
|
1631
1644
|
)
|
1632
1645
|
)
|
1633
1646
|
|
@@ -1651,6 +1664,7 @@ async def create_task_run_then_submit(
|
|
1651
1664
|
wait_for: Optional[Iterable[PrefectFuture]],
|
1652
1665
|
task_runner: BaseTaskRunner,
|
1653
1666
|
extra_task_inputs: Dict[str, Set[TaskRunInput]],
|
1667
|
+
user_thread: threading.Thread,
|
1654
1668
|
) -> None:
|
1655
1669
|
task_run = (
|
1656
1670
|
await create_task_run(
|
@@ -1677,6 +1691,7 @@ async def create_task_run_then_submit(
|
|
1677
1691
|
task_run=task_run,
|
1678
1692
|
wait_for=wait_for,
|
1679
1693
|
task_runner=task_runner,
|
1694
|
+
user_thread=user_thread,
|
1680
1695
|
)
|
1681
1696
|
|
1682
1697
|
future._submitted.set()
|
@@ -1724,6 +1739,7 @@ async def submit_task_run(
|
|
1724
1739
|
task_run: TaskRun,
|
1725
1740
|
wait_for: Optional[Iterable[PrefectFuture]],
|
1726
1741
|
task_runner: BaseTaskRunner,
|
1742
|
+
user_thread: threading.Thread,
|
1727
1743
|
) -> PrefectFuture:
|
1728
1744
|
logger = get_run_logger(flow_run_context)
|
1729
1745
|
|
@@ -1733,6 +1749,10 @@ async def submit_task_run(
|
|
1733
1749
|
):
|
1734
1750
|
logger.info(f"Executing {task_run.name!r} immediately...")
|
1735
1751
|
|
1752
|
+
if not isinstance(task_runner, (ConcurrentTaskRunner, SequentialTaskRunner)):
|
1753
|
+
# Only pass the user thread to "local" task runners
|
1754
|
+
user_thread = None
|
1755
|
+
|
1736
1756
|
future = await task_runner.submit(
|
1737
1757
|
key=future.key,
|
1738
1758
|
call=partial(
|
@@ -1746,6 +1766,8 @@ async def submit_task_run(
|
|
1746
1766
|
),
|
1747
1767
|
log_prints=should_log_prints(task),
|
1748
1768
|
settings=prefect.context.SettingsContext.get().copy(),
|
1769
|
+
user_thread=user_thread,
|
1770
|
+
concurrency_type=task_runner.concurrency_type,
|
1749
1771
|
),
|
1750
1772
|
)
|
1751
1773
|
|
@@ -1766,6 +1788,8 @@ async def begin_task_run(
|
|
1766
1788
|
result_factory: ResultFactory,
|
1767
1789
|
log_prints: bool,
|
1768
1790
|
settings: prefect.context.SettingsContext,
|
1791
|
+
user_thread: Optional[threading.Thread],
|
1792
|
+
concurrency_type: TaskConcurrencyType,
|
1769
1793
|
):
|
1770
1794
|
"""
|
1771
1795
|
Entrypoint for task run execution.
|
@@ -1836,6 +1860,8 @@ async def begin_task_run(
|
|
1836
1860
|
log_prints=log_prints,
|
1837
1861
|
interruptible=interruptible,
|
1838
1862
|
client=client,
|
1863
|
+
user_thread=user_thread,
|
1864
|
+
concurrency_type=concurrency_type,
|
1839
1865
|
)
|
1840
1866
|
|
1841
1867
|
if not maybe_flow_run_context:
|
@@ -1886,6 +1912,8 @@ async def orchestrate_task_run(
|
|
1886
1912
|
log_prints: bool,
|
1887
1913
|
interruptible: bool,
|
1888
1914
|
client: PrefectClient,
|
1915
|
+
concurrency_type: TaskConcurrencyType,
|
1916
|
+
user_thread: Optional[threading.Thread],
|
1889
1917
|
) -> State:
|
1890
1918
|
"""
|
1891
1919
|
Execute a task run
|
@@ -1917,6 +1945,7 @@ async def orchestrate_task_run(
|
|
1917
1945
|
flow_run = flow_run_context.flow_run
|
1918
1946
|
else:
|
1919
1947
|
flow_run = await client.read_flow_run(task_run.flow_run_id)
|
1948
|
+
|
1920
1949
|
logger = task_run_logger(task_run, task=task, flow_run=flow_run)
|
1921
1950
|
|
1922
1951
|
partial_task_run_context = PartialModel(
|
@@ -2115,9 +2144,41 @@ async def orchestrate_task_run(
|
|
2115
2144
|
"Beginning execution...", extra={"state_message": True}
|
2116
2145
|
)
|
2117
2146
|
|
2118
|
-
call =
|
2119
|
-
|
2120
|
-
|
2147
|
+
call = create_call(task.fn, *args, **kwargs)
|
2148
|
+
|
2149
|
+
if (
|
2150
|
+
flow_run_context
|
2151
|
+
and user_thread
|
2152
|
+
and (
|
2153
|
+
# Async and sync tasks can be executed on synchronous flows
|
2154
|
+
# if the task runner is sequential; if the task is sync and a
|
2155
|
+
# concurrent task runner is used, we must execute it in a worker
|
2156
|
+
# thread instead.
|
2157
|
+
(
|
2158
|
+
concurrency_type == TaskConcurrencyType.SEQUENTIAL
|
2159
|
+
and (
|
2160
|
+
flow_run_context.flow
|
2161
|
+
and not flow_run_context.flow.isasync
|
2162
|
+
)
|
2163
|
+
)
|
2164
|
+
# Async tasks can always be executed on asynchronous flow; if the
|
2165
|
+
# flow is async we do not want to block the event loop with
|
2166
|
+
# synchronous tasks
|
2167
|
+
or (
|
2168
|
+
flow_run_context.flow
|
2169
|
+
and flow_run_context.flow.isasync
|
2170
|
+
and task.isasync
|
2171
|
+
)
|
2172
|
+
)
|
2173
|
+
):
|
2174
|
+
from_async.call_soon_in_waiting_thread(
|
2175
|
+
call, thread=user_thread, timeout=task.timeout_seconds
|
2176
|
+
)
|
2177
|
+
else:
|
2178
|
+
from_async.call_soon_in_new_thread(
|
2179
|
+
call, timeout=task.timeout_seconds
|
2180
|
+
)
|
2181
|
+
|
2121
2182
|
result = await call.aresult()
|
2122
2183
|
|
2123
2184
|
except (CancelledError, asyncio.CancelledError) as exc:
|
@@ -2446,8 +2507,7 @@ async def resolve_inputs(
|
|
2446
2507
|
# incorrectly evaluate to false — to resolve this, we must track all
|
2447
2508
|
# annotations wrapping the current expression but this is not yet
|
2448
2509
|
# implemented.
|
2449
|
-
isinstance(context.get("annotation"), allow_failure)
|
2450
|
-
and state.is_failed()
|
2510
|
+
isinstance(context.get("annotation"), allow_failure) and state.is_failed()
|
2451
2511
|
):
|
2452
2512
|
raise UpstreamTaskError(
|
2453
2513
|
f"Upstream task run '{state.state_details.task_run_id}' did not reach a"
|
@@ -2931,10 +2991,10 @@ async def _create_autonomous_task_run(
|
|
2931
2991
|
task: Task, parameters: Dict[str, Any]
|
2932
2992
|
) -> TaskRun:
|
2933
2993
|
async with get_client() as client:
|
2934
|
-
|
2994
|
+
state = Scheduled()
|
2935
2995
|
if parameters:
|
2936
2996
|
parameters_id = uuid4()
|
2937
|
-
|
2997
|
+
state.state_details.task_parameters_id = parameters_id
|
2938
2998
|
|
2939
2999
|
# TODO: We want to use result storage for parameters, but we'll need
|
2940
3000
|
# a better way to use it than this.
|
@@ -2946,7 +3006,7 @@ async def _create_autonomous_task_run(
|
|
2946
3006
|
task=task,
|
2947
3007
|
flow_run_id=None,
|
2948
3008
|
dynamic_key=f"{task.task_key}-{str(uuid4())[:NUM_CHARS_DYNAMIC_KEY]}",
|
2949
|
-
state=
|
3009
|
+
state=state,
|
2950
3010
|
)
|
2951
3011
|
|
2952
3012
|
engine_logger.debug(f"Submitted run of task {task.name!r} for execution")
|
prefect/events/clients.py
CHANGED
@@ -21,7 +21,7 @@ try:
|
|
21
21
|
from cachetools import TTLCache
|
22
22
|
except ImportError:
|
23
23
|
pass
|
24
|
-
from starlette.status import WS_1008_POLICY_VIOLATION
|
24
|
+
from prefect._vendor.starlette.status import WS_1008_POLICY_VIOLATION
|
25
25
|
from websockets.client import WebSocketClientProtocol, connect
|
26
26
|
from websockets.exceptions import (
|
27
27
|
ConnectionClosed,
|
prefect/filesystems.py
CHANGED
@@ -709,13 +709,13 @@ class Azure(WritableFileSystem, WritableDeploymentStorage):
|
|
709
709
|
def filesystem(self) -> RemoteFileSystem:
|
710
710
|
settings = {}
|
711
711
|
if self.azure_storage_connection_string:
|
712
|
-
settings[
|
713
|
-
|
714
|
-
)
|
712
|
+
settings[
|
713
|
+
"connection_string"
|
714
|
+
] = self.azure_storage_connection_string.get_secret_value()
|
715
715
|
if self.azure_storage_account_name:
|
716
|
-
settings[
|
717
|
-
|
718
|
-
)
|
716
|
+
settings[
|
717
|
+
"account_name"
|
718
|
+
] = self.azure_storage_account_name.get_secret_value()
|
719
719
|
if self.azure_storage_account_key:
|
720
720
|
settings["account_key"] = self.azure_storage_account_key.get_secret_value()
|
721
721
|
if self.azure_storage_tenant_id:
|
@@ -723,9 +723,9 @@ class Azure(WritableFileSystem, WritableDeploymentStorage):
|
|
723
723
|
if self.azure_storage_client_id:
|
724
724
|
settings["client_id"] = self.azure_storage_client_id.get_secret_value()
|
725
725
|
if self.azure_storage_client_secret:
|
726
|
-
settings[
|
727
|
-
|
728
|
-
)
|
726
|
+
settings[
|
727
|
+
"client_secret"
|
728
|
+
] = self.azure_storage_client_secret.get_secret_value()
|
729
729
|
settings["anon"] = self.azure_storage_anon
|
730
730
|
self._remote_file_system = RemoteFileSystem(
|
731
731
|
basepath=self.basepath, settings=settings
|
prefect/flow_runs.py
CHANGED
@@ -7,6 +7,7 @@ from prefect.client.orchestration import PrefectClient
|
|
7
7
|
from prefect.client.schemas import FlowRun
|
8
8
|
from prefect.client.utilities import inject_client
|
9
9
|
from prefect.exceptions import FlowRunWaitTimeout
|
10
|
+
from prefect.logging import get_logger
|
10
11
|
|
11
12
|
|
12
13
|
@inject_client
|
@@ -15,6 +16,7 @@ async def wait_for_flow_run(
|
|
15
16
|
timeout: Optional[int] = 10800,
|
16
17
|
poll_interval: int = 5,
|
17
18
|
client: Optional[PrefectClient] = None,
|
19
|
+
log_states: bool = False,
|
18
20
|
) -> FlowRun:
|
19
21
|
"""
|
20
22
|
Waits for the prefect flow run to finish and returns the FlowRun
|
@@ -71,14 +73,16 @@ async def wait_for_flow_run(
|
|
71
73
|
```
|
72
74
|
"""
|
73
75
|
assert client is not None, "Client injection failed"
|
76
|
+
logger = get_logger()
|
74
77
|
with anyio.move_on_after(timeout):
|
75
78
|
while True:
|
76
79
|
flow_run = await client.read_flow_run(flow_run_id)
|
77
80
|
flow_state = flow_run.state
|
81
|
+
if log_states:
|
82
|
+
logger.info(f"Flow run is in state {flow_run.state.name!r}")
|
78
83
|
if flow_state and flow_state.is_final():
|
79
84
|
return flow_run
|
80
85
|
await anyio.sleep(poll_interval)
|
81
|
-
|
82
86
|
raise FlowRunWaitTimeout(
|
83
87
|
f"Flow run with ID {flow_run_id} exceeded watch timeout of {timeout} seconds"
|
84
88
|
)
|
prefect/futures.py
CHANGED
@@ -360,7 +360,7 @@ async def resolve_futures_to_data(
|
|
360
360
|
|
361
361
|
|
362
362
|
async def resolve_futures_to_states(
|
363
|
-
expr: Union[PrefectFuture[R, Any], Any]
|
363
|
+
expr: Union[PrefectFuture[R, Any], Any],
|
364
364
|
) -> Union[State[R], Any]:
|
365
365
|
"""
|
366
366
|
Given a Python built-in collection, recursively find `PrefectFutures` and build a
|