prefect-client 2.20.2__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +74 -110
- prefect/_internal/compatibility/deprecated.py +6 -115
- prefect/_internal/compatibility/experimental.py +4 -79
- prefect/_internal/compatibility/migration.py +166 -0
- prefect/_internal/concurrency/__init__.py +2 -2
- prefect/_internal/concurrency/api.py +1 -35
- prefect/_internal/concurrency/calls.py +0 -6
- prefect/_internal/concurrency/cancellation.py +0 -3
- prefect/_internal/concurrency/event_loop.py +0 -20
- prefect/_internal/concurrency/inspection.py +3 -3
- prefect/_internal/concurrency/primitives.py +1 -0
- prefect/_internal/concurrency/services.py +23 -0
- prefect/_internal/concurrency/threads.py +35 -0
- prefect/_internal/concurrency/waiters.py +0 -28
- prefect/_internal/integrations.py +7 -0
- prefect/_internal/pydantic/__init__.py +0 -45
- prefect/_internal/pydantic/annotations/pendulum.py +2 -2
- prefect/_internal/pydantic/v1_schema.py +21 -22
- prefect/_internal/pydantic/v2_schema.py +0 -2
- prefect/_internal/pydantic/v2_validated_func.py +18 -23
- prefect/_internal/pytz.py +1 -1
- prefect/_internal/retries.py +61 -0
- prefect/_internal/schemas/bases.py +45 -177
- prefect/_internal/schemas/fields.py +1 -43
- prefect/_internal/schemas/validators.py +47 -233
- prefect/agent.py +3 -695
- prefect/artifacts.py +173 -14
- prefect/automations.py +39 -4
- prefect/blocks/abstract.py +1 -1
- prefect/blocks/core.py +423 -164
- prefect/blocks/fields.py +2 -57
- prefect/blocks/notifications.py +43 -28
- prefect/blocks/redis.py +168 -0
- prefect/blocks/system.py +67 -20
- prefect/blocks/webhook.py +2 -9
- prefect/cache_policies.py +239 -0
- prefect/client/__init__.py +4 -0
- prefect/client/base.py +33 -27
- prefect/client/cloud.py +65 -20
- prefect/client/collections.py +1 -1
- prefect/client/orchestration.py +667 -440
- prefect/client/schemas/actions.py +115 -100
- prefect/client/schemas/filters.py +46 -52
- prefect/client/schemas/objects.py +228 -178
- prefect/client/schemas/responses.py +18 -36
- prefect/client/schemas/schedules.py +55 -36
- prefect/client/schemas/sorting.py +2 -0
- prefect/client/subscriptions.py +8 -7
- prefect/client/types/flexible_schedule_list.py +11 -0
- prefect/client/utilities.py +9 -6
- prefect/concurrency/asyncio.py +60 -11
- prefect/concurrency/context.py +24 -0
- prefect/concurrency/events.py +2 -2
- prefect/concurrency/services.py +46 -16
- prefect/concurrency/sync.py +51 -7
- prefect/concurrency/v1/asyncio.py +143 -0
- prefect/concurrency/v1/context.py +27 -0
- prefect/concurrency/v1/events.py +61 -0
- prefect/concurrency/v1/services.py +116 -0
- prefect/concurrency/v1/sync.py +92 -0
- prefect/context.py +246 -149
- prefect/deployments/__init__.py +33 -18
- prefect/deployments/base.py +10 -15
- prefect/deployments/deployments.py +2 -1048
- prefect/deployments/flow_runs.py +178 -0
- prefect/deployments/runner.py +72 -173
- prefect/deployments/schedules.py +31 -25
- prefect/deployments/steps/__init__.py +0 -1
- prefect/deployments/steps/core.py +7 -0
- prefect/deployments/steps/pull.py +15 -21
- prefect/deployments/steps/utility.py +2 -1
- prefect/docker/__init__.py +20 -0
- prefect/docker/docker_image.py +82 -0
- prefect/engine.py +15 -2466
- prefect/events/actions.py +17 -23
- prefect/events/cli/automations.py +20 -7
- prefect/events/clients.py +142 -80
- prefect/events/filters.py +14 -18
- prefect/events/related.py +74 -75
- prefect/events/schemas/__init__.py +0 -5
- prefect/events/schemas/automations.py +55 -46
- prefect/events/schemas/deployment_triggers.py +7 -197
- prefect/events/schemas/events.py +46 -65
- prefect/events/schemas/labelling.py +10 -14
- prefect/events/utilities.py +4 -5
- prefect/events/worker.py +23 -8
- prefect/exceptions.py +15 -0
- prefect/filesystems.py +30 -529
- prefect/flow_engine.py +827 -0
- prefect/flow_runs.py +379 -7
- prefect/flows.py +470 -360
- prefect/futures.py +382 -331
- prefect/infrastructure/__init__.py +5 -26
- prefect/infrastructure/base.py +3 -320
- prefect/infrastructure/provisioners/__init__.py +5 -3
- prefect/infrastructure/provisioners/cloud_run.py +13 -8
- prefect/infrastructure/provisioners/container_instance.py +14 -9
- prefect/infrastructure/provisioners/ecs.py +10 -8
- prefect/infrastructure/provisioners/modal.py +8 -5
- prefect/input/__init__.py +4 -0
- prefect/input/actions.py +2 -4
- prefect/input/run_input.py +9 -9
- prefect/logging/formatters.py +2 -4
- prefect/logging/handlers.py +9 -14
- prefect/logging/loggers.py +5 -5
- prefect/main.py +72 -0
- prefect/plugins.py +2 -64
- prefect/profiles.toml +16 -2
- prefect/records/__init__.py +1 -0
- prefect/records/base.py +223 -0
- prefect/records/filesystem.py +207 -0
- prefect/records/memory.py +178 -0
- prefect/records/result_store.py +64 -0
- prefect/results.py +577 -504
- prefect/runner/runner.py +124 -51
- prefect/runner/server.py +32 -34
- prefect/runner/storage.py +3 -12
- prefect/runner/submit.py +2 -10
- prefect/runner/utils.py +2 -2
- prefect/runtime/__init__.py +1 -0
- prefect/runtime/deployment.py +1 -0
- prefect/runtime/flow_run.py +40 -5
- prefect/runtime/task_run.py +1 -0
- prefect/serializers.py +28 -39
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
- prefect/settings.py +209 -332
- prefect/states.py +160 -63
- prefect/task_engine.py +1478 -57
- prefect/task_runners.py +383 -287
- prefect/task_runs.py +240 -0
- prefect/task_worker.py +463 -0
- prefect/tasks.py +684 -374
- prefect/transactions.py +410 -0
- prefect/types/__init__.py +72 -86
- prefect/types/entrypoint.py +13 -0
- prefect/utilities/annotations.py +4 -3
- prefect/utilities/asyncutils.py +227 -148
- prefect/utilities/callables.py +138 -48
- prefect/utilities/collections.py +134 -86
- prefect/utilities/dispatch.py +27 -14
- prefect/utilities/dockerutils.py +11 -4
- prefect/utilities/engine.py +186 -32
- prefect/utilities/filesystem.py +4 -5
- prefect/utilities/importtools.py +26 -27
- prefect/utilities/pydantic.py +128 -38
- prefect/utilities/schema_tools/hydration.py +18 -1
- prefect/utilities/schema_tools/validation.py +30 -0
- prefect/utilities/services.py +35 -9
- prefect/utilities/templating.py +12 -2
- prefect/utilities/timeout.py +20 -5
- prefect/utilities/urls.py +195 -0
- prefect/utilities/visualization.py +1 -0
- prefect/variables.py +78 -59
- prefect/workers/__init__.py +0 -1
- prefect/workers/base.py +237 -244
- prefect/workers/block.py +5 -226
- prefect/workers/cloud.py +6 -0
- prefect/workers/process.py +265 -12
- prefect/workers/server.py +29 -11
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/METADATA +30 -26
- prefect_client-3.0.0.dist-info/RECORD +201 -0
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/WHEEL +1 -1
- prefect/_internal/pydantic/_base_model.py +0 -51
- prefect/_internal/pydantic/_compat.py +0 -82
- prefect/_internal/pydantic/_flags.py +0 -20
- prefect/_internal/pydantic/_types.py +0 -8
- prefect/_internal/pydantic/utilities/config_dict.py +0 -72
- prefect/_internal/pydantic/utilities/field_validator.py +0 -150
- prefect/_internal/pydantic/utilities/model_construct.py +0 -56
- prefect/_internal/pydantic/utilities/model_copy.py +0 -55
- prefect/_internal/pydantic/utilities/model_dump.py +0 -136
- prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
- prefect/_internal/pydantic/utilities/model_fields.py +0 -50
- prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
- prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
- prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
- prefect/_internal/pydantic/utilities/model_validate.py +0 -75
- prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
- prefect/_internal/pydantic/utilities/model_validator.py +0 -87
- prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
- prefect/_vendor/fastapi/__init__.py +0 -25
- prefect/_vendor/fastapi/applications.py +0 -946
- prefect/_vendor/fastapi/background.py +0 -3
- prefect/_vendor/fastapi/concurrency.py +0 -44
- prefect/_vendor/fastapi/datastructures.py +0 -58
- prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
- prefect/_vendor/fastapi/dependencies/models.py +0 -64
- prefect/_vendor/fastapi/dependencies/utils.py +0 -877
- prefect/_vendor/fastapi/encoders.py +0 -177
- prefect/_vendor/fastapi/exception_handlers.py +0 -40
- prefect/_vendor/fastapi/exceptions.py +0 -46
- prefect/_vendor/fastapi/logger.py +0 -3
- prefect/_vendor/fastapi/middleware/__init__.py +0 -1
- prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
- prefect/_vendor/fastapi/middleware/cors.py +0 -3
- prefect/_vendor/fastapi/middleware/gzip.py +0 -3
- prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
- prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
- prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
- prefect/_vendor/fastapi/openapi/__init__.py +0 -0
- prefect/_vendor/fastapi/openapi/constants.py +0 -2
- prefect/_vendor/fastapi/openapi/docs.py +0 -203
- prefect/_vendor/fastapi/openapi/models.py +0 -480
- prefect/_vendor/fastapi/openapi/utils.py +0 -485
- prefect/_vendor/fastapi/param_functions.py +0 -340
- prefect/_vendor/fastapi/params.py +0 -453
- prefect/_vendor/fastapi/py.typed +0 -0
- prefect/_vendor/fastapi/requests.py +0 -4
- prefect/_vendor/fastapi/responses.py +0 -40
- prefect/_vendor/fastapi/routing.py +0 -1331
- prefect/_vendor/fastapi/security/__init__.py +0 -15
- prefect/_vendor/fastapi/security/api_key.py +0 -98
- prefect/_vendor/fastapi/security/base.py +0 -6
- prefect/_vendor/fastapi/security/http.py +0 -172
- prefect/_vendor/fastapi/security/oauth2.py +0 -227
- prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
- prefect/_vendor/fastapi/security/utils.py +0 -10
- prefect/_vendor/fastapi/staticfiles.py +0 -1
- prefect/_vendor/fastapi/templating.py +0 -3
- prefect/_vendor/fastapi/testclient.py +0 -1
- prefect/_vendor/fastapi/types.py +0 -3
- prefect/_vendor/fastapi/utils.py +0 -235
- prefect/_vendor/fastapi/websockets.py +0 -7
- prefect/_vendor/starlette/__init__.py +0 -1
- prefect/_vendor/starlette/_compat.py +0 -28
- prefect/_vendor/starlette/_exception_handler.py +0 -80
- prefect/_vendor/starlette/_utils.py +0 -88
- prefect/_vendor/starlette/applications.py +0 -261
- prefect/_vendor/starlette/authentication.py +0 -159
- prefect/_vendor/starlette/background.py +0 -43
- prefect/_vendor/starlette/concurrency.py +0 -59
- prefect/_vendor/starlette/config.py +0 -151
- prefect/_vendor/starlette/convertors.py +0 -87
- prefect/_vendor/starlette/datastructures.py +0 -707
- prefect/_vendor/starlette/endpoints.py +0 -130
- prefect/_vendor/starlette/exceptions.py +0 -60
- prefect/_vendor/starlette/formparsers.py +0 -276
- prefect/_vendor/starlette/middleware/__init__.py +0 -17
- prefect/_vendor/starlette/middleware/authentication.py +0 -52
- prefect/_vendor/starlette/middleware/base.py +0 -220
- prefect/_vendor/starlette/middleware/cors.py +0 -176
- prefect/_vendor/starlette/middleware/errors.py +0 -265
- prefect/_vendor/starlette/middleware/exceptions.py +0 -74
- prefect/_vendor/starlette/middleware/gzip.py +0 -113
- prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
- prefect/_vendor/starlette/middleware/sessions.py +0 -82
- prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
- prefect/_vendor/starlette/middleware/wsgi.py +0 -147
- prefect/_vendor/starlette/py.typed +0 -0
- prefect/_vendor/starlette/requests.py +0 -328
- prefect/_vendor/starlette/responses.py +0 -347
- prefect/_vendor/starlette/routing.py +0 -933
- prefect/_vendor/starlette/schemas.py +0 -154
- prefect/_vendor/starlette/staticfiles.py +0 -248
- prefect/_vendor/starlette/status.py +0 -199
- prefect/_vendor/starlette/templating.py +0 -231
- prefect/_vendor/starlette/testclient.py +0 -804
- prefect/_vendor/starlette/types.py +0 -30
- prefect/_vendor/starlette/websockets.py +0 -193
- prefect/blocks/kubernetes.py +0 -119
- prefect/deprecated/__init__.py +0 -0
- prefect/deprecated/data_documents.py +0 -350
- prefect/deprecated/packaging/__init__.py +0 -12
- prefect/deprecated/packaging/base.py +0 -96
- prefect/deprecated/packaging/docker.py +0 -146
- prefect/deprecated/packaging/file.py +0 -92
- prefect/deprecated/packaging/orion.py +0 -80
- prefect/deprecated/packaging/serializers.py +0 -171
- prefect/events/instrument.py +0 -135
- prefect/infrastructure/container.py +0 -824
- prefect/infrastructure/kubernetes.py +0 -920
- prefect/infrastructure/process.py +0 -289
- prefect/manifests.py +0 -20
- prefect/new_flow_engine.py +0 -449
- prefect/new_task_engine.py +0 -423
- prefect/pydantic/__init__.py +0 -76
- prefect/pydantic/main.py +0 -39
- prefect/software/__init__.py +0 -2
- prefect/software/base.py +0 -50
- prefect/software/conda.py +0 -199
- prefect/software/pip.py +0 -122
- prefect/software/python.py +0 -52
- prefect/task_server.py +0 -322
- prefect_client-2.20.2.dist-info/RECORD +0 -294
- /prefect/{_internal/pydantic/utilities → client/types}/__init__.py +0 -0
- /prefect/{_vendor → concurrency/v1}/__init__.py +0 -0
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/LICENSE +0 -0
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/top_level.txt +0 -0
@@ -1,30 +0,0 @@
|
|
1
|
-
import typing
|
2
|
-
|
3
|
-
if typing.TYPE_CHECKING:
|
4
|
-
from prefect._vendor.starlette.requests import Request
|
5
|
-
from prefect._vendor.starlette.responses import Response
|
6
|
-
from prefect._vendor.starlette.websockets import WebSocket
|
7
|
-
|
8
|
-
AppType = typing.TypeVar("AppType")
|
9
|
-
|
10
|
-
Scope = typing.MutableMapping[str, typing.Any]
|
11
|
-
Message = typing.MutableMapping[str, typing.Any]
|
12
|
-
|
13
|
-
Receive = typing.Callable[[], typing.Awaitable[Message]]
|
14
|
-
Send = typing.Callable[[Message], typing.Awaitable[None]]
|
15
|
-
|
16
|
-
ASGIApp = typing.Callable[[Scope, Receive, Send], typing.Awaitable[None]]
|
17
|
-
|
18
|
-
StatelessLifespan = typing.Callable[[AppType], typing.AsyncContextManager[None]]
|
19
|
-
StatefulLifespan = typing.Callable[
|
20
|
-
[AppType], typing.AsyncContextManager[typing.Mapping[str, typing.Any]]
|
21
|
-
]
|
22
|
-
Lifespan = typing.Union[StatelessLifespan[AppType], StatefulLifespan[AppType]]
|
23
|
-
|
24
|
-
HTTPExceptionHandler = typing.Callable[
|
25
|
-
["Request", Exception], typing.Union["Response", typing.Awaitable["Response"]]
|
26
|
-
]
|
27
|
-
WebSocketExceptionHandler = typing.Callable[
|
28
|
-
["WebSocket", Exception], typing.Awaitable[None]
|
29
|
-
]
|
30
|
-
ExceptionHandler = typing.Union[HTTPExceptionHandler, WebSocketExceptionHandler]
|
@@ -1,193 +0,0 @@
|
|
1
|
-
import enum
|
2
|
-
import json
|
3
|
-
import typing
|
4
|
-
|
5
|
-
from prefect._vendor.starlette.requests import HTTPConnection
|
6
|
-
from prefect._vendor.starlette.types import Message, Receive, Scope, Send
|
7
|
-
|
8
|
-
|
9
|
-
class WebSocketState(enum.Enum):
|
10
|
-
CONNECTING = 0
|
11
|
-
CONNECTED = 1
|
12
|
-
DISCONNECTED = 2
|
13
|
-
|
14
|
-
|
15
|
-
class WebSocketDisconnect(Exception):
|
16
|
-
def __init__(self, code: int = 1000, reason: typing.Optional[str] = None) -> None:
|
17
|
-
self.code = code
|
18
|
-
self.reason = reason or ""
|
19
|
-
|
20
|
-
|
21
|
-
class WebSocket(HTTPConnection):
|
22
|
-
def __init__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
23
|
-
super().__init__(scope)
|
24
|
-
assert scope["type"] == "websocket"
|
25
|
-
self._receive = receive
|
26
|
-
self._send = send
|
27
|
-
self.client_state = WebSocketState.CONNECTING
|
28
|
-
self.application_state = WebSocketState.CONNECTING
|
29
|
-
|
30
|
-
async def receive(self) -> Message:
|
31
|
-
"""
|
32
|
-
Receive ASGI websocket messages, ensuring valid state transitions.
|
33
|
-
"""
|
34
|
-
if self.client_state == WebSocketState.CONNECTING:
|
35
|
-
message = await self._receive()
|
36
|
-
message_type = message["type"]
|
37
|
-
if message_type != "websocket.connect":
|
38
|
-
raise RuntimeError(
|
39
|
-
'Expected ASGI message "websocket.connect", '
|
40
|
-
f"but got {message_type!r}"
|
41
|
-
)
|
42
|
-
self.client_state = WebSocketState.CONNECTED
|
43
|
-
return message
|
44
|
-
elif self.client_state == WebSocketState.CONNECTED:
|
45
|
-
message = await self._receive()
|
46
|
-
message_type = message["type"]
|
47
|
-
if message_type not in {"websocket.receive", "websocket.disconnect"}:
|
48
|
-
raise RuntimeError(
|
49
|
-
'Expected ASGI message "websocket.receive" or '
|
50
|
-
f'"websocket.disconnect", but got {message_type!r}'
|
51
|
-
)
|
52
|
-
if message_type == "websocket.disconnect":
|
53
|
-
self.client_state = WebSocketState.DISCONNECTED
|
54
|
-
return message
|
55
|
-
else:
|
56
|
-
raise RuntimeError(
|
57
|
-
'Cannot call "receive" once a disconnect message has been received.'
|
58
|
-
)
|
59
|
-
|
60
|
-
async def send(self, message: Message) -> None:
|
61
|
-
"""
|
62
|
-
Send ASGI websocket messages, ensuring valid state transitions.
|
63
|
-
"""
|
64
|
-
if self.application_state == WebSocketState.CONNECTING:
|
65
|
-
message_type = message["type"]
|
66
|
-
if message_type not in {"websocket.accept", "websocket.close"}:
|
67
|
-
raise RuntimeError(
|
68
|
-
'Expected ASGI message "websocket.accept" or '
|
69
|
-
f'"websocket.close", but got {message_type!r}'
|
70
|
-
)
|
71
|
-
if message_type == "websocket.close":
|
72
|
-
self.application_state = WebSocketState.DISCONNECTED
|
73
|
-
else:
|
74
|
-
self.application_state = WebSocketState.CONNECTED
|
75
|
-
await self._send(message)
|
76
|
-
elif self.application_state == WebSocketState.CONNECTED:
|
77
|
-
message_type = message["type"]
|
78
|
-
if message_type not in {"websocket.send", "websocket.close"}:
|
79
|
-
raise RuntimeError(
|
80
|
-
'Expected ASGI message "websocket.send" or "websocket.close", '
|
81
|
-
f"but got {message_type!r}"
|
82
|
-
)
|
83
|
-
if message_type == "websocket.close":
|
84
|
-
self.application_state = WebSocketState.DISCONNECTED
|
85
|
-
await self._send(message)
|
86
|
-
else:
|
87
|
-
raise RuntimeError('Cannot call "send" once a close message has been sent.')
|
88
|
-
|
89
|
-
async def accept(
|
90
|
-
self,
|
91
|
-
subprotocol: typing.Optional[str] = None,
|
92
|
-
headers: typing.Optional[typing.Iterable[typing.Tuple[bytes, bytes]]] = None,
|
93
|
-
) -> None:
|
94
|
-
headers = headers or []
|
95
|
-
|
96
|
-
if self.client_state == WebSocketState.CONNECTING:
|
97
|
-
# If we haven't yet seen the 'connect' message, then wait for it first.
|
98
|
-
await self.receive()
|
99
|
-
await self.send(
|
100
|
-
{"type": "websocket.accept", "subprotocol": subprotocol, "headers": headers}
|
101
|
-
)
|
102
|
-
|
103
|
-
def _raise_on_disconnect(self, message: Message) -> None:
|
104
|
-
if message["type"] == "websocket.disconnect":
|
105
|
-
raise WebSocketDisconnect(message["code"], message.get("reason"))
|
106
|
-
|
107
|
-
async def receive_text(self) -> str:
|
108
|
-
if self.application_state != WebSocketState.CONNECTED:
|
109
|
-
raise RuntimeError(
|
110
|
-
'WebSocket is not connected. Need to call "accept" first.'
|
111
|
-
)
|
112
|
-
message = await self.receive()
|
113
|
-
self._raise_on_disconnect(message)
|
114
|
-
return typing.cast(str, message["text"])
|
115
|
-
|
116
|
-
async def receive_bytes(self) -> bytes:
|
117
|
-
if self.application_state != WebSocketState.CONNECTED:
|
118
|
-
raise RuntimeError(
|
119
|
-
'WebSocket is not connected. Need to call "accept" first.'
|
120
|
-
)
|
121
|
-
message = await self.receive()
|
122
|
-
self._raise_on_disconnect(message)
|
123
|
-
return typing.cast(bytes, message["bytes"])
|
124
|
-
|
125
|
-
async def receive_json(self, mode: str = "text") -> typing.Any:
|
126
|
-
if mode not in {"text", "binary"}:
|
127
|
-
raise RuntimeError('The "mode" argument should be "text" or "binary".')
|
128
|
-
if self.application_state != WebSocketState.CONNECTED:
|
129
|
-
raise RuntimeError(
|
130
|
-
'WebSocket is not connected. Need to call "accept" first.'
|
131
|
-
)
|
132
|
-
message = await self.receive()
|
133
|
-
self._raise_on_disconnect(message)
|
134
|
-
|
135
|
-
if mode == "text":
|
136
|
-
text = message["text"]
|
137
|
-
else:
|
138
|
-
text = message["bytes"].decode("utf-8")
|
139
|
-
return json.loads(text)
|
140
|
-
|
141
|
-
async def iter_text(self) -> typing.AsyncIterator[str]:
|
142
|
-
try:
|
143
|
-
while True:
|
144
|
-
yield await self.receive_text()
|
145
|
-
except WebSocketDisconnect:
|
146
|
-
pass
|
147
|
-
|
148
|
-
async def iter_bytes(self) -> typing.AsyncIterator[bytes]:
|
149
|
-
try:
|
150
|
-
while True:
|
151
|
-
yield await self.receive_bytes()
|
152
|
-
except WebSocketDisconnect:
|
153
|
-
pass
|
154
|
-
|
155
|
-
async def iter_json(self) -> typing.AsyncIterator[typing.Any]:
|
156
|
-
try:
|
157
|
-
while True:
|
158
|
-
yield await self.receive_json()
|
159
|
-
except WebSocketDisconnect:
|
160
|
-
pass
|
161
|
-
|
162
|
-
async def send_text(self, data: str) -> None:
|
163
|
-
await self.send({"type": "websocket.send", "text": data})
|
164
|
-
|
165
|
-
async def send_bytes(self, data: bytes) -> None:
|
166
|
-
await self.send({"type": "websocket.send", "bytes": data})
|
167
|
-
|
168
|
-
async def send_json(self, data: typing.Any, mode: str = "text") -> None:
|
169
|
-
if mode not in {"text", "binary"}:
|
170
|
-
raise RuntimeError('The "mode" argument should be "text" or "binary".')
|
171
|
-
text = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
172
|
-
if mode == "text":
|
173
|
-
await self.send({"type": "websocket.send", "text": text})
|
174
|
-
else:
|
175
|
-
await self.send({"type": "websocket.send", "bytes": text.encode("utf-8")})
|
176
|
-
|
177
|
-
async def close(
|
178
|
-
self, code: int = 1000, reason: typing.Optional[str] = None
|
179
|
-
) -> None:
|
180
|
-
await self.send(
|
181
|
-
{"type": "websocket.close", "code": code, "reason": reason or ""}
|
182
|
-
)
|
183
|
-
|
184
|
-
|
185
|
-
class WebSocketClose:
|
186
|
-
def __init__(self, code: int = 1000, reason: typing.Optional[str] = None) -> None:
|
187
|
-
self.code = code
|
188
|
-
self.reason = reason or ""
|
189
|
-
|
190
|
-
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
191
|
-
await send(
|
192
|
-
{"type": "websocket.close", "code": self.code, "reason": self.reason}
|
193
|
-
)
|
prefect/blocks/kubernetes.py
DELETED
@@ -1,119 +0,0 @@
|
|
1
|
-
from pathlib import Path
|
2
|
-
from typing import TYPE_CHECKING, Dict, Type
|
3
|
-
|
4
|
-
import yaml
|
5
|
-
|
6
|
-
from prefect._internal.compatibility.deprecated import deprecated_class
|
7
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
8
|
-
from prefect._internal.schemas.validators import validate_yaml
|
9
|
-
|
10
|
-
if HAS_PYDANTIC_V2:
|
11
|
-
from pydantic.v1 import Field, validator
|
12
|
-
else:
|
13
|
-
from pydantic import Field, validator
|
14
|
-
|
15
|
-
from typing_extensions import Self
|
16
|
-
|
17
|
-
from prefect.blocks.core import Block
|
18
|
-
from prefect.utilities.collections import listrepr
|
19
|
-
from prefect.utilities.importtools import lazy_import
|
20
|
-
|
21
|
-
if TYPE_CHECKING:
|
22
|
-
import kubernetes
|
23
|
-
from kubernetes.client.api_client import ApiClient
|
24
|
-
else:
|
25
|
-
kubernetes = lazy_import("kubernetes")
|
26
|
-
|
27
|
-
|
28
|
-
@deprecated_class(
|
29
|
-
start_date="Mar 2024",
|
30
|
-
help="Use the KubernetesClusterConfig block from prefect-kubernetes instead.",
|
31
|
-
)
|
32
|
-
class KubernetesClusterConfig(Block):
|
33
|
-
"""
|
34
|
-
Stores configuration for interaction with Kubernetes clusters.
|
35
|
-
|
36
|
-
See `from_file` for creation.
|
37
|
-
|
38
|
-
Attributes:
|
39
|
-
config: The entire loaded YAML contents of a kubectl config file
|
40
|
-
context_name: The name of the kubectl context to use
|
41
|
-
|
42
|
-
Example:
|
43
|
-
Load a saved Kubernetes cluster config:
|
44
|
-
```python
|
45
|
-
from prefect.blocks.kubernetes import KubernetesClusterConfig
|
46
|
-
|
47
|
-
cluster_config_block = KubernetesClusterConfig.load("BLOCK_NAME")
|
48
|
-
```
|
49
|
-
"""
|
50
|
-
|
51
|
-
_block_type_name = "Kubernetes Cluster Config"
|
52
|
-
_logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/2d0b896006ad463b49c28aaac14f31e00e32cfab-250x250.png"
|
53
|
-
_documentation_url = "https://docs.prefect.io/api-ref/prefect/blocks/kubernetes/#prefect.blocks.kubernetes.KubernetesClusterConfig"
|
54
|
-
|
55
|
-
config: Dict = Field(
|
56
|
-
default=..., description="The entire contents of a kubectl config file."
|
57
|
-
)
|
58
|
-
context_name: str = Field(
|
59
|
-
default=..., description="The name of the kubectl context to use."
|
60
|
-
)
|
61
|
-
|
62
|
-
@validator("config", pre=True)
|
63
|
-
def parse_yaml_config(cls, value):
|
64
|
-
return validate_yaml(value)
|
65
|
-
|
66
|
-
@classmethod
|
67
|
-
def from_file(cls: Type[Self], path: Path = None, context_name: str = None) -> Self:
|
68
|
-
"""
|
69
|
-
Create a cluster config from the a Kubernetes config file.
|
70
|
-
|
71
|
-
By default, the current context in the default Kubernetes config file will be
|
72
|
-
used.
|
73
|
-
|
74
|
-
An alternative file or context may be specified.
|
75
|
-
|
76
|
-
The entire config file will be loaded and stored.
|
77
|
-
"""
|
78
|
-
kube_config = kubernetes.config.kube_config
|
79
|
-
|
80
|
-
path = Path(path or kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
|
81
|
-
path = path.expanduser().resolve()
|
82
|
-
|
83
|
-
# Determine the context
|
84
|
-
existing_contexts, current_context = kube_config.list_kube_config_contexts(
|
85
|
-
config_file=str(path)
|
86
|
-
)
|
87
|
-
context_names = {ctx["name"] for ctx in existing_contexts}
|
88
|
-
if context_name:
|
89
|
-
if context_name not in context_names:
|
90
|
-
raise ValueError(
|
91
|
-
f"Context {context_name!r} not found. "
|
92
|
-
f"Specify one of: {listrepr(context_names, sep=', ')}."
|
93
|
-
)
|
94
|
-
else:
|
95
|
-
context_name = current_context["name"]
|
96
|
-
|
97
|
-
# Load the entire config file
|
98
|
-
config_file_contents = path.read_text()
|
99
|
-
config_dict = yaml.safe_load(config_file_contents)
|
100
|
-
|
101
|
-
return cls(config=config_dict, context_name=context_name)
|
102
|
-
|
103
|
-
def get_api_client(self) -> "ApiClient":
|
104
|
-
"""
|
105
|
-
Returns a Kubernetes API client for this cluster config.
|
106
|
-
"""
|
107
|
-
return kubernetes.config.kube_config.new_client_from_config_dict(
|
108
|
-
config_dict=self.config, context=self.context_name
|
109
|
-
)
|
110
|
-
|
111
|
-
def configure_client(self) -> None:
|
112
|
-
"""
|
113
|
-
Activates this cluster configuration by loading the configuration into the
|
114
|
-
Kubernetes Python client. After calling this, Kubernetes API clients can use
|
115
|
-
this config's context.
|
116
|
-
"""
|
117
|
-
kubernetes.config.kube_config.load_kube_config_from_dict(
|
118
|
-
config_dict=self.config, context=self.context_name
|
119
|
-
)
|
prefect/deprecated/__init__.py
DELETED
File without changes
|
@@ -1,350 +0,0 @@
|
|
1
|
-
import base64
|
2
|
-
import json
|
3
|
-
import uuid
|
4
|
-
import warnings
|
5
|
-
from typing import (
|
6
|
-
TYPE_CHECKING,
|
7
|
-
Any,
|
8
|
-
Dict,
|
9
|
-
Generic,
|
10
|
-
Iterable,
|
11
|
-
Tuple,
|
12
|
-
Type,
|
13
|
-
TypeVar,
|
14
|
-
Union,
|
15
|
-
)
|
16
|
-
|
17
|
-
import cloudpickle
|
18
|
-
|
19
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
20
|
-
|
21
|
-
if HAS_PYDANTIC_V2:
|
22
|
-
import pydantic.v1 as pydantic
|
23
|
-
else:
|
24
|
-
import pydantic
|
25
|
-
|
26
|
-
from typing_extensions import Protocol
|
27
|
-
|
28
|
-
from prefect._internal.compatibility.deprecated import (
|
29
|
-
deprecated_callable,
|
30
|
-
generate_deprecation_message,
|
31
|
-
)
|
32
|
-
from prefect._internal.schemas.bases import PrefectBaseModel
|
33
|
-
|
34
|
-
if TYPE_CHECKING:
|
35
|
-
from prefect.deprecated.packaging.base import PackageManifest
|
36
|
-
|
37
|
-
T = TypeVar("T", bound="DataDocument") # Generic for DataDocument class types
|
38
|
-
D = TypeVar("D", bound=Any) # Generic for DataDocument data types
|
39
|
-
|
40
|
-
|
41
|
-
_SERIALIZERS: Dict[str, "Serializer"] = {}
|
42
|
-
|
43
|
-
|
44
|
-
class Serializer(Protocol[D]):
|
45
|
-
"""
|
46
|
-
Define a serializer that can encode data of type 'D' into bytes
|
47
|
-
"""
|
48
|
-
|
49
|
-
@staticmethod
|
50
|
-
def dumps(data: D, **kwargs: Any) -> bytes:
|
51
|
-
raise NotImplementedError
|
52
|
-
|
53
|
-
@staticmethod
|
54
|
-
def loads(blob: bytes) -> D:
|
55
|
-
raise NotImplementedError
|
56
|
-
|
57
|
-
|
58
|
-
def register_serializer(
|
59
|
-
encoding: Union[str, Tuple[str, ...]], serializer: Serializer = None
|
60
|
-
):
|
61
|
-
"""Register dispatch of `func` on arguments of encoding `encoding`"""
|
62
|
-
|
63
|
-
def wrapper(serializer):
|
64
|
-
if isinstance(encoding, tuple):
|
65
|
-
for e in encoding:
|
66
|
-
register_serializer(e, serializer)
|
67
|
-
else:
|
68
|
-
_SERIALIZERS[encoding] = serializer
|
69
|
-
return serializer
|
70
|
-
|
71
|
-
return wrapper(serializer) if serializer is not None else wrapper
|
72
|
-
|
73
|
-
|
74
|
-
def lookup_serializer(encoding: str) -> Serializer:
|
75
|
-
"""Return the serializer implementation for the given ``encoding``"""
|
76
|
-
try:
|
77
|
-
return _SERIALIZERS[encoding]
|
78
|
-
except KeyError:
|
79
|
-
raise ValueError(f"Unregistered encoding {encoding!r}")
|
80
|
-
|
81
|
-
|
82
|
-
class DataDocument(PrefectBaseModel, Generic[D]):
|
83
|
-
"""
|
84
|
-
A data document includes an encoding string and a blob of encoded data
|
85
|
-
|
86
|
-
Subclasses can define the expected type for the blob's underlying type using the
|
87
|
-
generic variable `D`.
|
88
|
-
|
89
|
-
For example `DataDocument[str]` indicates that a string should be passed when
|
90
|
-
creating the document and a string will be returned when it is decoded.
|
91
|
-
"""
|
92
|
-
|
93
|
-
encoding: str
|
94
|
-
blob: bytes
|
95
|
-
|
96
|
-
# A cache for the decoded data, see `DataDocument.decode`
|
97
|
-
_data: D
|
98
|
-
__slots__ = ["_data"]
|
99
|
-
|
100
|
-
@classmethod
|
101
|
-
@deprecated_callable(
|
102
|
-
start_date="Sep 2022",
|
103
|
-
help="Data documents should not be created. Use result persistence instead.",
|
104
|
-
)
|
105
|
-
def encode(
|
106
|
-
cls: Type["DataDocument"], encoding: str, data: D, **kwargs: Any
|
107
|
-
) -> "DataDocument[D]":
|
108
|
-
"""
|
109
|
-
Create a new data document
|
110
|
-
|
111
|
-
A serializer must be registered for the given `encoding`
|
112
|
-
"""
|
113
|
-
# Dispatch encoding
|
114
|
-
blob = lookup_serializer(encoding).dumps(data, **kwargs)
|
115
|
-
|
116
|
-
inst = cls(blob=blob, encoding=encoding)
|
117
|
-
inst._cache_data(data)
|
118
|
-
return inst
|
119
|
-
|
120
|
-
def decode(self) -> D:
|
121
|
-
"""
|
122
|
-
Get the data from a data document
|
123
|
-
|
124
|
-
A serializer must be registered for the document's encoding
|
125
|
-
"""
|
126
|
-
if self.has_cached_data():
|
127
|
-
return self._data
|
128
|
-
|
129
|
-
# Dispatch decoding
|
130
|
-
data = lookup_serializer(self.encoding).loads(self.blob)
|
131
|
-
|
132
|
-
self._cache_data(data)
|
133
|
-
return data
|
134
|
-
|
135
|
-
def _cache_data(self, data) -> None:
|
136
|
-
# Use object's setattr to avoid a pydantic 'field does not exist' error
|
137
|
-
# See https://github.com/samuelcolvin/pydantic/issues/655
|
138
|
-
object.__setattr__(self, "_data", data)
|
139
|
-
|
140
|
-
def has_cached_data(self):
|
141
|
-
return hasattr(self, "_data")
|
142
|
-
|
143
|
-
def __str__(self) -> str:
|
144
|
-
if self.has_cached_data():
|
145
|
-
return repr(self._data)
|
146
|
-
else:
|
147
|
-
return repr(self)
|
148
|
-
|
149
|
-
def __repr__(self) -> str:
|
150
|
-
return f"{type(self).__name__}(encoding={self.encoding!r})"
|
151
|
-
|
152
|
-
|
153
|
-
@register_serializer("json")
|
154
|
-
class DocumentJSONSerializer:
|
155
|
-
"""
|
156
|
-
Serializes data to JSON.
|
157
|
-
|
158
|
-
Input types must be compatible with the stdlib json library.
|
159
|
-
|
160
|
-
Wraps the `json` library to serialize to UTF-8 bytes instead of string types.
|
161
|
-
"""
|
162
|
-
|
163
|
-
@staticmethod
|
164
|
-
@deprecated_callable(
|
165
|
-
start_date="Sep 2022",
|
166
|
-
help=(
|
167
|
-
"Data document serializers should not be used. Use result serializers"
|
168
|
-
" instead."
|
169
|
-
),
|
170
|
-
)
|
171
|
-
def dumps(data: Any) -> bytes:
|
172
|
-
return json.dumps(data).encode()
|
173
|
-
|
174
|
-
@staticmethod
|
175
|
-
def loads(blob: bytes) -> Any:
|
176
|
-
return json.loads(blob.decode())
|
177
|
-
|
178
|
-
|
179
|
-
@register_serializer("text")
|
180
|
-
class TextSerializer:
|
181
|
-
@staticmethod
|
182
|
-
@deprecated_callable(
|
183
|
-
start_date="Sep 2022",
|
184
|
-
help=(
|
185
|
-
"Data document serializers should not be used. Use result serializers"
|
186
|
-
" instead."
|
187
|
-
),
|
188
|
-
)
|
189
|
-
def dumps(data: str) -> bytes:
|
190
|
-
return data.encode()
|
191
|
-
|
192
|
-
@staticmethod
|
193
|
-
def loads(blob: bytes) -> str:
|
194
|
-
return blob.decode()
|
195
|
-
|
196
|
-
|
197
|
-
@register_serializer("cloudpickle")
|
198
|
-
class DocumentPickleSerializer:
|
199
|
-
"""
|
200
|
-
Serializes arbitrary objects using the pickle protocol.
|
201
|
-
|
202
|
-
Wraps `cloudpickle` to encode bytes in base64 for safe transmission.
|
203
|
-
"""
|
204
|
-
|
205
|
-
@staticmethod
|
206
|
-
@deprecated_callable(
|
207
|
-
start_date="Sep 2022",
|
208
|
-
help=(
|
209
|
-
"Data document serializers should not be used. Use result serializers"
|
210
|
-
" instead."
|
211
|
-
),
|
212
|
-
)
|
213
|
-
def dumps(data: Any) -> bytes:
|
214
|
-
data_bytes = cloudpickle.dumps(data)
|
215
|
-
|
216
|
-
return base64.encodebytes(data_bytes)
|
217
|
-
|
218
|
-
@staticmethod
|
219
|
-
def loads(blob: bytes) -> Any:
|
220
|
-
return cloudpickle.loads(base64.decodebytes(blob))
|
221
|
-
# TODO: Consider adding python version data to pickle payloads to raise
|
222
|
-
# more helpful errors for users.
|
223
|
-
# A TypeError("expected bytes-like object, not int") will be raised if
|
224
|
-
# a document is deserialized by Python 3.7 and serialized by 3.8+
|
225
|
-
|
226
|
-
|
227
|
-
@register_serializer("package-manifest")
|
228
|
-
class PackageManifestSerializer:
|
229
|
-
"""
|
230
|
-
Serializes a package manifest.
|
231
|
-
"""
|
232
|
-
|
233
|
-
@staticmethod
|
234
|
-
@deprecated_callable(
|
235
|
-
start_date="Sep 2022",
|
236
|
-
help=(
|
237
|
-
"Data document serializers should not be used. Use result serializers"
|
238
|
-
" instead."
|
239
|
-
),
|
240
|
-
)
|
241
|
-
def dumps(data: "PackageManifest") -> bytes:
|
242
|
-
return data.json().encode()
|
243
|
-
|
244
|
-
@staticmethod
|
245
|
-
def loads(blob: bytes) -> "PackageManifest":
|
246
|
-
from prefect.deprecated.packaging.base import PackageManifest
|
247
|
-
|
248
|
-
return PackageManifest.parse_raw(blob)
|
249
|
-
|
250
|
-
|
251
|
-
@register_serializer("result")
|
252
|
-
class ResultSerializer:
|
253
|
-
"""
|
254
|
-
Serializes a result object
|
255
|
-
"""
|
256
|
-
|
257
|
-
@staticmethod
|
258
|
-
@deprecated_callable(
|
259
|
-
start_date="Sep 2022",
|
260
|
-
help=(
|
261
|
-
"Data document serializers should not be used. Use result serializers"
|
262
|
-
" instead."
|
263
|
-
),
|
264
|
-
)
|
265
|
-
def dumps(data: "_Result") -> bytes:
|
266
|
-
return data.json().encode()
|
267
|
-
|
268
|
-
@staticmethod
|
269
|
-
def loads(blob: bytes) -> "_Result":
|
270
|
-
return _Result.parse_raw(blob)
|
271
|
-
|
272
|
-
|
273
|
-
def result_from_state_with_data_document(state, raise_on_failure: bool) -> Any:
|
274
|
-
# Complain about usage
|
275
|
-
warnings.warn(
|
276
|
-
generate_deprecation_message(
|
277
|
-
"Retrieving results from states with data documents", start_date="Sep 2022"
|
278
|
-
),
|
279
|
-
DeprecationWarning,
|
280
|
-
stacklevel=3,
|
281
|
-
)
|
282
|
-
|
283
|
-
data = None
|
284
|
-
|
285
|
-
if state.data:
|
286
|
-
data = state.data.decode()
|
287
|
-
|
288
|
-
from prefect.states import State
|
289
|
-
|
290
|
-
if (
|
291
|
-
state.is_failed() or state.is_crashed() or state.is_cancelled()
|
292
|
-
) and raise_on_failure:
|
293
|
-
if isinstance(data, Exception):
|
294
|
-
raise data
|
295
|
-
elif isinstance(data, BaseException):
|
296
|
-
warnings.warn(
|
297
|
-
f"State result is a {type(data).__name__!r} type and is not safe "
|
298
|
-
"to re-raise, it will be returned instead."
|
299
|
-
)
|
300
|
-
return data
|
301
|
-
elif isinstance(data, State):
|
302
|
-
data.result(fetch=False)
|
303
|
-
elif isinstance(data, Iterable) and all([isinstance(o, State) for o in data]):
|
304
|
-
# raise the first failure we find
|
305
|
-
for state in data:
|
306
|
-
state.result(fetch=False)
|
307
|
-
|
308
|
-
# we don't make this an else in case any of the above conditionals doesn't raise
|
309
|
-
raise TypeError(
|
310
|
-
f"Unexpected result for failure state: {data!r} —— "
|
311
|
-
f"{type(data).__name__} cannot be resolved into an exception"
|
312
|
-
)
|
313
|
-
|
314
|
-
return data
|
315
|
-
|
316
|
-
|
317
|
-
async def _persist_serialized_result(
|
318
|
-
content: bytes,
|
319
|
-
filesystem,
|
320
|
-
) -> DataDocument:
|
321
|
-
key = uuid.uuid4().hex
|
322
|
-
await filesystem.write_path(key, content)
|
323
|
-
result = _Result(key=key, filesystem_document_id=filesystem._block_document_id)
|
324
|
-
return DataDocument.encode("result", result)
|
325
|
-
|
326
|
-
|
327
|
-
async def _retrieve_serialized_result(document: DataDocument, client) -> bytes:
|
328
|
-
from prefect.blocks.core import Block
|
329
|
-
|
330
|
-
if document.encoding != "result":
|
331
|
-
raise TypeError(
|
332
|
-
f"Got unsupported data document encoding of {document.encoding!r}. "
|
333
|
-
"Expected 'result'."
|
334
|
-
)
|
335
|
-
result = document.decode()
|
336
|
-
filesystem_document = await client.read_block_document(
|
337
|
-
result.filesystem_document_id
|
338
|
-
)
|
339
|
-
filesystem = Block._from_block_document(filesystem_document)
|
340
|
-
return await filesystem.read_path(result.key)
|
341
|
-
|
342
|
-
|
343
|
-
async def _retrieve_result(state, client):
|
344
|
-
serialized_result = await _retrieve_serialized_result(state.data, client)
|
345
|
-
return DataDocument.parse_raw(serialized_result).decode()
|
346
|
-
|
347
|
-
|
348
|
-
class _Result(pydantic.BaseModel):
|
349
|
-
key: str
|
350
|
-
filesystem_document_id: uuid.UUID
|