prefect-client 3.0.0rc14__py3-none-any.whl → 3.0.0rc16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/compatibility/deprecated.py +0 -53
- prefect/_internal/schemas/validators.py +3 -91
- prefect/client/orchestration.py +4 -47
- prefect/client/schemas/actions.py +7 -10
- prefect/client/schemas/filters.py +0 -12
- prefect/client/schemas/objects.py +0 -6
- prefect/client/schemas/responses.py +0 -7
- prefect/context.py +2 -1
- prefect/deployments/base.py +0 -9
- prefect/deployments/runner.py +1 -36
- prefect/events/clients.py +39 -56
- prefect/events/utilities.py +0 -2
- prefect/flows.py +10 -48
- prefect/futures.py +0 -5
- prefect/profiles.toml +3 -0
- prefect/runner/runner.py +0 -10
- prefect/runner/storage.py +4 -0
- prefect/runtime/flow_run.py +19 -1
- prefect/settings.py +78 -13
- prefect/task_engine.py +60 -58
- prefect/task_worker.py +4 -1
- prefect/tasks.py +9 -5
- prefect/transactions.py +30 -3
- prefect/utilities/asyncutils.py +0 -6
- prefect/utilities/engine.py +3 -0
- {prefect_client-3.0.0rc14.dist-info → prefect_client-3.0.0rc16.dist-info}/METADATA +39 -40
- {prefect_client-3.0.0rc14.dist-info → prefect_client-3.0.0rc16.dist-info}/RECORD +30 -30
- {prefect_client-3.0.0rc14.dist-info → prefect_client-3.0.0rc16.dist-info}/WHEEL +1 -1
- {prefect_client-3.0.0rc14.dist-info → prefect_client-3.0.0rc16.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc14.dist-info → prefect_client-3.0.0rc16.dist-info}/top_level.txt +0 -0
prefect/events/clients.py
CHANGED
@@ -15,7 +15,6 @@ from typing import (
|
|
15
15
|
)
|
16
16
|
from uuid import UUID
|
17
17
|
|
18
|
-
import httpx
|
19
18
|
import orjson
|
20
19
|
import pendulum
|
21
20
|
from cachetools import TTLCache
|
@@ -29,10 +28,14 @@ from websockets.exceptions import (
|
|
29
28
|
ConnectionClosedOK,
|
30
29
|
)
|
31
30
|
|
32
|
-
from prefect.client.base import PrefectHttpxAsyncClient
|
33
31
|
from prefect.events import Event
|
34
32
|
from prefect.logging import get_logger
|
35
|
-
from prefect.settings import
|
33
|
+
from prefect.settings import (
|
34
|
+
PREFECT_API_KEY,
|
35
|
+
PREFECT_API_URL,
|
36
|
+
PREFECT_CLOUD_API_URL,
|
37
|
+
PREFECT_SERVER_ALLOW_EPHEMERAL_MODE,
|
38
|
+
)
|
36
39
|
|
37
40
|
if TYPE_CHECKING:
|
38
41
|
from prefect.events.filters import EventFilter
|
@@ -66,7 +69,7 @@ logger = get_logger(__name__)
|
|
66
69
|
|
67
70
|
def get_events_client(
|
68
71
|
reconnection_attempts: int = 10,
|
69
|
-
checkpoint_every: int =
|
72
|
+
checkpoint_every: int = 700,
|
70
73
|
) -> "EventsClient":
|
71
74
|
api_url = PREFECT_API_URL.value()
|
72
75
|
if isinstance(api_url, str) and api_url.startswith(PREFECT_CLOUD_API_URL.value()):
|
@@ -74,13 +77,25 @@ def get_events_client(
|
|
74
77
|
reconnection_attempts=reconnection_attempts,
|
75
78
|
checkpoint_every=checkpoint_every,
|
76
79
|
)
|
77
|
-
elif
|
80
|
+
elif api_url:
|
81
|
+
return PrefectEventsClient(
|
82
|
+
reconnection_attempts=reconnection_attempts,
|
83
|
+
checkpoint_every=checkpoint_every,
|
84
|
+
)
|
85
|
+
elif PREFECT_SERVER_ALLOW_EPHEMERAL_MODE:
|
86
|
+
from prefect.server.api.server import SubprocessASGIServer
|
87
|
+
|
88
|
+
server = SubprocessASGIServer()
|
89
|
+
server.start()
|
78
90
|
return PrefectEventsClient(
|
91
|
+
api_url=server.api_url,
|
79
92
|
reconnection_attempts=reconnection_attempts,
|
80
93
|
checkpoint_every=checkpoint_every,
|
81
94
|
)
|
82
95
|
else:
|
83
|
-
|
96
|
+
raise ValueError(
|
97
|
+
"No Prefect API URL provided. Please set PREFECT_API_URL to the address of a running Prefect server."
|
98
|
+
)
|
84
99
|
|
85
100
|
|
86
101
|
def get_events_subscriber(
|
@@ -88,20 +103,29 @@ def get_events_subscriber(
|
|
88
103
|
reconnection_attempts: int = 10,
|
89
104
|
) -> "PrefectEventSubscriber":
|
90
105
|
api_url = PREFECT_API_URL.value()
|
91
|
-
if not api_url:
|
92
|
-
raise ValueError(
|
93
|
-
"A Prefect server or Prefect Cloud is required to start an event "
|
94
|
-
"subscriber. Please check the PREFECT_API_URL setting in your profile."
|
95
|
-
)
|
96
106
|
|
97
107
|
if isinstance(api_url, str) and api_url.startswith(PREFECT_CLOUD_API_URL.value()):
|
98
108
|
return PrefectCloudEventSubscriber(
|
99
109
|
filter=filter, reconnection_attempts=reconnection_attempts
|
100
110
|
)
|
101
|
-
|
111
|
+
elif api_url:
|
102
112
|
return PrefectEventSubscriber(
|
103
113
|
filter=filter, reconnection_attempts=reconnection_attempts
|
104
114
|
)
|
115
|
+
elif PREFECT_SERVER_ALLOW_EPHEMERAL_MODE:
|
116
|
+
from prefect.server.api.server import SubprocessASGIServer
|
117
|
+
|
118
|
+
server = SubprocessASGIServer()
|
119
|
+
server.start()
|
120
|
+
return PrefectEventSubscriber(
|
121
|
+
api_url=server.api_url,
|
122
|
+
filter=filter,
|
123
|
+
reconnection_attempts=reconnection_attempts,
|
124
|
+
)
|
125
|
+
else:
|
126
|
+
raise ValueError(
|
127
|
+
"No Prefect API URL provided. Please set PREFECT_API_URL to the address of a running Prefect server."
|
128
|
+
)
|
105
129
|
|
106
130
|
|
107
131
|
class EventsClient(abc.ABC):
|
@@ -201,47 +225,6 @@ def _get_api_url_and_key(
|
|
201
225
|
return api_url, api_key
|
202
226
|
|
203
227
|
|
204
|
-
class PrefectEphemeralEventsClient(EventsClient):
|
205
|
-
"""A Prefect Events client that sends events to an ephemeral Prefect server"""
|
206
|
-
|
207
|
-
def __init__(self):
|
208
|
-
if PREFECT_API_KEY.value():
|
209
|
-
raise ValueError(
|
210
|
-
"PrefectEphemeralEventsClient cannot be used when PREFECT_API_KEY is set."
|
211
|
-
" Please use PrefectEventsClient or PrefectCloudEventsClient instead."
|
212
|
-
)
|
213
|
-
from prefect.server.api.server import create_app
|
214
|
-
|
215
|
-
app = create_app(ephemeral=True)
|
216
|
-
|
217
|
-
self._http_client = PrefectHttpxAsyncClient(
|
218
|
-
transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),
|
219
|
-
base_url="http://ephemeral-prefect/api",
|
220
|
-
enable_csrf_support=False,
|
221
|
-
)
|
222
|
-
|
223
|
-
async def __aenter__(self) -> Self:
|
224
|
-
await super().__aenter__()
|
225
|
-
await self._http_client.__aenter__()
|
226
|
-
return self
|
227
|
-
|
228
|
-
async def __aexit__(
|
229
|
-
self,
|
230
|
-
exc_type: Optional[Type[Exception]],
|
231
|
-
exc_val: Optional[Exception],
|
232
|
-
exc_tb: Optional[TracebackType],
|
233
|
-
) -> None:
|
234
|
-
self._websocket = None
|
235
|
-
await self._http_client.__aexit__(exc_type, exc_val, exc_tb)
|
236
|
-
return await super().__aexit__(exc_type, exc_val, exc_tb)
|
237
|
-
|
238
|
-
async def _emit(self, event: Event) -> None:
|
239
|
-
await self._http_client.post(
|
240
|
-
"/events",
|
241
|
-
json=[event.model_dump(mode="json")],
|
242
|
-
)
|
243
|
-
|
244
|
-
|
245
228
|
class PrefectEventsClient(EventsClient):
|
246
229
|
"""A Prefect Events client that streams events to a Prefect server"""
|
247
230
|
|
@@ -252,7 +235,7 @@ class PrefectEventsClient(EventsClient):
|
|
252
235
|
self,
|
253
236
|
api_url: Optional[str] = None,
|
254
237
|
reconnection_attempts: int = 10,
|
255
|
-
checkpoint_every: int =
|
238
|
+
checkpoint_every: int = 700,
|
256
239
|
):
|
257
240
|
"""
|
258
241
|
Args:
|
@@ -371,7 +354,7 @@ class PrefectCloudEventsClient(PrefectEventsClient):
|
|
371
354
|
api_url: Optional[str] = None,
|
372
355
|
api_key: Optional[str] = None,
|
373
356
|
reconnection_attempts: int = 10,
|
374
|
-
checkpoint_every: int =
|
357
|
+
checkpoint_every: int = 700,
|
375
358
|
):
|
376
359
|
"""
|
377
360
|
Args:
|
@@ -435,9 +418,9 @@ class PrefectEventSubscriber:
|
|
435
418
|
reconnection_attempts: When the client is disconnected, how many times
|
436
419
|
the client should attempt to reconnect
|
437
420
|
"""
|
421
|
+
self._api_key = None
|
438
422
|
if not api_url:
|
439
423
|
api_url = cast(str, PREFECT_API_URL.value())
|
440
|
-
self._api_key = None
|
441
424
|
|
442
425
|
from prefect.events.filters import EventFilter
|
443
426
|
|
prefect/events/utilities.py
CHANGED
@@ -8,7 +8,6 @@ from pydantic_extra_types.pendulum_dt import DateTime
|
|
8
8
|
from .clients import (
|
9
9
|
AssertingEventsClient,
|
10
10
|
PrefectCloudEventsClient,
|
11
|
-
PrefectEphemeralEventsClient,
|
12
11
|
PrefectEventsClient,
|
13
12
|
)
|
14
13
|
from .schemas.events import Event, RelatedResource
|
@@ -53,7 +52,6 @@ def emit_event(
|
|
53
52
|
AssertingEventsClient,
|
54
53
|
PrefectCloudEventsClient,
|
55
54
|
PrefectEventsClient,
|
56
|
-
PrefectEphemeralEventsClient,
|
57
55
|
]
|
58
56
|
worker_instance = EventsWorker.instance()
|
59
57
|
|
prefect/flows.py
CHANGED
@@ -47,16 +47,12 @@ from pydantic.v1.errors import ConfigError # TODO
|
|
47
47
|
from rich.console import Console
|
48
48
|
from typing_extensions import Literal, ParamSpec, Self
|
49
49
|
|
50
|
-
from prefect._internal.compatibility.deprecated import (
|
51
|
-
deprecated_parameter,
|
52
|
-
)
|
53
50
|
from prefect._internal.concurrency.api import create_call, from_async
|
54
51
|
from prefect.blocks.core import Block
|
55
52
|
from prefect.client.orchestration import get_client
|
56
53
|
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
57
54
|
from prefect.client.schemas.objects import Flow as FlowSchema
|
58
55
|
from prefect.client.schemas.objects import FlowRun
|
59
|
-
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
60
56
|
from prefect.client.utilities import client_injector
|
61
57
|
from prefect.docker.docker_image import DockerImage
|
62
58
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
@@ -66,6 +62,7 @@ from prefect.exceptions import (
|
|
66
62
|
ObjectNotFound,
|
67
63
|
ParameterTypeError,
|
68
64
|
ScriptError,
|
65
|
+
TerminationSignal,
|
69
66
|
UnspecifiedFlowError,
|
70
67
|
)
|
71
68
|
from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
|
@@ -626,18 +623,6 @@ class Flow(Generic[P, R]):
|
|
626
623
|
return serialized_parameters
|
627
624
|
|
628
625
|
@sync_compatible
|
629
|
-
@deprecated_parameter(
|
630
|
-
"schedule",
|
631
|
-
start_date="Mar 2024",
|
632
|
-
when=lambda p: p is not None,
|
633
|
-
help="Use `schedules` instead.",
|
634
|
-
)
|
635
|
-
@deprecated_parameter(
|
636
|
-
"is_schedule_active",
|
637
|
-
start_date="Mar 2024",
|
638
|
-
when=lambda p: p is not None,
|
639
|
-
help="Use `paused` instead.",
|
640
|
-
)
|
641
626
|
async def to_deployment(
|
642
627
|
self,
|
643
628
|
name: str,
|
@@ -653,8 +638,6 @@ class Flow(Generic[P, R]):
|
|
653
638
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
654
639
|
paused: Optional[bool] = None,
|
655
640
|
schedules: Optional[List["FlexibleScheduleList"]] = None,
|
656
|
-
schedule: Optional[SCHEDULE_TYPES] = None,
|
657
|
-
is_schedule_active: Optional[bool] = None,
|
658
641
|
parameters: Optional[dict] = None,
|
659
642
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
660
643
|
description: Optional[str] = None,
|
@@ -678,10 +661,6 @@ class Flow(Generic[P, R]):
|
|
678
661
|
paused: Whether or not to set this deployment as paused.
|
679
662
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
680
663
|
Used to define multiple schedules or additional scheduling options such as `timezone`.
|
681
|
-
schedule: A schedule object defining when to execute runs of this deployment.
|
682
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
683
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
684
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
685
664
|
parameters: A dictionary of default parameter values to pass to runs of this deployment.
|
686
665
|
triggers: A list of triggers that will kick off runs of this deployment.
|
687
666
|
description: A description for the created deployment. Defaults to the flow's
|
@@ -734,8 +713,6 @@ class Flow(Generic[P, R]):
|
|
734
713
|
rrule=rrule,
|
735
714
|
paused=paused,
|
736
715
|
schedules=schedules,
|
737
|
-
schedule=schedule,
|
738
|
-
is_schedule_active=is_schedule_active,
|
739
716
|
tags=tags,
|
740
717
|
triggers=triggers,
|
741
718
|
parameters=parameters or {},
|
@@ -755,8 +732,6 @@ class Flow(Generic[P, R]):
|
|
755
732
|
rrule=rrule,
|
756
733
|
paused=paused,
|
757
734
|
schedules=schedules,
|
758
|
-
schedule=schedule,
|
759
|
-
is_schedule_active=is_schedule_active,
|
760
735
|
tags=tags,
|
761
736
|
triggers=triggers,
|
762
737
|
parameters=parameters or {},
|
@@ -814,8 +789,6 @@ class Flow(Generic[P, R]):
|
|
814
789
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
815
790
|
paused: Optional[bool] = None,
|
816
791
|
schedules: Optional["FlexibleScheduleList"] = None,
|
817
|
-
schedule: Optional[SCHEDULE_TYPES] = None,
|
818
|
-
is_schedule_active: Optional[bool] = None,
|
819
792
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
820
793
|
parameters: Optional[dict] = None,
|
821
794
|
description: Optional[str] = None,
|
@@ -845,11 +818,6 @@ class Flow(Generic[P, R]):
|
|
845
818
|
paused: Whether or not to set this deployment as paused.
|
846
819
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
847
820
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
848
|
-
schedule: A schedule object defining when to execute runs of this deployment. Used to
|
849
|
-
define additional scheduling options such as `timezone`.
|
850
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
851
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
852
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
853
821
|
parameters: A dictionary of default parameter values to pass to runs of this deployment.
|
854
822
|
description: A description for the created deployment. Defaults to the flow's
|
855
823
|
description if not provided.
|
@@ -913,8 +881,6 @@ class Flow(Generic[P, R]):
|
|
913
881
|
rrule=rrule,
|
914
882
|
paused=paused,
|
915
883
|
schedules=schedules,
|
916
|
-
schedule=schedule,
|
917
|
-
is_schedule_active=is_schedule_active,
|
918
884
|
parameters=parameters,
|
919
885
|
description=description,
|
920
886
|
tags=tags,
|
@@ -946,10 +912,15 @@ class Flow(Generic[P, R]):
|
|
946
912
|
else:
|
947
913
|
raise
|
948
914
|
|
949
|
-
|
950
|
-
loop
|
951
|
-
|
952
|
-
|
915
|
+
try:
|
916
|
+
if loop is not None:
|
917
|
+
loop.run_until_complete(runner.start(webserver=webserver))
|
918
|
+
else:
|
919
|
+
asyncio.run(runner.start(webserver=webserver))
|
920
|
+
except (KeyboardInterrupt, TerminationSignal) as exc:
|
921
|
+
logger.info(f"Received {type(exc).__name__}, shutting down...")
|
922
|
+
if loop is not None:
|
923
|
+
loop.stop()
|
953
924
|
|
954
925
|
@classmethod
|
955
926
|
@sync_compatible
|
@@ -1079,8 +1050,6 @@ class Flow(Generic[P, R]):
|
|
1079
1050
|
rrule: Optional[str] = None,
|
1080
1051
|
paused: Optional[bool] = None,
|
1081
1052
|
schedules: Optional[List[DeploymentScheduleCreate]] = None,
|
1082
|
-
schedule: Optional[SCHEDULE_TYPES] = None,
|
1083
|
-
is_schedule_active: Optional[bool] = None,
|
1084
1053
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
1085
1054
|
parameters: Optional[dict] = None,
|
1086
1055
|
description: Optional[str] = None,
|
@@ -1127,11 +1096,6 @@ class Flow(Generic[P, R]):
|
|
1127
1096
|
paused: Whether or not to set this deployment as paused.
|
1128
1097
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
1129
1098
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
1130
|
-
schedule: A schedule object defining when to execute runs of this deployment. Used to
|
1131
|
-
define additional scheduling options like `timezone`.
|
1132
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
1133
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
1134
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
1135
1099
|
parameters: A dictionary of default parameter values to pass to runs of this deployment.
|
1136
1100
|
description: A description for the created deployment. Defaults to the flow's
|
1137
1101
|
description if not provided.
|
@@ -1207,8 +1171,6 @@ class Flow(Generic[P, R]):
|
|
1207
1171
|
rrule=rrule,
|
1208
1172
|
schedules=schedules,
|
1209
1173
|
paused=paused,
|
1210
|
-
schedule=schedule,
|
1211
|
-
is_schedule_active=is_schedule_active,
|
1212
1174
|
triggers=triggers,
|
1213
1175
|
parameters=parameters,
|
1214
1176
|
description=description,
|
prefect/futures.py
CHANGED
@@ -10,7 +10,6 @@ from typing import Any, Callable, Generic, List, Optional, Set, Union, cast
|
|
10
10
|
|
11
11
|
from typing_extensions import TypeVar
|
12
12
|
|
13
|
-
from prefect._internal.compatibility.deprecated import deprecated_async_method
|
14
13
|
from prefect.client.orchestration import get_client
|
15
14
|
from prefect.client.schemas.objects import TaskRun
|
16
15
|
from prefect.exceptions import ObjectNotFound
|
@@ -135,7 +134,6 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
|
|
135
134
|
when the task run is submitted to a ThreadPoolExecutor.
|
136
135
|
"""
|
137
136
|
|
138
|
-
@deprecated_async_method
|
139
137
|
def wait(self, timeout: Optional[float] = None) -> None:
|
140
138
|
try:
|
141
139
|
result = self._wrapped_future.result(timeout=timeout)
|
@@ -144,7 +142,6 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
|
|
144
142
|
if isinstance(result, State):
|
145
143
|
self._final_state = result
|
146
144
|
|
147
|
-
@deprecated_async_method
|
148
145
|
def result(
|
149
146
|
self,
|
150
147
|
timeout: Optional[float] = None,
|
@@ -198,7 +195,6 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
198
195
|
done_callbacks: List[Callable[[PrefectFuture], None]] = []
|
199
196
|
waiter = None
|
200
197
|
|
201
|
-
@deprecated_async_method
|
202
198
|
def wait(self, timeout: Optional[float] = None) -> None:
|
203
199
|
return run_coro_as_sync(self.wait_async(timeout=timeout))
|
204
200
|
|
@@ -235,7 +231,6 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
235
231
|
self._final_state = task_run.state
|
236
232
|
return
|
237
233
|
|
238
|
-
@deprecated_async_method
|
239
234
|
def result(
|
240
235
|
self,
|
241
236
|
timeout: Optional[float] = None,
|
prefect/profiles.toml
CHANGED
@@ -10,5 +10,8 @@ PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = "true"
|
|
10
10
|
# You will need to set these values appropriately for your local development environment
|
11
11
|
PREFECT_API_URL = "http://127.0.0.1:4200/api"
|
12
12
|
|
13
|
+
[profiles.test]
|
14
|
+
PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = "true"
|
15
|
+
PREFECT_API_DATABASE_CONNECTION_URL = "sqlite+aiosqlite:///:memory:"
|
13
16
|
|
14
17
|
[profiles.cloud]
|
prefect/runner/runner.py
CHANGED
@@ -67,7 +67,6 @@ from prefect.client.schemas.filters import (
|
|
67
67
|
)
|
68
68
|
from prefect.client.schemas.objects import Flow as APIFlow
|
69
69
|
from prefect.client.schemas.objects import FlowRun, State, StateType
|
70
|
-
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
71
70
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
72
71
|
from prefect.events.related import tags_as_related_resources
|
73
72
|
from prefect.events.schemas.events import RelatedResource
|
@@ -224,8 +223,6 @@ class Runner:
|
|
224
223
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
225
224
|
paused: Optional[bool] = None,
|
226
225
|
schedules: Optional["FlexibleScheduleList"] = None,
|
227
|
-
schedule: Optional[SCHEDULE_TYPES] = None,
|
228
|
-
is_schedule_active: Optional[bool] = None,
|
229
226
|
parameters: Optional[dict] = None,
|
230
227
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
231
228
|
description: Optional[str] = None,
|
@@ -248,11 +245,6 @@ class Runner:
|
|
248
245
|
or a timedelta object. If a number is given, it will be interpreted as seconds.
|
249
246
|
cron: A cron schedule of when to execute runs of this flow.
|
250
247
|
rrule: An rrule schedule of when to execute runs of this flow.
|
251
|
-
schedule: A schedule object of when to execute runs of this flow. Used for
|
252
|
-
advanced scheduling options like timezone.
|
253
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
254
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
255
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
256
248
|
triggers: A list of triggers that should kick of a run of this flow.
|
257
249
|
parameters: A dictionary of default parameter values to pass to runs of this flow.
|
258
250
|
description: A description for the created deployment. Defaults to the flow's
|
@@ -277,9 +269,7 @@ class Runner:
|
|
277
269
|
cron=cron,
|
278
270
|
rrule=rrule,
|
279
271
|
schedules=schedules,
|
280
|
-
schedule=schedule,
|
281
272
|
paused=paused,
|
282
|
-
is_schedule_active=is_schedule_active,
|
283
273
|
triggers=triggers,
|
284
274
|
parameters=parameters,
|
285
275
|
description=description,
|
prefect/runner/storage.py
CHANGED
@@ -280,6 +280,10 @@ class GitRepository:
|
|
280
280
|
"branch": self._branch,
|
281
281
|
}
|
282
282
|
}
|
283
|
+
if self._include_submodules:
|
284
|
+
pull_step["prefect.deployments.steps.git_clone"][
|
285
|
+
"include_submodules"
|
286
|
+
] = self._include_submodules
|
283
287
|
if isinstance(self._credentials, Block):
|
284
288
|
pull_step["prefect.deployments.steps.git_clone"][
|
285
289
|
"credentials"
|
prefect/runtime/flow_run.py
CHANGED
@@ -12,6 +12,7 @@ Available attributes:
|
|
12
12
|
- `scheduled_start_time`: the flow run's expected scheduled start time; defaults to now if not present
|
13
13
|
- `name`: the name of the flow run
|
14
14
|
- `flow_name`: the name of the flow
|
15
|
+
- `flow_version`: the version of the flow
|
15
16
|
- `parameters`: the parameters that were passed to this run; note that these do not necessarily
|
16
17
|
include default values set on the flow function, only the parameter values explicitly passed for the run
|
17
18
|
- `parent_flow_run_id`: the ID of the flow run that triggered this run, if any
|
@@ -35,6 +36,7 @@ __all__ = [
|
|
35
36
|
"scheduled_start_time",
|
36
37
|
"name",
|
37
38
|
"flow_name",
|
39
|
+
"flow_version",
|
38
40
|
"parameters",
|
39
41
|
"parent_flow_run_id",
|
40
42
|
"parent_deployment_id",
|
@@ -119,7 +121,7 @@ async def _get_flow_from_run(flow_run_id):
|
|
119
121
|
return await client.read_flow(flow_run.flow_id)
|
120
122
|
|
121
123
|
|
122
|
-
def get_id() -> str:
|
124
|
+
def get_id() -> Optional[str]:
|
123
125
|
flow_run_ctx = FlowRunContext.get()
|
124
126
|
task_run_ctx = TaskRunContext.get()
|
125
127
|
if flow_run_ctx is not None:
|
@@ -190,6 +192,21 @@ def get_flow_name() -> Optional[str]:
|
|
190
192
|
return flow_run_ctx.flow.name
|
191
193
|
|
192
194
|
|
195
|
+
def get_flow_version() -> Optional[str]:
|
196
|
+
flow_run_ctx = FlowRunContext.get()
|
197
|
+
run_id = get_id()
|
198
|
+
if flow_run_ctx is None and run_id is None:
|
199
|
+
return None
|
200
|
+
elif flow_run_ctx is None:
|
201
|
+
flow = from_sync.call_soon_in_loop_thread(
|
202
|
+
create_call(_get_flow_from_run, run_id)
|
203
|
+
).result()
|
204
|
+
|
205
|
+
return flow.version
|
206
|
+
else:
|
207
|
+
return flow_run_ctx.flow.version
|
208
|
+
|
209
|
+
|
193
210
|
def get_scheduled_start_time() -> pendulum.DateTime:
|
194
211
|
flow_run_ctx = FlowRunContext.get()
|
195
212
|
run_id = get_id()
|
@@ -313,4 +330,5 @@ FIELDS = {
|
|
313
330
|
"run_count": get_run_count,
|
314
331
|
"api_url": get_flow_run_api_url,
|
315
332
|
"ui_url": get_flow_run_ui_url,
|
333
|
+
"flow_version": get_flow_version,
|
316
334
|
}
|
prefect/settings.py
CHANGED
@@ -323,7 +323,11 @@ def template_with_settings(*upstream_settings: Setting) -> Callable[["Settings",
|
|
323
323
|
setting.name: setting.value_from(settings) for setting in upstream_settings
|
324
324
|
}
|
325
325
|
template = string.Template(str(value))
|
326
|
-
|
326
|
+
# Note the use of `safe_substitute` to avoid raising an exception if a
|
327
|
+
# template value is missing. In this case, template values will be left
|
328
|
+
# as-is in the string. Using `safe_substitute` prevents us raising when
|
329
|
+
# the DB password contains a `$` character.
|
330
|
+
return original_type(template.safe_substitute(template_values))
|
327
331
|
|
328
332
|
return templater
|
329
333
|
|
@@ -351,6 +355,7 @@ def warn_on_database_password_value_without_usage(values):
|
|
351
355
|
if (
|
352
356
|
value
|
353
357
|
and not value.startswith(OBFUSCATED_PREFIX)
|
358
|
+
and values["PREFECT_API_DATABASE_CONNECTION_URL"] is not None
|
354
359
|
and (
|
355
360
|
"PREFECT_API_DATABASE_PASSWORD"
|
356
361
|
not in values["PREFECT_API_DATABASE_CONNECTION_URL"]
|
@@ -403,7 +408,38 @@ def warn_on_misconfigured_api_url(values):
|
|
403
408
|
return values
|
404
409
|
|
405
410
|
|
406
|
-
def default_database_connection_url(settings, value):
|
411
|
+
def default_database_connection_url(settings: "Settings", value: Optional[str]):
|
412
|
+
driver = PREFECT_API_DATABASE_DRIVER.value_from(settings)
|
413
|
+
if driver == "postgresql+asyncpg":
|
414
|
+
required = [
|
415
|
+
PREFECT_API_DATABASE_HOST,
|
416
|
+
PREFECT_API_DATABASE_USER,
|
417
|
+
PREFECT_API_DATABASE_NAME,
|
418
|
+
PREFECT_API_DATABASE_PASSWORD,
|
419
|
+
]
|
420
|
+
missing = [
|
421
|
+
setting.name for setting in required if not setting.value_from(settings)
|
422
|
+
]
|
423
|
+
if missing:
|
424
|
+
raise ValueError(
|
425
|
+
f"Missing required database connection settings: {', '.join(missing)}"
|
426
|
+
)
|
427
|
+
|
428
|
+
host = PREFECT_API_DATABASE_HOST.value_from(settings)
|
429
|
+
port = PREFECT_API_DATABASE_PORT.value_from(settings) or 5432
|
430
|
+
user = PREFECT_API_DATABASE_USER.value_from(settings)
|
431
|
+
name = PREFECT_API_DATABASE_NAME.value_from(settings)
|
432
|
+
password = PREFECT_API_DATABASE_PASSWORD.value_from(settings)
|
433
|
+
|
434
|
+
return f"{driver}://{user}:{password}@{host}:{port}/{name}"
|
435
|
+
|
436
|
+
elif driver == "sqlite+aiosqlite":
|
437
|
+
path = PREFECT_API_DATABASE_NAME.value_from(settings)
|
438
|
+
if path:
|
439
|
+
return f"{driver}:///{path}"
|
440
|
+
elif driver:
|
441
|
+
raise ValueError(f"Unsupported database driver: {driver}")
|
442
|
+
|
407
443
|
templater = template_with_settings(PREFECT_HOME, PREFECT_API_DATABASE_PASSWORD)
|
408
444
|
|
409
445
|
# If the user has provided a value, use it
|
@@ -941,17 +977,6 @@ backend on application startup. If not set, block types must be manually
|
|
941
977
|
registered.
|
942
978
|
"""
|
943
979
|
|
944
|
-
PREFECT_API_DATABASE_PASSWORD = Setting(
|
945
|
-
Optional[str],
|
946
|
-
default=None,
|
947
|
-
is_secret=True,
|
948
|
-
)
|
949
|
-
"""
|
950
|
-
Password to template into the `PREFECT_API_DATABASE_CONNECTION_URL`.
|
951
|
-
This is useful if the password must be provided separately from the connection URL.
|
952
|
-
To use this setting, you must include it in your connection URL.
|
953
|
-
"""
|
954
|
-
|
955
980
|
PREFECT_API_DATABASE_CONNECTION_URL = Setting(
|
956
981
|
Optional[str],
|
957
982
|
default=None,
|
@@ -981,6 +1006,46 @@ PREFECT_API_DATABASE_CONNECTION_URL='postgresql+asyncpg://postgres:${PREFECT_API
|
|
981
1006
|
```
|
982
1007
|
"""
|
983
1008
|
|
1009
|
+
PREFECT_API_DATABASE_DRIVER = Setting(
|
1010
|
+
Optional[Literal["postgresql+asyncpg", "sqlite+aiosqlite"]],
|
1011
|
+
default=None,
|
1012
|
+
)
|
1013
|
+
"""
|
1014
|
+
The database driver to use when connecting to the database.
|
1015
|
+
"""
|
1016
|
+
|
1017
|
+
PREFECT_API_DATABASE_HOST = Setting(Optional[str], default=None)
|
1018
|
+
"""
|
1019
|
+
The database server host.
|
1020
|
+
"""
|
1021
|
+
|
1022
|
+
PREFECT_API_DATABASE_PORT = Setting(Optional[int], default=None)
|
1023
|
+
"""
|
1024
|
+
The database server port.
|
1025
|
+
"""
|
1026
|
+
|
1027
|
+
PREFECT_API_DATABASE_USER = Setting(Optional[str], default=None)
|
1028
|
+
"""
|
1029
|
+
The user to use when connecting to the database.
|
1030
|
+
"""
|
1031
|
+
|
1032
|
+
PREFECT_API_DATABASE_NAME = Setting(Optional[str], default=None)
|
1033
|
+
"""
|
1034
|
+
The name of the Prefect database on the remote server, or the path to the database file
|
1035
|
+
for SQLite.
|
1036
|
+
"""
|
1037
|
+
|
1038
|
+
PREFECT_API_DATABASE_PASSWORD = Setting(
|
1039
|
+
Optional[str],
|
1040
|
+
default=None,
|
1041
|
+
is_secret=True,
|
1042
|
+
)
|
1043
|
+
"""
|
1044
|
+
Password to template into the `PREFECT_API_DATABASE_CONNECTION_URL`.
|
1045
|
+
This is useful if the password must be provided separately from the connection URL.
|
1046
|
+
To use this setting, you must include it in your connection URL.
|
1047
|
+
"""
|
1048
|
+
|
984
1049
|
PREFECT_API_DATABASE_ECHO = Setting(
|
985
1050
|
bool,
|
986
1051
|
default=False,
|