prefect-client 3.4.6.dev2__py3-none-any.whl → 3.4.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/AGENTS.md +28 -0
- prefect/_build_info.py +3 -3
- prefect/_internal/websockets.py +109 -0
- prefect/artifacts.py +51 -2
- prefect/assets/core.py +2 -2
- prefect/blocks/core.py +82 -11
- prefect/client/cloud.py +11 -1
- prefect/client/orchestration/__init__.py +21 -15
- prefect/client/orchestration/_deployments/client.py +139 -4
- prefect/client/orchestration/_flows/client.py +4 -4
- prefect/client/schemas/__init__.py +5 -2
- prefect/client/schemas/actions.py +1 -0
- prefect/client/schemas/filters.py +3 -0
- prefect/client/schemas/objects.py +27 -10
- prefect/context.py +6 -4
- prefect/events/clients.py +2 -76
- prefect/events/schemas/automations.py +4 -0
- prefect/events/schemas/labelling.py +2 -0
- prefect/flow_engine.py +6 -3
- prefect/flows.py +64 -45
- prefect/futures.py +25 -4
- prefect/locking/filesystem.py +1 -1
- prefect/logging/clients.py +347 -0
- prefect/runner/runner.py +1 -1
- prefect/runner/submit.py +10 -4
- prefect/serializers.py +8 -3
- prefect/server/api/logs.py +64 -9
- prefect/server/api/server.py +2 -0
- prefect/server/api/templates.py +8 -2
- prefect/settings/context.py +17 -14
- prefect/settings/models/server/logs.py +28 -0
- prefect/settings/models/server/root.py +5 -0
- prefect/settings/models/server/services.py +26 -0
- prefect/task_engine.py +17 -17
- prefect/task_runners.py +10 -10
- prefect/tasks.py +52 -9
- prefect/types/__init__.py +2 -0
- prefect/types/names.py +50 -0
- prefect/utilities/_ast.py +2 -2
- prefect/utilities/callables.py +1 -1
- prefect/utilities/collections.py +6 -6
- prefect/utilities/engine.py +67 -72
- prefect/utilities/pydantic.py +19 -1
- prefect/workers/base.py +2 -0
- {prefect_client-3.4.6.dev2.dist-info → prefect_client-3.4.7.dist-info}/METADATA +1 -1
- {prefect_client-3.4.6.dev2.dist-info → prefect_client-3.4.7.dist-info}/RECORD +48 -44
- {prefect_client-3.4.6.dev2.dist-info → prefect_client-3.4.7.dist-info}/WHEEL +0 -0
- {prefect_client-3.4.6.dev2.dist-info → prefect_client-3.4.7.dist-info}/licenses/LICENSE +0 -0
prefect/server/api/logs.py
CHANGED
@@ -2,14 +2,21 @@
|
|
2
2
|
Routes for interacting with log objects.
|
3
3
|
"""
|
4
4
|
|
5
|
-
from typing import
|
5
|
+
from typing import Optional, Sequence
|
6
6
|
|
7
|
-
from fastapi import Body, Depends, status
|
7
|
+
from fastapi import Body, Depends, WebSocket, status
|
8
|
+
from pydantic import TypeAdapter
|
9
|
+
from starlette.status import WS_1002_PROTOCOL_ERROR
|
8
10
|
|
9
11
|
import prefect.server.api.dependencies as dependencies
|
10
12
|
import prefect.server.models as models
|
11
|
-
import prefect.server.schemas as schemas
|
12
13
|
from prefect.server.database import PrefectDBInterface, provide_database_interface
|
14
|
+
from prefect.server.logs import stream
|
15
|
+
from prefect.server.schemas.actions import LogCreate
|
16
|
+
from prefect.server.schemas.core import Log
|
17
|
+
from prefect.server.schemas.filters import LogFilter
|
18
|
+
from prefect.server.schemas.sorting import LogSort
|
19
|
+
from prefect.server.utilities import subscriptions
|
13
20
|
from prefect.server.utilities.server import PrefectRouter
|
14
21
|
|
15
22
|
router: PrefectRouter = PrefectRouter(prefix="/logs", tags=["Logs"])
|
@@ -17,7 +24,7 @@ router: PrefectRouter = PrefectRouter(prefix="/logs", tags=["Logs"])
|
|
17
24
|
|
18
25
|
@router.post("/", status_code=status.HTTP_201_CREATED)
|
19
26
|
async def create_logs(
|
20
|
-
logs:
|
27
|
+
logs: Sequence[LogCreate],
|
21
28
|
db: PrefectDBInterface = Depends(provide_database_interface),
|
22
29
|
) -> None:
|
23
30
|
"""
|
@@ -30,18 +37,66 @@ async def create_logs(
|
|
30
37
|
await models.logs.create_logs(session=session, logs=batch)
|
31
38
|
|
32
39
|
|
40
|
+
logs_adapter: TypeAdapter[Sequence[Log]] = TypeAdapter(Sequence[Log])
|
41
|
+
|
42
|
+
|
33
43
|
@router.post("/filter")
|
34
44
|
async def read_logs(
|
35
45
|
limit: int = dependencies.LimitBody(),
|
36
46
|
offset: int = Body(0, ge=0),
|
37
|
-
logs:
|
38
|
-
sort:
|
47
|
+
logs: Optional[LogFilter] = None,
|
48
|
+
sort: LogSort = Body(LogSort.TIMESTAMP_ASC),
|
39
49
|
db: PrefectDBInterface = Depends(provide_database_interface),
|
40
|
-
) ->
|
50
|
+
) -> Sequence[Log]:
|
41
51
|
"""
|
42
52
|
Query for logs.
|
43
53
|
"""
|
44
54
|
async with db.session_context() as session:
|
45
|
-
return
|
46
|
-
|
55
|
+
return logs_adapter.validate_python(
|
56
|
+
await models.logs.read_logs(
|
57
|
+
session=session, log_filter=logs, offset=offset, limit=limit, sort=sort
|
58
|
+
)
|
47
59
|
)
|
60
|
+
|
61
|
+
|
62
|
+
@router.websocket("/out")
|
63
|
+
async def stream_logs_out(websocket: WebSocket) -> None:
|
64
|
+
"""Serve a WebSocket to stream live logs"""
|
65
|
+
websocket = await subscriptions.accept_prefect_socket(websocket)
|
66
|
+
if not websocket:
|
67
|
+
return
|
68
|
+
|
69
|
+
try:
|
70
|
+
# After authentication, the next message is expected to be a filter message, any
|
71
|
+
# other type of message will close the connection.
|
72
|
+
message = await websocket.receive_json()
|
73
|
+
|
74
|
+
if message["type"] != "filter":
|
75
|
+
return await websocket.close(
|
76
|
+
WS_1002_PROTOCOL_ERROR, reason="Expected 'filter' message"
|
77
|
+
)
|
78
|
+
|
79
|
+
try:
|
80
|
+
filter = LogFilter.model_validate(message["filter"])
|
81
|
+
except Exception as e:
|
82
|
+
return await websocket.close(
|
83
|
+
WS_1002_PROTOCOL_ERROR, reason=f"Invalid filter: {e}"
|
84
|
+
)
|
85
|
+
|
86
|
+
# No backfill support for logs - only live streaming
|
87
|
+
# Subscribe to the ongoing log stream
|
88
|
+
async with stream.logs(filter) as log_stream:
|
89
|
+
async for log in log_stream:
|
90
|
+
if not log:
|
91
|
+
if await subscriptions.still_connected(websocket):
|
92
|
+
continue
|
93
|
+
break
|
94
|
+
|
95
|
+
await websocket.send_json(
|
96
|
+
{"type": "log", "log": log.model_dump(mode="json")}
|
97
|
+
)
|
98
|
+
|
99
|
+
except subscriptions.NORMAL_DISCONNECT_EXCEPTIONS: # pragma: no cover
|
100
|
+
pass # it's fine if a client disconnects either normally or abnormally
|
101
|
+
|
102
|
+
return None
|
prefect/server/api/server.py
CHANGED
@@ -922,6 +922,8 @@ class SubprocessASGIServer:
|
|
922
922
|
self.server_process.wait(timeout=5)
|
923
923
|
except subprocess.TimeoutExpired:
|
924
924
|
self.server_process.kill()
|
925
|
+
# Ensure the process is reaped to avoid ResourceWarning
|
926
|
+
self.server_process.wait()
|
925
927
|
finally:
|
926
928
|
self.server_process = None
|
927
929
|
if self.port in self._instances:
|
prefect/server/api/templates.py
CHANGED
@@ -8,11 +8,17 @@ from prefect.server.utilities.user_templates import (
|
|
8
8
|
validate_user_template,
|
9
9
|
)
|
10
10
|
|
11
|
-
router: PrefectRouter = PrefectRouter(
|
11
|
+
router: PrefectRouter = PrefectRouter(tags=["Automations"])
|
12
12
|
|
13
13
|
|
14
|
+
# deprecated and can be removed after the ui removes its dependency on it
|
15
|
+
# use /templates/validate instead
|
14
16
|
@router.post(
|
15
|
-
"/validate",
|
17
|
+
"/automations/templates/validate",
|
18
|
+
response_class=Response,
|
19
|
+
)
|
20
|
+
@router.post(
|
21
|
+
"/templates/validate",
|
16
22
|
response_class=Response,
|
17
23
|
)
|
18
24
|
def validate_template(template: str = Body(default="")) -> Response:
|
prefect/settings/context.py
CHANGED
@@ -33,20 +33,23 @@ def temporary_settings(
|
|
33
33
|
See `Settings.copy_with_update` for details on different argument behavior.
|
34
34
|
|
35
35
|
Examples:
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
36
|
+
|
37
|
+
```python
|
38
|
+
from prefect.settings import PREFECT_API_URL
|
39
|
+
|
40
|
+
with temporary_settings(updates={PREFECT_API_URL: "foo"}):
|
41
|
+
assert PREFECT_API_URL.value() == "foo"
|
42
|
+
|
43
|
+
with temporary_settings(set_defaults={PREFECT_API_URL: "bar"}):
|
44
|
+
assert PREFECT_API_URL.value() == "foo"
|
45
|
+
|
46
|
+
with temporary_settings(restore_defaults={PREFECT_API_URL}):
|
47
|
+
assert PREFECT_API_URL.value() is None
|
48
|
+
|
49
|
+
with temporary_settings(set_defaults={PREFECT_API_URL: "bar"})
|
50
|
+
assert PREFECT_API_URL.value() == "bar"
|
51
|
+
assert PREFECT_API_URL.value() is None
|
52
|
+
```
|
50
53
|
"""
|
51
54
|
import prefect.context
|
52
55
|
|
@@ -0,0 +1,28 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from typing import ClassVar
|
4
|
+
|
5
|
+
from pydantic import Field
|
6
|
+
from pydantic_settings import SettingsConfigDict
|
7
|
+
|
8
|
+
from prefect.settings.base import PrefectBaseSettings, build_settings_config
|
9
|
+
|
10
|
+
|
11
|
+
class ServerLogsSettings(PrefectBaseSettings):
|
12
|
+
"""
|
13
|
+
Settings for controlling behavior of the logs subsystem
|
14
|
+
"""
|
15
|
+
|
16
|
+
model_config: ClassVar[SettingsConfigDict] = build_settings_config(
|
17
|
+
("server", "logs")
|
18
|
+
)
|
19
|
+
|
20
|
+
stream_out_enabled: bool = Field(
|
21
|
+
default=False,
|
22
|
+
description="Whether or not to stream logs out to the API via websockets.",
|
23
|
+
)
|
24
|
+
|
25
|
+
stream_publishing_enabled: bool = Field(
|
26
|
+
default=False,
|
27
|
+
description="Whether or not to publish logs to the streaming system.",
|
28
|
+
)
|
@@ -13,6 +13,7 @@ from .deployments import ServerDeploymentsSettings
|
|
13
13
|
from .ephemeral import ServerEphemeralSettings
|
14
14
|
from .events import ServerEventsSettings
|
15
15
|
from .flow_run_graph import ServerFlowRunGraphSettings
|
16
|
+
from .logs import ServerLogsSettings
|
16
17
|
from .services import ServerServicesSettings
|
17
18
|
from .tasks import ServerTasksSettings
|
18
19
|
from .ui import ServerUISettings
|
@@ -127,6 +128,10 @@ class ServerSettings(PrefectBaseSettings):
|
|
127
128
|
default_factory=ServerFlowRunGraphSettings,
|
128
129
|
description="Settings for controlling flow run graph behavior",
|
129
130
|
)
|
131
|
+
logs: ServerLogsSettings = Field(
|
132
|
+
default_factory=ServerLogsSettings,
|
133
|
+
description="Settings for controlling server logs behavior",
|
134
|
+
)
|
130
135
|
services: ServerServicesSettings = Field(
|
131
136
|
default_factory=ServerServicesSettings,
|
132
137
|
description="Settings for controlling server services behavior",
|
@@ -448,6 +448,32 @@ class ServerServicesTriggersSettings(ServicesBaseSetting):
|
|
448
448
|
),
|
449
449
|
)
|
450
450
|
|
451
|
+
pg_notify_reconnect_interval_seconds: int = Field(
|
452
|
+
default=10,
|
453
|
+
description="""
|
454
|
+
The number of seconds to wait before reconnecting to the PostgreSQL NOTIFY/LISTEN
|
455
|
+
connection after an error. Only used when using PostgreSQL as the database.
|
456
|
+
Defaults to `10`.
|
457
|
+
""",
|
458
|
+
validation_alias=AliasChoices(
|
459
|
+
AliasPath("pg_notify_reconnect_interval_seconds"),
|
460
|
+
"prefect_server_services_triggers_pg_notify_reconnect_interval_seconds",
|
461
|
+
),
|
462
|
+
)
|
463
|
+
|
464
|
+
pg_notify_heartbeat_interval_seconds: int = Field(
|
465
|
+
default=5,
|
466
|
+
description="""
|
467
|
+
The number of seconds between heartbeat checks for the PostgreSQL NOTIFY/LISTEN
|
468
|
+
connection to ensure it's still alive. Only used when using PostgreSQL as the database.
|
469
|
+
Defaults to `5`.
|
470
|
+
""",
|
471
|
+
validation_alias=AliasChoices(
|
472
|
+
AliasPath("pg_notify_heartbeat_interval_seconds"),
|
473
|
+
"prefect_server_services_triggers_pg_notify_heartbeat_interval_seconds",
|
474
|
+
),
|
475
|
+
)
|
476
|
+
|
451
477
|
|
452
478
|
class ServerServicesSettings(PrefectBaseSettings):
|
453
479
|
"""
|
prefect/task_engine.py
CHANGED
@@ -37,7 +37,7 @@ import prefect.types._datetime
|
|
37
37
|
from prefect.cache_policies import CachePolicy
|
38
38
|
from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
|
39
39
|
from prefect.client.schemas import TaskRun
|
40
|
-
from prefect.client.schemas.objects import
|
40
|
+
from prefect.client.schemas.objects import RunInput, State
|
41
41
|
from prefect.concurrency.context import ConcurrencyContext
|
42
42
|
from prefect.concurrency.v1.asyncio import concurrency as aconcurrency
|
43
43
|
from prefect.concurrency.v1.context import ConcurrencyContext as ConcurrencyContextV1
|
@@ -96,7 +96,7 @@ from prefect.utilities.callables import call_with_parameters, parameters_to_args
|
|
96
96
|
from prefect.utilities.collections import visit_collection
|
97
97
|
from prefect.utilities.engine import (
|
98
98
|
emit_task_run_state_change_event,
|
99
|
-
|
99
|
+
link_state_to_task_run_result,
|
100
100
|
resolve_to_final_result,
|
101
101
|
)
|
102
102
|
from prefect.utilities.math import clamped_poisson_interval
|
@@ -453,7 +453,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
453
453
|
else:
|
454
454
|
result = state.data
|
455
455
|
|
456
|
-
|
456
|
+
link_state_to_task_run_result(new_state, result)
|
457
457
|
|
458
458
|
# emit a state change event
|
459
459
|
self._last_event = emit_task_run_state_change_event(
|
@@ -683,7 +683,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
683
683
|
def initialize_run(
|
684
684
|
self,
|
685
685
|
task_run_id: Optional[UUID] = None,
|
686
|
-
dependencies: Optional[dict[str, set[
|
686
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
687
687
|
) -> Generator[Self, Any, Any]:
|
688
688
|
"""
|
689
689
|
Enters a client context and creates a task run if needed.
|
@@ -777,7 +777,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
777
777
|
def start(
|
778
778
|
self,
|
779
779
|
task_run_id: Optional[UUID] = None,
|
780
|
-
dependencies: Optional[dict[str, set[
|
780
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
781
781
|
) -> Generator[None, None, None]:
|
782
782
|
with self.initialize_run(task_run_id=task_run_id, dependencies=dependencies):
|
783
783
|
with (
|
@@ -1038,7 +1038,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1038
1038
|
else:
|
1039
1039
|
result = new_state.data
|
1040
1040
|
|
1041
|
-
|
1041
|
+
link_state_to_task_run_result(new_state, result)
|
1042
1042
|
|
1043
1043
|
# emit a state change event
|
1044
1044
|
self._last_event = emit_task_run_state_change_event(
|
@@ -1267,7 +1267,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1267
1267
|
async def initialize_run(
|
1268
1268
|
self,
|
1269
1269
|
task_run_id: Optional[UUID] = None,
|
1270
|
-
dependencies: Optional[dict[str, set[
|
1270
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1271
1271
|
) -> AsyncGenerator[Self, Any]:
|
1272
1272
|
"""
|
1273
1273
|
Enters a client context and creates a task run if needed.
|
@@ -1359,7 +1359,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1359
1359
|
async def start(
|
1360
1360
|
self,
|
1361
1361
|
task_run_id: Optional[UUID] = None,
|
1362
|
-
dependencies: Optional[dict[str, set[
|
1362
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1363
1363
|
) -> AsyncGenerator[None, None]:
|
1364
1364
|
async with self.initialize_run(
|
1365
1365
|
task_run_id=task_run_id, dependencies=dependencies
|
@@ -1465,7 +1465,7 @@ def run_task_sync(
|
|
1465
1465
|
parameters: Optional[dict[str, Any]] = None,
|
1466
1466
|
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1467
1467
|
return_type: Literal["state", "result"] = "result",
|
1468
|
-
dependencies: Optional[dict[str, set[
|
1468
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1469
1469
|
context: Optional[dict[str, Any]] = None,
|
1470
1470
|
) -> Union[R, State, None]:
|
1471
1471
|
engine = SyncTaskRunEngine[P, R](
|
@@ -1496,7 +1496,7 @@ async def run_task_async(
|
|
1496
1496
|
parameters: Optional[dict[str, Any]] = None,
|
1497
1497
|
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1498
1498
|
return_type: Literal["state", "result"] = "result",
|
1499
|
-
dependencies: Optional[dict[str, set[
|
1499
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1500
1500
|
context: Optional[dict[str, Any]] = None,
|
1501
1501
|
) -> Union[R, State, None]:
|
1502
1502
|
engine = AsyncTaskRunEngine[P, R](
|
@@ -1527,7 +1527,7 @@ def run_generator_task_sync(
|
|
1527
1527
|
parameters: Optional[dict[str, Any]] = None,
|
1528
1528
|
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1529
1529
|
return_type: Literal["state", "result"] = "result",
|
1530
|
-
dependencies: Optional[dict[str, set[
|
1530
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1531
1531
|
context: Optional[dict[str, Any]] = None,
|
1532
1532
|
) -> Generator[R, None, None]:
|
1533
1533
|
if return_type != "result":
|
@@ -1568,7 +1568,7 @@ def run_generator_task_sync(
|
|
1568
1568
|
# dictionary in an unbounded way, so finding a
|
1569
1569
|
# way to periodically clean it up (using
|
1570
1570
|
# weakrefs or similar) would be good
|
1571
|
-
|
1571
|
+
link_state_to_task_run_result(engine.state, gen_result)
|
1572
1572
|
yield gen_result
|
1573
1573
|
except StopIteration as exc:
|
1574
1574
|
engine.handle_success(exc.value, transaction=txn)
|
@@ -1586,7 +1586,7 @@ async def run_generator_task_async(
|
|
1586
1586
|
parameters: Optional[dict[str, Any]] = None,
|
1587
1587
|
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1588
1588
|
return_type: Literal["state", "result"] = "result",
|
1589
|
-
dependencies: Optional[dict[str, set[
|
1589
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1590
1590
|
context: Optional[dict[str, Any]] = None,
|
1591
1591
|
) -> AsyncGenerator[R, None]:
|
1592
1592
|
if return_type != "result":
|
@@ -1627,7 +1627,7 @@ async def run_generator_task_async(
|
|
1627
1627
|
# dictionary in an unbounded way, so finding a
|
1628
1628
|
# way to periodically clean it up (using
|
1629
1629
|
# weakrefs or similar) would be good
|
1630
|
-
|
1630
|
+
link_state_to_task_run_result(engine.state, gen_result)
|
1631
1631
|
yield gen_result
|
1632
1632
|
except (StopAsyncIteration, GeneratorExit) as exc:
|
1633
1633
|
await engine.handle_success(None, transaction=txn)
|
@@ -1647,7 +1647,7 @@ def run_task(
|
|
1647
1647
|
parameters: Optional[dict[str, Any]] = None,
|
1648
1648
|
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1649
1649
|
return_type: Literal["state"] = "state",
|
1650
|
-
dependencies: Optional[dict[str, set[
|
1650
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1651
1651
|
context: Optional[dict[str, Any]] = None,
|
1652
1652
|
) -> State[R]: ...
|
1653
1653
|
|
@@ -1660,7 +1660,7 @@ def run_task(
|
|
1660
1660
|
parameters: Optional[dict[str, Any]] = None,
|
1661
1661
|
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1662
1662
|
return_type: Literal["result"] = "result",
|
1663
|
-
dependencies: Optional[dict[str, set[
|
1663
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1664
1664
|
context: Optional[dict[str, Any]] = None,
|
1665
1665
|
) -> R: ...
|
1666
1666
|
|
@@ -1672,7 +1672,7 @@ def run_task(
|
|
1672
1672
|
parameters: Optional[dict[str, Any]] = None,
|
1673
1673
|
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1674
1674
|
return_type: Literal["state", "result"] = "result",
|
1675
|
-
dependencies: Optional[dict[str, set[
|
1675
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1676
1676
|
context: Optional[dict[str, Any]] = None,
|
1677
1677
|
) -> Union[R, State, None, Coroutine[Any, Any, Union[R, State, None]]]:
|
1678
1678
|
"""
|
prefect/task_runners.py
CHANGED
@@ -19,7 +19,7 @@ from typing import (
|
|
19
19
|
from typing_extensions import ParamSpec, Self, TypeVar
|
20
20
|
|
21
21
|
from prefect._internal.uuid7 import uuid7
|
22
|
-
from prefect.client.schemas.objects import
|
22
|
+
from prefect.client.schemas.objects import RunInput
|
23
23
|
from prefect.exceptions import MappingLengthMismatch, MappingMissingIterable
|
24
24
|
from prefect.futures import (
|
25
25
|
PrefectConcurrentFuture,
|
@@ -81,7 +81,7 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
81
81
|
task: "Task[P, Coroutine[Any, Any, R]]",
|
82
82
|
parameters: dict[str, Any],
|
83
83
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
84
|
-
dependencies: dict[str, set[
|
84
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
85
85
|
) -> F: ...
|
86
86
|
|
87
87
|
@overload
|
@@ -91,7 +91,7 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
91
91
|
task: "Task[Any, R]",
|
92
92
|
parameters: dict[str, Any],
|
93
93
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
94
|
-
dependencies: dict[str, set[
|
94
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
95
95
|
) -> F: ...
|
96
96
|
|
97
97
|
@abc.abstractmethod
|
@@ -100,7 +100,7 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
100
100
|
task: "Task[P, R]",
|
101
101
|
parameters: dict[str, Any],
|
102
102
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
103
|
-
dependencies: dict[str, set[
|
103
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
104
104
|
) -> F: ...
|
105
105
|
|
106
106
|
def map(
|
@@ -262,7 +262,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture[R]]):
|
|
262
262
|
task: "Task[P, Coroutine[Any, Any, R]]",
|
263
263
|
parameters: dict[str, Any],
|
264
264
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
265
|
-
dependencies: dict[str, set[
|
265
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
266
266
|
) -> PrefectConcurrentFuture[R]: ...
|
267
267
|
|
268
268
|
@overload
|
@@ -271,7 +271,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture[R]]):
|
|
271
271
|
task: "Task[Any, R]",
|
272
272
|
parameters: dict[str, Any],
|
273
273
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
274
|
-
dependencies: dict[str, set[
|
274
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
275
275
|
) -> PrefectConcurrentFuture[R]: ...
|
276
276
|
|
277
277
|
def submit(
|
@@ -279,7 +279,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture[R]]):
|
|
279
279
|
task: "Task[P, R | Coroutine[Any, Any, R]]",
|
280
280
|
parameters: dict[str, Any],
|
281
281
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
282
|
-
dependencies: dict[str, set[
|
282
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
283
283
|
) -> PrefectConcurrentFuture[R]:
|
284
284
|
"""
|
285
285
|
Submit a task to the task run engine running in a separate thread.
|
@@ -415,7 +415,7 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture[R]]):
|
|
415
415
|
task: "Task[P, Coroutine[Any, Any, R]]",
|
416
416
|
parameters: dict[str, Any],
|
417
417
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
418
|
-
dependencies: dict[str, set[
|
418
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
419
419
|
) -> PrefectDistributedFuture[R]: ...
|
420
420
|
|
421
421
|
@overload
|
@@ -424,7 +424,7 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture[R]]):
|
|
424
424
|
task: "Task[Any, R]",
|
425
425
|
parameters: dict[str, Any],
|
426
426
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
427
|
-
dependencies: dict[str, set[
|
427
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
428
428
|
) -> PrefectDistributedFuture[R]: ...
|
429
429
|
|
430
430
|
def submit(
|
@@ -432,7 +432,7 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture[R]]):
|
|
432
432
|
task: "Task[P, R]",
|
433
433
|
parameters: dict[str, Any],
|
434
434
|
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
435
|
-
dependencies: dict[str, set[
|
435
|
+
dependencies: dict[str, set[RunInput]] | None = None,
|
436
436
|
) -> PrefectDistributedFuture[R]:
|
437
437
|
"""
|
438
438
|
Submit a task to the task run engine running in a separate thread.
|
prefect/tasks.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
"""
|
2
|
-
Module containing the base workflow task class and decorator - for most use cases, using the
|
2
|
+
Module containing the base workflow task class and decorator - for most use cases, using the `@task` decorator is preferred.
|
3
3
|
"""
|
4
4
|
|
5
5
|
# This file requires type-checking with pyright because mypy does not yet support PEP612
|
@@ -47,8 +47,8 @@ from prefect.cache_policies import DEFAULT, NO_CACHE, CachePolicy
|
|
47
47
|
from prefect.client.orchestration import get_client
|
48
48
|
from prefect.client.schemas import TaskRun
|
49
49
|
from prefect.client.schemas.objects import (
|
50
|
+
RunInput,
|
50
51
|
StateDetails,
|
51
|
-
TaskRunInput,
|
52
52
|
TaskRunPolicy,
|
53
53
|
TaskRunResult,
|
54
54
|
)
|
@@ -244,12 +244,17 @@ def _infer_parent_task_runs(
|
|
244
244
|
# tracked within the same flow run.
|
245
245
|
if flow_run_context:
|
246
246
|
for v in parameters.values():
|
247
|
+
upstream_state = None
|
248
|
+
|
247
249
|
if isinstance(v, State):
|
248
250
|
upstream_state = v
|
249
251
|
elif isinstance(v, PrefectFuture):
|
250
252
|
upstream_state = v.state
|
251
253
|
else:
|
252
|
-
|
254
|
+
res = flow_run_context.run_results.get(id(v))
|
255
|
+
if res:
|
256
|
+
upstream_state, _ = res
|
257
|
+
|
253
258
|
if upstream_state and upstream_state.is_running():
|
254
259
|
parents.append(
|
255
260
|
TaskRunResult(id=upstream_state.state_details.task_run_id)
|
@@ -296,9 +301,6 @@ class Task(Generic[P, R]):
|
|
296
301
|
"""
|
297
302
|
A Prefect task definition.
|
298
303
|
|
299
|
-
!!! note
|
300
|
-
We recommend using [the `@task` decorator][prefect.tasks.task] for most use-cases.
|
301
|
-
|
302
304
|
Wraps a function with an entrypoint to the Prefect engine. Calling this class within a flow function
|
303
305
|
creates a new task run.
|
304
306
|
|
@@ -840,7 +842,7 @@ class Task(Generic[P, R]):
|
|
840
842
|
flow_run_context: Optional[FlowRunContext] = None,
|
841
843
|
parent_task_run_context: Optional[TaskRunContext] = None,
|
842
844
|
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
843
|
-
extra_task_inputs: Optional[dict[str, set[
|
845
|
+
extra_task_inputs: Optional[dict[str, set[RunInput]]] = None,
|
844
846
|
deferred: bool = False,
|
845
847
|
) -> TaskRun:
|
846
848
|
from prefect.utilities._engine import dynamic_key_for_task_run
|
@@ -943,7 +945,7 @@ class Task(Generic[P, R]):
|
|
943
945
|
flow_run_context: Optional[FlowRunContext] = None,
|
944
946
|
parent_task_run_context: Optional[TaskRunContext] = None,
|
945
947
|
wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
|
946
|
-
extra_task_inputs: Optional[dict[str, set[
|
948
|
+
extra_task_inputs: Optional[dict[str, set[RunInput]]] = None,
|
947
949
|
deferred: bool = False,
|
948
950
|
) -> TaskRun:
|
949
951
|
from prefect.utilities._engine import dynamic_key_for_task_run
|
@@ -1530,7 +1532,7 @@ class Task(Generic[P, R]):
|
|
1530
1532
|
args: Optional[tuple[Any, ...]] = None,
|
1531
1533
|
kwargs: Optional[dict[str, Any]] = None,
|
1532
1534
|
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
1533
|
-
dependencies: Optional[dict[str, set[
|
1535
|
+
dependencies: Optional[dict[str, set[RunInput]]] = None,
|
1534
1536
|
) -> PrefectDistributedFuture[R]:
|
1535
1537
|
"""
|
1536
1538
|
Create a pending task run for a task worker to execute.
|
@@ -2033,3 +2035,44 @@ class MaterializingTask(Task[P, R]):
|
|
2033
2035
|
Asset(key=a) if isinstance(a, str) else a for a in assets
|
2034
2036
|
]
|
2035
2037
|
self.materialized_by = materialized_by
|
2038
|
+
|
2039
|
+
def with_options(
|
2040
|
+
self,
|
2041
|
+
assets: Optional[Sequence[Union[str, Asset]]] = None,
|
2042
|
+
**task_kwargs: Unpack[TaskOptions],
|
2043
|
+
) -> "MaterializingTask[P, R]":
|
2044
|
+
import inspect
|
2045
|
+
|
2046
|
+
sig = inspect.signature(Task.__init__)
|
2047
|
+
|
2048
|
+
# Map parameter names to attribute names where they differ
|
2049
|
+
# from parameter to attribute.
|
2050
|
+
param_to_attr = {
|
2051
|
+
"on_completion": "on_completion_hooks",
|
2052
|
+
"on_failure": "on_failure_hooks",
|
2053
|
+
"on_rollback": "on_rollback_hooks",
|
2054
|
+
"on_commit": "on_commit_hooks",
|
2055
|
+
}
|
2056
|
+
|
2057
|
+
# Build kwargs for Task constructor
|
2058
|
+
init_kwargs = {}
|
2059
|
+
for param_name in sig.parameters:
|
2060
|
+
if param_name in ("self", "fn", "assets", "materialized_by"):
|
2061
|
+
continue
|
2062
|
+
|
2063
|
+
attr_name = param_to_attr.get(param_name, param_name)
|
2064
|
+
init_kwargs[param_name] = task_kwargs.get(
|
2065
|
+
param_name, getattr(self, attr_name)
|
2066
|
+
)
|
2067
|
+
|
2068
|
+
return MaterializingTask(
|
2069
|
+
fn=self.fn,
|
2070
|
+
assets=(
|
2071
|
+
[Asset(key=a) if isinstance(a, str) else a for a in assets]
|
2072
|
+
if assets is not None
|
2073
|
+
else self.assets
|
2074
|
+
),
|
2075
|
+
materialized_by=self.materialized_by,
|
2076
|
+
# Now, the rest
|
2077
|
+
**init_kwargs,
|
2078
|
+
)
|
prefect/types/__init__.py
CHANGED
@@ -15,6 +15,7 @@ from .names import (
|
|
15
15
|
WITHOUT_BANNED_CHARACTERS,
|
16
16
|
MAX_VARIABLE_NAME_LENGTH,
|
17
17
|
URILike,
|
18
|
+
ValidAssetKey,
|
18
19
|
)
|
19
20
|
from pydantic import (
|
20
21
|
BeforeValidator,
|
@@ -216,6 +217,7 @@ __all__ = [
|
|
216
217
|
"Name",
|
217
218
|
"NameOrEmpty",
|
218
219
|
"NonEmptyishName",
|
220
|
+
"ValidAssetKey",
|
219
221
|
"SecretDict",
|
220
222
|
"StatusCode",
|
221
223
|
"StrictVariableValue",
|