prefect-client 3.0.0rc8__py3-none-any.whl → 3.0.0rc10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/compatibility/deprecated.py +53 -0
- prefect/_internal/compatibility/migration.py +53 -11
- prefect/_internal/integrations.py +7 -0
- prefect/agent.py +6 -0
- prefect/blocks/core.py +1 -1
- prefect/client/__init__.py +4 -0
- prefect/client/schemas/objects.py +6 -3
- prefect/client/utilities.py +4 -4
- prefect/context.py +6 -0
- prefect/deployments/schedules.py +5 -2
- prefect/deployments/steps/core.py +6 -0
- prefect/engine.py +4 -4
- prefect/events/schemas/automations.py +3 -3
- prefect/exceptions.py +4 -1
- prefect/filesystems.py +4 -3
- prefect/flow_engine.py +102 -15
- prefect/flow_runs.py +1 -1
- prefect/flows.py +65 -15
- prefect/futures.py +5 -0
- prefect/infrastructure/__init__.py +6 -0
- prefect/infrastructure/base.py +6 -0
- prefect/logging/loggers.py +1 -1
- prefect/results.py +85 -68
- prefect/serializers.py +3 -3
- prefect/settings.py +7 -33
- prefect/task_engine.py +78 -21
- prefect/task_runners.py +28 -16
- prefect/task_worker.py +19 -6
- prefect/tasks.py +39 -7
- prefect/transactions.py +41 -3
- prefect/utilities/asyncutils.py +37 -8
- prefect/utilities/collections.py +1 -1
- prefect/utilities/importtools.py +1 -1
- prefect/utilities/timeout.py +20 -5
- prefect/workers/block.py +6 -0
- prefect/workers/cloud.py +6 -0
- {prefect_client-3.0.0rc8.dist-info → prefect_client-3.0.0rc10.dist-info}/METADATA +3 -2
- {prefect_client-3.0.0rc8.dist-info → prefect_client-3.0.0rc10.dist-info}/RECORD +41 -36
- {prefect_client-3.0.0rc8.dist-info → prefect_client-3.0.0rc10.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc8.dist-info → prefect_client-3.0.0rc10.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc8.dist-info → prefect_client-3.0.0rc10.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -53,8 +53,6 @@ from prefect.client.schemas.objects import Flow as FlowSchema
|
|
53
53
|
from prefect.client.schemas.objects import FlowRun
|
54
54
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
55
55
|
from prefect.client.utilities import client_injector
|
56
|
-
from prefect.deployments.runner import deploy
|
57
|
-
from prefect.deployments.steps.core import run_steps
|
58
56
|
from prefect.docker.docker_image import DockerImage
|
59
57
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
60
58
|
from prefect.exceptions import (
|
@@ -69,12 +67,6 @@ from prefect.futures import PrefectFuture
|
|
69
67
|
from prefect.logging import get_logger
|
70
68
|
from prefect.logging.loggers import flow_run_logger
|
71
69
|
from prefect.results import ResultSerializer, ResultStorage
|
72
|
-
from prefect.runner.storage import (
|
73
|
-
BlockStorageAdapter,
|
74
|
-
LocalStorage,
|
75
|
-
RunnerStorage,
|
76
|
-
create_storage_from_source,
|
77
|
-
)
|
78
70
|
from prefect.settings import (
|
79
71
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
80
72
|
PREFECT_FLOW_DEFAULT_RETRIES,
|
@@ -120,6 +112,7 @@ if TYPE_CHECKING:
|
|
120
112
|
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
121
113
|
from prefect.deployments.runner import RunnerDeployment
|
122
114
|
from prefect.flows import FlowRun
|
115
|
+
from prefect.runner.storage import RunnerStorage
|
123
116
|
|
124
117
|
|
125
118
|
class Flow(Generic[P, R]):
|
@@ -194,7 +187,7 @@ class Flow(Generic[P, R]):
|
|
194
187
|
timeout_seconds: Union[int, float, None] = None,
|
195
188
|
validate_parameters: bool = True,
|
196
189
|
persist_result: Optional[bool] = None,
|
197
|
-
result_storage: Optional[ResultStorage] = None,
|
190
|
+
result_storage: Optional[Union[ResultStorage, str]] = None,
|
198
191
|
result_serializer: Optional[ResultSerializer] = None,
|
199
192
|
cache_result_in_memory: bool = True,
|
200
193
|
log_prints: Optional[bool] = None,
|
@@ -342,7 +335,18 @@ class Flow(Generic[P, R]):
|
|
342
335
|
"Disable validation or change the argument names."
|
343
336
|
) from exc
|
344
337
|
|
338
|
+
# result persistence settings
|
339
|
+
if persist_result is None:
|
340
|
+
if result_storage is not None or result_serializer is not None:
|
341
|
+
persist_result = True
|
342
|
+
|
345
343
|
self.persist_result = persist_result
|
344
|
+
if result_storage and not isinstance(result_storage, str):
|
345
|
+
if getattr(result_storage, "_block_document_id", None) is None:
|
346
|
+
raise TypeError(
|
347
|
+
"Result storage configuration must be persisted server-side."
|
348
|
+
" Please call `.save()` on your block before passing it in."
|
349
|
+
)
|
346
350
|
self.result_storage = result_storage
|
347
351
|
self.result_serializer = result_serializer
|
348
352
|
self.cache_result_in_memory = cache_result_in_memory
|
@@ -353,7 +357,7 @@ class Flow(Generic[P, R]):
|
|
353
357
|
self.on_running_hooks = on_running or []
|
354
358
|
|
355
359
|
# Used for flows loaded from remote storage
|
356
|
-
self._storage: Optional[RunnerStorage] = None
|
360
|
+
self._storage: Optional["RunnerStorage"] = None
|
357
361
|
self._entrypoint: Optional[str] = None
|
358
362
|
|
359
363
|
module = fn.__module__
|
@@ -919,7 +923,7 @@ class Flow(Generic[P, R]):
|
|
919
923
|
@sync_compatible
|
920
924
|
async def from_source(
|
921
925
|
cls: Type[F],
|
922
|
-
source: Union[str, RunnerStorage, ReadableDeploymentStorage],
|
926
|
+
source: Union[str, "RunnerStorage", ReadableDeploymentStorage],
|
923
927
|
entrypoint: str,
|
924
928
|
) -> F:
|
925
929
|
"""
|
@@ -967,7 +971,38 @@ class Flow(Generic[P, R]):
|
|
967
971
|
|
968
972
|
my_flow()
|
969
973
|
```
|
974
|
+
|
975
|
+
Load a flow from a local directory:
|
976
|
+
|
977
|
+
``` python
|
978
|
+
# from_local_source.py
|
979
|
+
|
980
|
+
from pathlib import Path
|
981
|
+
from prefect import flow
|
982
|
+
|
983
|
+
@flow(log_prints=True)
|
984
|
+
def my_flow(name: str = "world"):
|
985
|
+
print(f"Hello {name}! I'm a flow from a Python script!")
|
986
|
+
|
987
|
+
if __name__ == "__main__":
|
988
|
+
my_flow.from_source(
|
989
|
+
source=str(Path(__file__).parent),
|
990
|
+
entrypoint="from_local_source.py:my_flow",
|
991
|
+
).deploy(
|
992
|
+
name="my-deployment",
|
993
|
+
parameters=dict(name="Marvin"),
|
994
|
+
work_pool_name="local",
|
995
|
+
)
|
996
|
+
```
|
970
997
|
"""
|
998
|
+
|
999
|
+
from prefect.runner.storage import (
|
1000
|
+
BlockStorageAdapter,
|
1001
|
+
LocalStorage,
|
1002
|
+
RunnerStorage,
|
1003
|
+
create_storage_from_source,
|
1004
|
+
)
|
1005
|
+
|
971
1006
|
if isinstance(source, str):
|
972
1007
|
storage = create_storage_from_source(source)
|
973
1008
|
elif isinstance(source, RunnerStorage):
|
@@ -987,7 +1022,7 @@ class Flow(Generic[P, R]):
|
|
987
1022
|
await storage.pull_code()
|
988
1023
|
|
989
1024
|
full_entrypoint = str(storage.destination / entrypoint)
|
990
|
-
flow:
|
1025
|
+
flow: Flow = await from_async.wait_for_call_in_new_thread(
|
991
1026
|
create_call(load_flow_from_entrypoint, full_entrypoint)
|
992
1027
|
)
|
993
1028
|
flow._storage = storage
|
@@ -1114,7 +1149,13 @@ class Flow(Generic[P, R]):
|
|
1114
1149
|
)
|
1115
1150
|
```
|
1116
1151
|
"""
|
1117
|
-
|
1152
|
+
if not (
|
1153
|
+
work_pool_name := work_pool_name or PREFECT_DEFAULT_WORK_POOL_NAME.value()
|
1154
|
+
):
|
1155
|
+
raise ValueError(
|
1156
|
+
"No work pool name provided. Please provide a `work_pool_name` or set the"
|
1157
|
+
" `PREFECT_DEFAULT_WORK_POOL_NAME` environment variable."
|
1158
|
+
)
|
1118
1159
|
|
1119
1160
|
try:
|
1120
1161
|
async with get_client() as client:
|
@@ -1145,7 +1186,9 @@ class Flow(Generic[P, R]):
|
|
1145
1186
|
entrypoint_type=entrypoint_type,
|
1146
1187
|
)
|
1147
1188
|
|
1148
|
-
|
1189
|
+
from prefect.deployments import runner
|
1190
|
+
|
1191
|
+
deployment_ids = await runner.deploy(
|
1149
1192
|
deployment,
|
1150
1193
|
work_pool_name=work_pool_name,
|
1151
1194
|
image=image,
|
@@ -1833,6 +1876,9 @@ async def load_flow_from_flow_run(
|
|
1833
1876
|
run_logger.debug(
|
1834
1877
|
f"Running {len(deployment.pull_steps)} deployment pull step(s)"
|
1835
1878
|
)
|
1879
|
+
|
1880
|
+
from prefect.deployments.steps.core import run_steps
|
1881
|
+
|
1836
1882
|
output = await run_steps(deployment.pull_steps)
|
1837
1883
|
if output.get("directory"):
|
1838
1884
|
run_logger.debug(f"Changing working directory to {output['directory']!r}")
|
@@ -1908,8 +1954,12 @@ def load_flow_argument_from_entrypoint(
|
|
1908
1954
|
literal_arg_value = ast.get_source_segment(
|
1909
1955
|
source_code, keyword.value
|
1910
1956
|
)
|
1957
|
+
cleaned_value = (
|
1958
|
+
literal_arg_value.replace("\n", "") if literal_arg_value else ""
|
1959
|
+
)
|
1960
|
+
|
1911
1961
|
try:
|
1912
|
-
evaluated_value = eval(
|
1962
|
+
evaluated_value = eval(cleaned_value, namespace) # type: ignore
|
1913
1963
|
except Exception as e:
|
1914
1964
|
logger.info(
|
1915
1965
|
"Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
|
prefect/futures.py
CHANGED
@@ -8,6 +8,7 @@ from typing import Any, Generic, List, Optional, Set, Union, cast
|
|
8
8
|
|
9
9
|
from typing_extensions import TypeVar
|
10
10
|
|
11
|
+
from prefect._internal.compatibility.deprecated import deprecated_async_method
|
11
12
|
from prefect.client.orchestration import get_client
|
12
13
|
from prefect.client.schemas.objects import TaskRun
|
13
14
|
from prefect.exceptions import ObjectNotFound
|
@@ -111,6 +112,7 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
|
|
111
112
|
when the task run is submitted to a ThreadPoolExecutor.
|
112
113
|
"""
|
113
114
|
|
115
|
+
@deprecated_async_method
|
114
116
|
def wait(self, timeout: Optional[float] = None) -> None:
|
115
117
|
try:
|
116
118
|
result = self._wrapped_future.result(timeout=timeout)
|
@@ -119,6 +121,7 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
|
|
119
121
|
if isinstance(result, State):
|
120
122
|
self._final_state = result
|
121
123
|
|
124
|
+
@deprecated_async_method
|
122
125
|
def result(
|
123
126
|
self,
|
124
127
|
timeout: Optional[float] = None,
|
@@ -168,6 +171,7 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
168
171
|
any task run scheduled in Prefect's API.
|
169
172
|
"""
|
170
173
|
|
174
|
+
@deprecated_async_method
|
171
175
|
def wait(self, timeout: Optional[float] = None) -> None:
|
172
176
|
return run_coro_as_sync(self.wait_async(timeout=timeout))
|
173
177
|
|
@@ -204,6 +208,7 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
204
208
|
self._final_state = task_run.state
|
205
209
|
return
|
206
210
|
|
211
|
+
@deprecated_async_method
|
207
212
|
def result(
|
208
213
|
self,
|
209
214
|
timeout: Optional[float] = None,
|
prefect/logging/loggers.py
CHANGED
@@ -115,7 +115,7 @@ def get_run_logger(
|
|
115
115
|
addition to the run metadata
|
116
116
|
|
117
117
|
Raises:
|
118
|
-
|
118
|
+
MissingContextError: If no context can be found
|
119
119
|
"""
|
120
120
|
# Check for existing contexts
|
121
121
|
task_run_context = prefect.context.TaskRunContext.get()
|
prefect/results.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
import abc
|
2
|
+
import inspect
|
2
3
|
import uuid
|
3
4
|
from functools import partial
|
4
5
|
from typing import (
|
@@ -27,7 +28,6 @@ from prefect.client.utilities import inject_client
|
|
27
28
|
from prefect.exceptions import MissingResult, ObjectAlreadyExists
|
28
29
|
from prefect.filesystems import (
|
29
30
|
LocalFileSystem,
|
30
|
-
ReadableFileSystem,
|
31
31
|
WritableFileSystem,
|
32
32
|
)
|
33
33
|
from prefect.logging import get_logger
|
@@ -110,22 +110,32 @@ async def _get_or_create_default_storage(block_document_slug: str) -> ResultStor
|
|
110
110
|
|
111
111
|
|
112
112
|
@sync_compatible
|
113
|
-
async def
|
113
|
+
async def get_default_result_storage() -> ResultStorage:
|
114
114
|
"""
|
115
115
|
Generate a default file system for result storage.
|
116
116
|
"""
|
117
|
-
|
118
|
-
|
119
|
-
|
117
|
+
default_block = PREFECT_DEFAULT_RESULT_STORAGE_BLOCK.value()
|
118
|
+
|
119
|
+
if default_block is not None:
|
120
|
+
return await Block.load(default_block)
|
121
|
+
|
122
|
+
# otherwise, use the local file system
|
123
|
+
basepath = PREFECT_LOCAL_STORAGE_PATH.value()
|
124
|
+
return LocalFileSystem(basepath=basepath)
|
120
125
|
|
121
126
|
|
122
127
|
async def get_or_create_default_task_scheduling_storage() -> ResultStorage:
|
123
128
|
"""
|
124
129
|
Generate a default file system for background task parameter/result storage.
|
125
130
|
"""
|
126
|
-
|
127
|
-
|
128
|
-
|
131
|
+
default_block = PREFECT_TASK_SCHEDULING_DEFAULT_STORAGE_BLOCK.value()
|
132
|
+
|
133
|
+
if default_block is not None:
|
134
|
+
return await Block.load(default_block)
|
135
|
+
|
136
|
+
# otherwise, use the local file system
|
137
|
+
basepath = PREFECT_LOCAL_STORAGE_PATH.value()
|
138
|
+
return LocalFileSystem(basepath=basepath)
|
129
139
|
|
130
140
|
|
131
141
|
def get_default_result_serializer() -> ResultSerializer:
|
@@ -176,9 +186,7 @@ class ResultFactory(BaseModel):
|
|
176
186
|
kwargs.pop(key)
|
177
187
|
|
178
188
|
# Apply defaults
|
179
|
-
kwargs.setdefault(
|
180
|
-
"result_storage", await get_or_create_default_result_storage()
|
181
|
-
)
|
189
|
+
kwargs.setdefault("result_storage", await get_default_result_storage())
|
182
190
|
kwargs.setdefault("result_serializer", get_default_result_serializer())
|
183
191
|
kwargs.setdefault("persist_result", get_default_persist_setting())
|
184
192
|
kwargs.setdefault("cache_result_in_memory", True)
|
@@ -229,9 +237,7 @@ class ResultFactory(BaseModel):
|
|
229
237
|
"""
|
230
238
|
Create a new result factory for a task.
|
231
239
|
"""
|
232
|
-
return await cls._from_task(
|
233
|
-
task, get_or_create_default_result_storage, client=client
|
234
|
-
)
|
240
|
+
return await cls._from_task(task, get_default_result_storage, client=client)
|
235
241
|
|
236
242
|
@classmethod
|
237
243
|
@inject_client
|
@@ -267,7 +273,14 @@ class ResultFactory(BaseModel):
|
|
267
273
|
if ctx and ctx.result_factory
|
268
274
|
else get_default_result_serializer()
|
269
275
|
)
|
270
|
-
|
276
|
+
if task.persist_result is None:
|
277
|
+
persist_result = (
|
278
|
+
ctx.result_factory.persist_result
|
279
|
+
if ctx and ctx.result_factory
|
280
|
+
else get_default_persist_setting()
|
281
|
+
)
|
282
|
+
else:
|
283
|
+
persist_result = task.persist_result
|
271
284
|
|
272
285
|
cache_result_in_memory = task.cache_result_in_memory
|
273
286
|
|
@@ -329,16 +342,7 @@ class ResultFactory(BaseModel):
|
|
329
342
|
# Avoid saving the block if it already has an identifier assigned
|
330
343
|
storage_block_id = storage_block._block_document_id
|
331
344
|
else:
|
332
|
-
|
333
|
-
# TODO: Overwrite is true to avoid issues where the save collides with
|
334
|
-
# a previously saved document with a matching hash
|
335
|
-
storage_block_id = await storage_block._save(
|
336
|
-
is_anonymous=True, overwrite=True, client=client
|
337
|
-
)
|
338
|
-
else:
|
339
|
-
# a None-type UUID on unpersisted storage should not matter
|
340
|
-
# since the ID is generated on the server
|
341
|
-
storage_block_id = None
|
345
|
+
storage_block_id = None
|
342
346
|
elif isinstance(result_storage, str):
|
343
347
|
storage_block = await Block.load(result_storage, client=client)
|
344
348
|
storage_block_id = storage_block._block_document_id
|
@@ -411,9 +415,6 @@ class ResultFactory(BaseModel):
|
|
411
415
|
|
412
416
|
@sync_compatible
|
413
417
|
async def store_parameters(self, identifier: UUID, parameters: Dict[str, Any]):
|
414
|
-
assert (
|
415
|
-
self.storage_block_id is not None
|
416
|
-
), "Unexpected storage block ID. Was it persisted?"
|
417
418
|
data = self.serializer.dumps(parameters)
|
418
419
|
blob = PersistedResultBlob(serializer=self.serializer, data=data)
|
419
420
|
await self.storage_block.write_path(
|
@@ -422,9 +423,6 @@ class ResultFactory(BaseModel):
|
|
422
423
|
|
423
424
|
@sync_compatible
|
424
425
|
async def read_parameters(self, identifier: UUID) -> Dict[str, Any]:
|
425
|
-
assert (
|
426
|
-
self.storage_block_id is not None
|
427
|
-
), "Unexpected storage block ID. Was it persisted?"
|
428
426
|
blob = PersistedResultBlob.model_validate_json(
|
429
427
|
await self.storage_block.read_path(f"parameters/{identifier}")
|
430
428
|
)
|
@@ -434,10 +432,7 @@ class ResultFactory(BaseModel):
|
|
434
432
|
@register_base_type
|
435
433
|
class BaseResult(BaseModel, abc.ABC, Generic[R]):
|
436
434
|
model_config = ConfigDict(extra="forbid")
|
437
|
-
|
438
435
|
type: str
|
439
|
-
artifact_type: Optional[str] = None
|
440
|
-
artifact_description: Optional[str] = None
|
441
436
|
|
442
437
|
def __init__(self, **data: Any) -> None:
|
443
438
|
type_string = get_dispatch_key(self) if type(self) != BaseResult else "__base__"
|
@@ -503,11 +498,7 @@ class UnpersistedResult(BaseResult):
|
|
503
498
|
obj: R,
|
504
499
|
cache_object: bool = True,
|
505
500
|
) -> "UnpersistedResult[R]":
|
506
|
-
|
507
|
-
result = cls(
|
508
|
-
artifact_type="result",
|
509
|
-
artifact_description=description,
|
510
|
-
)
|
501
|
+
result = cls()
|
511
502
|
# Only store the object in local memory, it will not be sent to the API
|
512
503
|
if cache_object:
|
513
504
|
result._cache_object(obj)
|
@@ -527,8 +518,8 @@ class PersistedResult(BaseResult):
|
|
527
518
|
type: str = "reference"
|
528
519
|
|
529
520
|
serializer_type: str
|
530
|
-
storage_block_id: uuid.UUID
|
531
521
|
storage_key: str
|
522
|
+
storage_block_id: Optional[uuid.UUID] = None
|
532
523
|
expiration: Optional[DateTime] = None
|
533
524
|
|
534
525
|
_should_cache_object: bool = PrivateAttr(default=True)
|
@@ -546,6 +537,17 @@ class PersistedResult(BaseResult):
|
|
546
537
|
self._storage_block = storage_block
|
547
538
|
self._serializer = serializer
|
548
539
|
|
540
|
+
@inject_client
|
541
|
+
async def _get_storage_block(self, client: "PrefectClient") -> WritableFileSystem:
|
542
|
+
if self._storage_block is not None:
|
543
|
+
return self._storage_block
|
544
|
+
elif self.storage_block_id is not None:
|
545
|
+
block_document = await client.read_block_document(self.storage_block_id)
|
546
|
+
self._storage_block = Block._from_block_document(block_document)
|
547
|
+
else:
|
548
|
+
self._storage_block = await get_default_result_storage()
|
549
|
+
return self._storage_block
|
550
|
+
|
549
551
|
@sync_compatible
|
550
552
|
@inject_client
|
551
553
|
async def get(self, client: "PrefectClient") -> R:
|
@@ -566,12 +568,8 @@ class PersistedResult(BaseResult):
|
|
566
568
|
|
567
569
|
@inject_client
|
568
570
|
async def _read_blob(self, client: "PrefectClient") -> "PersistedResultBlob":
|
569
|
-
|
570
|
-
|
571
|
-
), "Unexpected storage block ID. Was it persisted?"
|
572
|
-
block_document = await client.read_block_document(self.storage_block_id)
|
573
|
-
storage_block: ReadableFileSystem = Block._from_block_document(block_document)
|
574
|
-
content = await storage_block.read_path(self.storage_key)
|
571
|
+
block = await self._get_storage_block(client=client)
|
572
|
+
content = await block.read_path(self.storage_key)
|
575
573
|
blob = PersistedResultBlob.model_validate_json(content)
|
576
574
|
return blob
|
577
575
|
|
@@ -606,10 +604,7 @@ class PersistedResult(BaseResult):
|
|
606
604
|
obj = obj if obj is not NotSet else self._cache
|
607
605
|
|
608
606
|
# next, the storage block
|
609
|
-
storage_block = self.
|
610
|
-
if storage_block is None:
|
611
|
-
block_document = await client.read_block_document(self.storage_block_id)
|
612
|
-
storage_block = Block._from_block_document(block_document)
|
607
|
+
storage_block = await self._get_storage_block(client=client)
|
613
608
|
|
614
609
|
# finally, the serializer
|
615
610
|
serializer = self._serializer
|
@@ -620,9 +615,42 @@ class PersistedResult(BaseResult):
|
|
620
615
|
try:
|
621
616
|
data = serializer.dumps(obj)
|
622
617
|
except Exception as exc:
|
618
|
+
extra_info = (
|
619
|
+
'You can try a different serializer (e.g. result_serializer="json") '
|
620
|
+
"or disabling persistence (persist_result=False) for this flow or task."
|
621
|
+
)
|
622
|
+
# check if this is a known issue with cloudpickle and pydantic
|
623
|
+
# and add extra information to help the user recover
|
624
|
+
|
625
|
+
if (
|
626
|
+
isinstance(exc, TypeError)
|
627
|
+
and isinstance(obj, BaseModel)
|
628
|
+
and str(exc).startswith("cannot pickle")
|
629
|
+
):
|
630
|
+
try:
|
631
|
+
from IPython import get_ipython
|
632
|
+
|
633
|
+
if get_ipython() is not None:
|
634
|
+
extra_info = inspect.cleandoc(
|
635
|
+
"""
|
636
|
+
This is a known issue in Pydantic that prevents
|
637
|
+
locally-defined (non-imported) models from being
|
638
|
+
serialized by cloudpickle in IPython/Jupyter
|
639
|
+
environments. Please see
|
640
|
+
https://github.com/pydantic/pydantic/issues/8232 for
|
641
|
+
more information. To fix the issue, either: (1) move
|
642
|
+
your Pydantic class definition to an importable
|
643
|
+
location, (2) use the JSON serializer for your flow
|
644
|
+
or task (`result_serializer="json"`), or (3)
|
645
|
+
disable result persistence for your flow or task
|
646
|
+
(`persist_result=False`).
|
647
|
+
"""
|
648
|
+
).replace("\n", " ")
|
649
|
+
except ImportError:
|
650
|
+
pass
|
623
651
|
raise ValueError(
|
624
652
|
f"Failed to serialize object of type {type(obj).__name__!r} with "
|
625
|
-
f"serializer {serializer.type!r}."
|
653
|
+
f"serializer {serializer.type!r}. {extra_info}"
|
626
654
|
) from exc
|
627
655
|
blob = PersistedResultBlob(
|
628
656
|
serializer=serializer, data=data, expiration=self.expiration
|
@@ -639,9 +667,9 @@ class PersistedResult(BaseResult):
|
|
639
667
|
cls: "Type[PersistedResult]",
|
640
668
|
obj: R,
|
641
669
|
storage_block: WritableFileSystem,
|
642
|
-
storage_block_id: uuid.UUID,
|
643
670
|
storage_key_fn: Callable[[], str],
|
644
671
|
serializer: Serializer,
|
672
|
+
storage_block_id: Optional[uuid.UUID] = None,
|
645
673
|
cache_object: bool = True,
|
646
674
|
expiration: Optional[DateTime] = None,
|
647
675
|
defer_persistence: bool = False,
|
@@ -652,31 +680,21 @@ class PersistedResult(BaseResult):
|
|
652
680
|
The object will be serialized and written to the storage block under a unique
|
653
681
|
key. It will then be cached on the returned result.
|
654
682
|
"""
|
655
|
-
assert (
|
656
|
-
storage_block_id is not None
|
657
|
-
), "Unexpected storage block ID. Was it saved?"
|
658
|
-
|
659
683
|
key = storage_key_fn()
|
660
684
|
if not isinstance(key, str):
|
661
685
|
raise TypeError(
|
662
686
|
f"Expected type 'str' for result storage key; got value {key!r}"
|
663
687
|
)
|
664
|
-
description = f"Result of type `{type(obj).__name__}`"
|
665
688
|
uri = cls._infer_path(storage_block, key)
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
description += f" persisted to [{uri}]({uri})."
|
671
|
-
else:
|
672
|
-
description += f" persisted with storage block `{storage_block_id}`."
|
689
|
+
|
690
|
+
# in this case we store an absolute path
|
691
|
+
if storage_block_id is None and uri is not None:
|
692
|
+
key = str(uri)
|
673
693
|
|
674
694
|
result = cls(
|
675
695
|
serializer_type=serializer.type,
|
676
696
|
storage_block_id=storage_block_id,
|
677
697
|
storage_key=key,
|
678
|
-
artifact_type="result",
|
679
|
-
artifact_description=description,
|
680
698
|
expiration=expiration,
|
681
699
|
)
|
682
700
|
|
@@ -753,5 +771,4 @@ class UnknownResult(BaseResult):
|
|
753
771
|
"Only None is supported."
|
754
772
|
)
|
755
773
|
|
756
|
-
|
757
|
-
return cls(value=obj, artifact_type="result", artifact_description=description)
|
774
|
+
return cls(value=obj)
|
prefect/serializers.py
CHANGED
@@ -13,7 +13,7 @@ bytes to an object respectively.
|
|
13
13
|
|
14
14
|
import abc
|
15
15
|
import base64
|
16
|
-
from typing import Any, Dict, Generic, Optional, Type
|
16
|
+
from typing import Any, Dict, Generic, Optional, Type
|
17
17
|
|
18
18
|
from pydantic import (
|
19
19
|
BaseModel,
|
@@ -23,7 +23,7 @@ from pydantic import (
|
|
23
23
|
ValidationError,
|
24
24
|
field_validator,
|
25
25
|
)
|
26
|
-
from typing_extensions import Literal, Self
|
26
|
+
from typing_extensions import Literal, Self, TypeVar
|
27
27
|
|
28
28
|
from prefect._internal.schemas.validators import (
|
29
29
|
cast_type_names_to_serializers,
|
@@ -36,7 +36,7 @@ from prefect.utilities.dispatch import get_dispatch_key, lookup_type, register_b
|
|
36
36
|
from prefect.utilities.importtools import from_qualified_name, to_qualified_name
|
37
37
|
from prefect.utilities.pydantic import custom_pydantic_encoder
|
38
38
|
|
39
|
-
D = TypeVar("D")
|
39
|
+
D = TypeVar("D", default=Any)
|
40
40
|
|
41
41
|
|
42
42
|
def prefect_json_object_encoder(obj: Any) -> Any:
|
prefect/settings.py
CHANGED
@@ -42,7 +42,7 @@ dependent on the value of other settings or perform other dynamic effects.
|
|
42
42
|
|
43
43
|
import logging
|
44
44
|
import os
|
45
|
-
import
|
45
|
+
import re
|
46
46
|
import string
|
47
47
|
import warnings
|
48
48
|
from contextlib import contextmanager
|
@@ -85,7 +85,6 @@ from prefect._internal.schemas.validators import validate_settings
|
|
85
85
|
from prefect.exceptions import MissingProfileError
|
86
86
|
from prefect.utilities.names import OBFUSCATED_PREFIX, obfuscate
|
87
87
|
from prefect.utilities.pydantic import add_cloudpickle_reduction
|
88
|
-
from prefect.utilities.slugify import slugify
|
89
88
|
|
90
89
|
T = TypeVar("T")
|
91
90
|
|
@@ -404,18 +403,6 @@ def warn_on_misconfigured_api_url(values):
|
|
404
403
|
return values
|
405
404
|
|
406
405
|
|
407
|
-
def default_result_storage_block_name(
|
408
|
-
settings: Optional["Settings"] = None, value: Optional[str] = None
|
409
|
-
):
|
410
|
-
"""
|
411
|
-
`value_callback` for `PREFECT_DEFAULT_RESULT_STORAGE_BLOCK` that sets the default
|
412
|
-
value to the hostname of the machine.
|
413
|
-
"""
|
414
|
-
if value is None:
|
415
|
-
return f"local-file-system/{slugify(socket.gethostname())}-storage"
|
416
|
-
return value
|
417
|
-
|
418
|
-
|
419
406
|
def default_database_connection_url(settings, value):
|
420
407
|
templater = template_with_settings(PREFECT_HOME, PREFECT_API_DATABASE_PASSWORD)
|
421
408
|
|
@@ -474,10 +461,8 @@ def default_cloud_ui_url(settings, value):
|
|
474
461
|
# Otherwise, infer a value from the API URL
|
475
462
|
ui_url = api_url = PREFECT_CLOUD_API_URL.value_from(settings)
|
476
463
|
|
477
|
-
if
|
478
|
-
ui_url = ui_url.replace(
|
479
|
-
"https://api.prefect.cloud", "https://app.prefect.cloud", 1
|
480
|
-
)
|
464
|
+
if re.match(r"^https://api[\.\w]*.prefect.[^\.]+/", api_url):
|
465
|
+
ui_url = ui_url.replace("https://api", "https://app", 1)
|
481
466
|
|
482
467
|
if ui_url.endswith("/api"):
|
483
468
|
ui_url = ui_url[:-4]
|
@@ -734,7 +719,7 @@ PREFECT_RESULTS_DEFAULT_SERIALIZER = Setting(
|
|
734
719
|
|
735
720
|
PREFECT_RESULTS_PERSIST_BY_DEFAULT = Setting(
|
736
721
|
bool,
|
737
|
-
default=
|
722
|
+
default=False,
|
738
723
|
)
|
739
724
|
"""
|
740
725
|
The default setting for persisting results when not otherwise specified. If enabled,
|
@@ -1323,15 +1308,6 @@ PREFECT_API_MAX_FLOW_RUN_GRAPH_ARTIFACTS = Setting(int, default=10000)
|
|
1323
1308
|
The maximum number of artifacts to show on a flow run graph on the v2 API
|
1324
1309
|
"""
|
1325
1310
|
|
1326
|
-
PREFECT_EXPERIMENTAL_ENABLE_WORKERS = Setting(bool, default=True)
|
1327
|
-
"""
|
1328
|
-
Whether or not to enable experimental Prefect workers.
|
1329
|
-
"""
|
1330
|
-
|
1331
|
-
PREFECT_EXPERIMENTAL_WARN_WORKERS = Setting(bool, default=False)
|
1332
|
-
"""
|
1333
|
-
Whether or not to warn when experimental Prefect workers are used.
|
1334
|
-
"""
|
1335
1311
|
|
1336
1312
|
PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_CANCELLATION = Setting(bool, default=True)
|
1337
1313
|
"""
|
@@ -1423,10 +1399,7 @@ PREFECT_API_SERVICES_TASK_SCHEDULING_ENABLED = Setting(bool, default=True)
|
|
1423
1399
|
Whether or not to start the task scheduling service in the server application.
|
1424
1400
|
"""
|
1425
1401
|
|
1426
|
-
PREFECT_TASK_SCHEDULING_DEFAULT_STORAGE_BLOCK = Setting(
|
1427
|
-
str,
|
1428
|
-
default="local-file-system/prefect-task-scheduling",
|
1429
|
-
)
|
1402
|
+
PREFECT_TASK_SCHEDULING_DEFAULT_STORAGE_BLOCK = Setting(Optional[str], default=None)
|
1430
1403
|
"""The `block-type/block-document` slug of a block to use as the default storage
|
1431
1404
|
for autonomous tasks."""
|
1432
1405
|
|
@@ -1479,7 +1452,8 @@ PREFECT_EXPERIMENTAL_ENABLE_SCHEDULE_CONCURRENCY = Setting(bool, default=False)
|
|
1479
1452
|
# Defaults -----------------------------------------------------------------------------
|
1480
1453
|
|
1481
1454
|
PREFECT_DEFAULT_RESULT_STORAGE_BLOCK = Setting(
|
1482
|
-
Optional[str],
|
1455
|
+
Optional[str],
|
1456
|
+
default=None,
|
1483
1457
|
)
|
1484
1458
|
"""The `block-type/block-document` slug of a block to use as the default result storage."""
|
1485
1459
|
|