prefect-client 3.0.0rc10__py3-none-any.whl → 3.0.0rc12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/concurrency/api.py +1 -1
- prefect/_internal/concurrency/services.py +9 -0
- prefect/_internal/retries.py +61 -0
- prefect/artifacts.py +12 -0
- prefect/client/cloud.py +1 -1
- prefect/client/schemas/actions.py +4 -0
- prefect/client/schemas/objects.py +1 -1
- prefect/concurrency/asyncio.py +3 -3
- prefect/concurrency/events.py +1 -1
- prefect/concurrency/services.py +3 -2
- prefect/concurrency/sync.py +19 -5
- prefect/context.py +8 -2
- prefect/deployments/__init__.py +28 -15
- prefect/deployments/steps/pull.py +7 -0
- prefect/events/schemas/events.py +10 -0
- prefect/flow_engine.py +10 -9
- prefect/flows.py +194 -68
- prefect/futures.py +53 -7
- prefect/logging/loggers.py +1 -1
- prefect/results.py +1 -46
- prefect/runner/runner.py +96 -23
- prefect/runner/server.py +20 -22
- prefect/runner/submit.py +0 -8
- prefect/runtime/flow_run.py +38 -3
- prefect/settings.py +9 -30
- prefect/task_engine.py +158 -48
- prefect/task_worker.py +1 -1
- prefect/tasks.py +164 -17
- prefect/transactions.py +2 -15
- prefect/utilities/asyncutils.py +13 -9
- prefect/utilities/engine.py +34 -1
- prefect/workers/base.py +98 -208
- prefect/workers/process.py +262 -4
- prefect/workers/server.py +27 -9
- {prefect_client-3.0.0rc10.dist-info → prefect_client-3.0.0rc12.dist-info}/METADATA +4 -4
- {prefect_client-3.0.0rc10.dist-info → prefect_client-3.0.0rc12.dist-info}/RECORD +39 -38
- {prefect_client-3.0.0rc10.dist-info → prefect_client-3.0.0rc12.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc10.dist-info → prefect_client-3.0.0rc12.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc10.dist-info → prefect_client-3.0.0rc12.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -5,6 +5,7 @@ Module containing the base workflow class and decorator - for most use cases, us
|
|
5
5
|
# This file requires type-checking with pyright because mypy does not yet support PEP612
|
6
6
|
# See https://github.com/python/mypy/issues/8645
|
7
7
|
import ast
|
8
|
+
import asyncio
|
8
9
|
import datetime
|
9
10
|
import importlib.util
|
10
11
|
import inspect
|
@@ -28,6 +29,8 @@ from typing import (
|
|
28
29
|
List,
|
29
30
|
NoReturn,
|
30
31
|
Optional,
|
32
|
+
Set,
|
33
|
+
Tuple,
|
31
34
|
Type,
|
32
35
|
TypeVar,
|
33
36
|
Union,
|
@@ -44,7 +47,9 @@ from pydantic.v1.errors import ConfigError # TODO
|
|
44
47
|
from rich.console import Console
|
45
48
|
from typing_extensions import Literal, ParamSpec, Self
|
46
49
|
|
47
|
-
from prefect._internal.compatibility.deprecated import
|
50
|
+
from prefect._internal.compatibility.deprecated import (
|
51
|
+
deprecated_parameter,
|
52
|
+
)
|
48
53
|
from prefect._internal.concurrency.api import create_call, from_async
|
49
54
|
from prefect.blocks.core import Block
|
50
55
|
from prefect.client.orchestration import get_client
|
@@ -60,6 +65,7 @@ from prefect.exceptions import (
|
|
60
65
|
MissingFlowError,
|
61
66
|
ObjectNotFound,
|
62
67
|
ParameterTypeError,
|
68
|
+
ScriptError,
|
63
69
|
UnspecifiedFlowError,
|
64
70
|
)
|
65
71
|
from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
|
@@ -778,8 +784,7 @@ class Flow(Generic[P, R]):
|
|
778
784
|
self.on_failure_hooks.append(fn)
|
779
785
|
return fn
|
780
786
|
|
781
|
-
|
782
|
-
async def serve(
|
787
|
+
def serve(
|
783
788
|
self,
|
784
789
|
name: Optional[str] = None,
|
785
790
|
interval: Optional[
|
@@ -793,7 +798,7 @@ class Flow(Generic[P, R]):
|
|
793
798
|
cron: Optional[Union[Iterable[str], str]] = None,
|
794
799
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
795
800
|
paused: Optional[bool] = None,
|
796
|
-
schedules: Optional[
|
801
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
797
802
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
798
803
|
is_schedule_active: Optional[bool] = None,
|
799
804
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
@@ -884,7 +889,7 @@ class Flow(Generic[P, R]):
|
|
884
889
|
name = Path(name).stem
|
885
890
|
|
886
891
|
runner = Runner(name=name, pause_on_shutdown=pause_on_shutdown, limit=limit)
|
887
|
-
deployment_id =
|
892
|
+
deployment_id = runner.add_flow(
|
888
893
|
self,
|
889
894
|
name=name,
|
890
895
|
triggers=triggers,
|
@@ -917,15 +922,27 @@ class Flow(Generic[P, R]):
|
|
917
922
|
|
918
923
|
console = Console()
|
919
924
|
console.print(help_message, soft_wrap=True)
|
920
|
-
|
925
|
+
|
926
|
+
try:
|
927
|
+
loop = asyncio.get_running_loop()
|
928
|
+
except RuntimeError as exc:
|
929
|
+
if "no running event loop" in str(exc):
|
930
|
+
loop = None
|
931
|
+
else:
|
932
|
+
raise
|
933
|
+
|
934
|
+
if loop is not None:
|
935
|
+
loop.run_until_complete(runner.start(webserver=webserver))
|
936
|
+
else:
|
937
|
+
asyncio.run(runner.start(webserver=webserver))
|
921
938
|
|
922
939
|
@classmethod
|
923
940
|
@sync_compatible
|
924
941
|
async def from_source(
|
925
|
-
cls: Type[
|
942
|
+
cls: Type["Flow[P, R]"],
|
926
943
|
source: Union[str, "RunnerStorage", ReadableDeploymentStorage],
|
927
944
|
entrypoint: str,
|
928
|
-
) ->
|
945
|
+
) -> "Flow[P, R]":
|
929
946
|
"""
|
930
947
|
Loads a flow from a remote source.
|
931
948
|
|
@@ -1003,7 +1020,9 @@ class Flow(Generic[P, R]):
|
|
1003
1020
|
create_storage_from_source,
|
1004
1021
|
)
|
1005
1022
|
|
1006
|
-
if isinstance(source, str):
|
1023
|
+
if isinstance(source, (Path, str)):
|
1024
|
+
if isinstance(source, Path):
|
1025
|
+
source = str(source)
|
1007
1026
|
storage = create_storage_from_source(source)
|
1008
1027
|
elif isinstance(source, RunnerStorage):
|
1009
1028
|
storage = source
|
@@ -1186,9 +1205,9 @@ class Flow(Generic[P, R]):
|
|
1186
1205
|
entrypoint_type=entrypoint_type,
|
1187
1206
|
)
|
1188
1207
|
|
1189
|
-
from prefect.deployments import
|
1208
|
+
from prefect.deployments.runner import deploy
|
1190
1209
|
|
1191
|
-
deployment_ids = await
|
1210
|
+
deployment_ids = await deploy(
|
1192
1211
|
deployment,
|
1193
1212
|
work_pool_name=work_pool_name,
|
1194
1213
|
image=image,
|
@@ -1685,6 +1704,7 @@ def select_flow(
|
|
1685
1704
|
|
1686
1705
|
def load_flow_from_entrypoint(
|
1687
1706
|
entrypoint: str,
|
1707
|
+
use_placeholder_flow: bool = True,
|
1688
1708
|
) -> Flow:
|
1689
1709
|
"""
|
1690
1710
|
Extract a flow object from a script at an entrypoint by running all of the code in the file.
|
@@ -1692,6 +1712,8 @@ def load_flow_from_entrypoint(
|
|
1692
1712
|
Args:
|
1693
1713
|
entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
|
1694
1714
|
to a flow function
|
1715
|
+
use_placeholder_flow: if True, use a placeholder Flow object if the actual flow object
|
1716
|
+
cannot be loaded from the entrypoint (e.g. dependencies are missing)
|
1695
1717
|
|
1696
1718
|
Returns:
|
1697
1719
|
The flow object from the script
|
@@ -1712,6 +1734,16 @@ def load_flow_from_entrypoint(
|
|
1712
1734
|
raise MissingFlowError(
|
1713
1735
|
f"Flow function with name {func_name!r} not found in {path!r}. "
|
1714
1736
|
) from exc
|
1737
|
+
except ScriptError as exc:
|
1738
|
+
# If the flow has dependencies that are not installed in the current
|
1739
|
+
# environment, fallback to loading the flow via AST parsing. The
|
1740
|
+
# drawback of this approach is that we're unable to actually load the
|
1741
|
+
# function, so we create a placeholder flow that will re-raise this
|
1742
|
+
# exception when called.
|
1743
|
+
if use_placeholder_flow:
|
1744
|
+
flow = load_placeholder_flow(entrypoint=entrypoint, raises=exc)
|
1745
|
+
else:
|
1746
|
+
raise
|
1715
1747
|
|
1716
1748
|
if not isinstance(flow, Flow):
|
1717
1749
|
raise MissingFlowError(
|
@@ -1722,14 +1754,13 @@ def load_flow_from_entrypoint(
|
|
1722
1754
|
return flow
|
1723
1755
|
|
1724
1756
|
|
1725
|
-
|
1726
|
-
async def serve(
|
1757
|
+
def serve(
|
1727
1758
|
*args: "RunnerDeployment",
|
1728
1759
|
pause_on_shutdown: bool = True,
|
1729
1760
|
print_starting_message: bool = True,
|
1730
1761
|
limit: Optional[int] = None,
|
1731
1762
|
**kwargs,
|
1732
|
-
)
|
1763
|
+
):
|
1733
1764
|
"""
|
1734
1765
|
Serve the provided list of deployments.
|
1735
1766
|
|
@@ -1779,7 +1810,7 @@ async def serve(
|
|
1779
1810
|
|
1780
1811
|
runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
|
1781
1812
|
for deployment in args:
|
1782
|
-
|
1813
|
+
runner.add_deployment(deployment)
|
1783
1814
|
|
1784
1815
|
if print_starting_message:
|
1785
1816
|
help_message_top = (
|
@@ -1810,7 +1841,18 @@ async def serve(
|
|
1810
1841
|
Group(help_message_top, table, help_message_bottom), soft_wrap=True
|
1811
1842
|
)
|
1812
1843
|
|
1813
|
-
|
1844
|
+
try:
|
1845
|
+
loop = asyncio.get_running_loop()
|
1846
|
+
except RuntimeError as exc:
|
1847
|
+
if "no running event loop" in str(exc):
|
1848
|
+
loop = None
|
1849
|
+
else:
|
1850
|
+
raise
|
1851
|
+
|
1852
|
+
if loop is not None:
|
1853
|
+
loop.run_until_complete(runner.start())
|
1854
|
+
else:
|
1855
|
+
asyncio.run(runner.start())
|
1814
1856
|
|
1815
1857
|
|
1816
1858
|
@client_injector
|
@@ -1819,6 +1861,7 @@ async def load_flow_from_flow_run(
|
|
1819
1861
|
flow_run: "FlowRun",
|
1820
1862
|
ignore_storage: bool = False,
|
1821
1863
|
storage_base_path: Optional[str] = None,
|
1864
|
+
use_placeholder_flow: bool = True,
|
1822
1865
|
) -> Flow:
|
1823
1866
|
"""
|
1824
1867
|
Load a flow from the location/script provided in a deployment's storage document.
|
@@ -1845,7 +1888,9 @@ async def load_flow_from_flow_run(
|
|
1845
1888
|
f"Importing flow code from module path {deployment.entrypoint}"
|
1846
1889
|
)
|
1847
1890
|
flow = await run_sync_in_worker_thread(
|
1848
|
-
load_flow_from_entrypoint,
|
1891
|
+
load_flow_from_entrypoint,
|
1892
|
+
deployment.entrypoint,
|
1893
|
+
use_placeholder_flow=use_placeholder_flow,
|
1849
1894
|
)
|
1850
1895
|
return flow
|
1851
1896
|
|
@@ -1887,29 +1932,147 @@ async def load_flow_from_flow_run(
|
|
1887
1932
|
import_path = relative_path_to_current_platform(deployment.entrypoint)
|
1888
1933
|
run_logger.debug(f"Importing flow code from '{import_path}'")
|
1889
1934
|
|
1890
|
-
flow = await run_sync_in_worker_thread(
|
1935
|
+
flow = await run_sync_in_worker_thread(
|
1936
|
+
load_flow_from_entrypoint,
|
1937
|
+
str(import_path),
|
1938
|
+
use_placeholder_flow=use_placeholder_flow,
|
1939
|
+
)
|
1891
1940
|
|
1892
1941
|
return flow
|
1893
1942
|
|
1894
1943
|
|
1895
|
-
def
|
1896
|
-
entrypoint: str, arg: str = "name"
|
1897
|
-
) -> Optional[str]:
|
1944
|
+
def load_placeholder_flow(entrypoint: str, raises: Exception):
|
1898
1945
|
"""
|
1899
|
-
|
1946
|
+
Load a placeholder flow that is initialized with the same arguments as the
|
1947
|
+
flow specified in the entrypoint. If called the flow will raise `raises`.
|
1900
1948
|
|
1901
|
-
|
1902
|
-
|
1949
|
+
This is useful when a flow can't be loaded due to missing dependencies or
|
1950
|
+
other issues but the base metadata defining the flow is still needed.
|
1903
1951
|
|
1904
1952
|
Args:
|
1905
|
-
entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
|
1906
|
-
|
1953
|
+
entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
|
1954
|
+
or a module path to a flow function
|
1955
|
+
raises: an exception to raise when the flow is called
|
1956
|
+
"""
|
1957
|
+
|
1958
|
+
def _base_placeholder():
|
1959
|
+
raise raises
|
1960
|
+
|
1961
|
+
def sync_placeholder_flow(*args, **kwargs):
|
1962
|
+
_base_placeholder()
|
1963
|
+
|
1964
|
+
async def async_placeholder_flow(*args, **kwargs):
|
1965
|
+
_base_placeholder()
|
1966
|
+
|
1967
|
+
placeholder_flow = (
|
1968
|
+
async_placeholder_flow
|
1969
|
+
if is_entrypoint_async(entrypoint)
|
1970
|
+
else sync_placeholder_flow
|
1971
|
+
)
|
1972
|
+
|
1973
|
+
arguments = load_flow_arguments_from_entrypoint(entrypoint)
|
1974
|
+
arguments["fn"] = placeholder_flow
|
1975
|
+
|
1976
|
+
return Flow(**arguments)
|
1977
|
+
|
1978
|
+
|
1979
|
+
def load_flow_arguments_from_entrypoint(
|
1980
|
+
entrypoint: str, arguments: Optional[Union[List[str], Set[str]]] = None
|
1981
|
+
) -> dict[str, Any]:
|
1982
|
+
"""
|
1983
|
+
Extract flow arguments from an entrypoint string.
|
1984
|
+
|
1985
|
+
Loads the source code of the entrypoint and extracts the flow arguments
|
1986
|
+
from the `flow` decorator.
|
1987
|
+
|
1988
|
+
Args:
|
1989
|
+
entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
|
1990
|
+
or a module path to a flow function
|
1991
|
+
"""
|
1992
|
+
|
1993
|
+
func_def, source_code = _entrypoint_definition_and_source(entrypoint)
|
1994
|
+
|
1995
|
+
if arguments is None:
|
1996
|
+
# If no arguments are provided default to known arguments that are of
|
1997
|
+
# built-in types.
|
1998
|
+
arguments = {
|
1999
|
+
"name",
|
2000
|
+
"version",
|
2001
|
+
"retries",
|
2002
|
+
"retry_delay_seconds",
|
2003
|
+
"description",
|
2004
|
+
"timeout_seconds",
|
2005
|
+
"validate_parameters",
|
2006
|
+
"persist_result",
|
2007
|
+
"cache_result_in_memory",
|
2008
|
+
"log_prints",
|
2009
|
+
}
|
2010
|
+
|
2011
|
+
result = {}
|
2012
|
+
|
2013
|
+
for decorator in func_def.decorator_list:
|
2014
|
+
if (
|
2015
|
+
isinstance(decorator, ast.Call)
|
2016
|
+
and getattr(decorator.func, "id", "") == "flow"
|
2017
|
+
):
|
2018
|
+
for keyword in decorator.keywords:
|
2019
|
+
if keyword.arg not in arguments:
|
2020
|
+
continue
|
2021
|
+
|
2022
|
+
if isinstance(keyword.value, ast.Constant):
|
2023
|
+
# Use the string value of the argument
|
2024
|
+
result[keyword.arg] = str(keyword.value.value)
|
2025
|
+
continue
|
2026
|
+
|
2027
|
+
# if the arg value is not a raw str (i.e. a variable or expression),
|
2028
|
+
# then attempt to evaluate it
|
2029
|
+
namespace = safe_load_namespace(source_code)
|
2030
|
+
literal_arg_value = ast.get_source_segment(source_code, keyword.value)
|
2031
|
+
cleaned_value = (
|
2032
|
+
literal_arg_value.replace("\n", "") if literal_arg_value else ""
|
2033
|
+
)
|
2034
|
+
|
2035
|
+
try:
|
2036
|
+
evaluated_value = eval(cleaned_value, namespace) # type: ignore
|
2037
|
+
result[keyword.arg] = str(evaluated_value)
|
2038
|
+
except Exception as e:
|
2039
|
+
logger.info(
|
2040
|
+
"Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
|
2041
|
+
keyword.arg,
|
2042
|
+
literal_arg_value,
|
2043
|
+
exc_info=e,
|
2044
|
+
)
|
2045
|
+
# ignore the decorator arg and fallback to default behavior
|
2046
|
+
continue
|
2047
|
+
|
2048
|
+
if "name" in arguments and "name" not in result:
|
2049
|
+
# If no matching decorator or keyword argument for `name' is found
|
2050
|
+
# fallback to the function name.
|
2051
|
+
result["name"] = func_def.name.replace("_", "-")
|
2052
|
+
|
2053
|
+
return result
|
2054
|
+
|
2055
|
+
|
2056
|
+
def is_entrypoint_async(entrypoint: str) -> bool:
|
2057
|
+
"""
|
2058
|
+
Determine if the function specified in the entrypoint is asynchronous.
|
2059
|
+
|
2060
|
+
Args:
|
2061
|
+
entrypoint: A string in the format `<path_to_script>:<func_name>` or
|
2062
|
+
a module path to a function.
|
1907
2063
|
|
1908
2064
|
Returns:
|
1909
|
-
|
2065
|
+
True if the function is asynchronous, False otherwise.
|
1910
2066
|
"""
|
2067
|
+
func_def, _ = _entrypoint_definition_and_source(entrypoint)
|
2068
|
+
return isinstance(func_def, ast.AsyncFunctionDef)
|
2069
|
+
|
2070
|
+
|
2071
|
+
def _entrypoint_definition_and_source(
|
2072
|
+
entrypoint: str,
|
2073
|
+
) -> Tuple[Union[ast.FunctionDef, ast.AsyncFunctionDef], str]:
|
1911
2074
|
if ":" in entrypoint:
|
1912
|
-
#
|
2075
|
+
# Split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
|
1913
2076
|
path, func_name = entrypoint.rsplit(":", maxsplit=1)
|
1914
2077
|
source_code = Path(path).read_text()
|
1915
2078
|
else:
|
@@ -1918,6 +2081,7 @@ def load_flow_argument_from_entrypoint(
|
|
1918
2081
|
if not spec or not spec.origin:
|
1919
2082
|
raise ValueError(f"Could not find module {path!r}")
|
1920
2083
|
source_code = Path(spec.origin).read_text()
|
2084
|
+
|
1921
2085
|
parsed_code = ast.parse(source_code)
|
1922
2086
|
func_def = next(
|
1923
2087
|
(
|
@@ -1934,46 +2098,8 @@ def load_flow_argument_from_entrypoint(
|
|
1934
2098
|
),
|
1935
2099
|
None,
|
1936
2100
|
)
|
2101
|
+
|
1937
2102
|
if not func_def:
|
1938
2103
|
raise ValueError(f"Could not find flow {func_name!r} in {path!r}")
|
1939
|
-
for decorator in func_def.decorator_list:
|
1940
|
-
if (
|
1941
|
-
isinstance(decorator, ast.Call)
|
1942
|
-
and getattr(decorator.func, "id", "") == "flow"
|
1943
|
-
):
|
1944
|
-
for keyword in decorator.keywords:
|
1945
|
-
if keyword.arg == arg:
|
1946
|
-
if isinstance(keyword.value, ast.Constant):
|
1947
|
-
return (
|
1948
|
-
keyword.value.value
|
1949
|
-
) # Return the string value of the argument
|
1950
|
-
|
1951
|
-
# if the arg value is not a raw str (i.e. a variable or expression),
|
1952
|
-
# then attempt to evaluate it
|
1953
|
-
namespace = safe_load_namespace(source_code)
|
1954
|
-
literal_arg_value = ast.get_source_segment(
|
1955
|
-
source_code, keyword.value
|
1956
|
-
)
|
1957
|
-
cleaned_value = (
|
1958
|
-
literal_arg_value.replace("\n", "") if literal_arg_value else ""
|
1959
|
-
)
|
1960
|
-
|
1961
|
-
try:
|
1962
|
-
evaluated_value = eval(cleaned_value, namespace) # type: ignore
|
1963
|
-
except Exception as e:
|
1964
|
-
logger.info(
|
1965
|
-
"Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
|
1966
|
-
arg,
|
1967
|
-
literal_arg_value,
|
1968
|
-
exc_info=e,
|
1969
|
-
)
|
1970
|
-
# ignore the decorator arg and fallback to default behavior
|
1971
|
-
break
|
1972
|
-
return str(evaluated_value)
|
1973
|
-
|
1974
|
-
if arg == "name":
|
1975
|
-
return func_name.replace(
|
1976
|
-
"_", "-"
|
1977
|
-
) # If no matching decorator or keyword argument is found
|
1978
2104
|
|
1979
|
-
return
|
2105
|
+
return func_def, source_code
|
prefect/futures.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
import abc
|
2
|
+
import collections
|
2
3
|
import concurrent.futures
|
3
4
|
import inspect
|
4
5
|
import uuid
|
@@ -256,13 +257,7 @@ class PrefectFutureList(list, Iterator, Generic[F]):
|
|
256
257
|
timeout: The maximum number of seconds to wait for all futures to
|
257
258
|
complete. This method will not raise if the timeout is reached.
|
258
259
|
"""
|
259
|
-
|
260
|
-
with timeout_context(timeout):
|
261
|
-
for future in self:
|
262
|
-
future.wait()
|
263
|
-
except TimeoutError:
|
264
|
-
logger.debug("Timed out waiting for all futures to complete.")
|
265
|
-
return
|
260
|
+
wait(self, timeout=timeout)
|
266
261
|
|
267
262
|
def result(
|
268
263
|
self,
|
@@ -297,6 +292,57 @@ class PrefectFutureList(list, Iterator, Generic[F]):
|
|
297
292
|
) from exc
|
298
293
|
|
299
294
|
|
295
|
+
DoneAndNotDoneFutures = collections.namedtuple("DoneAndNotDoneFutures", "done not_done")
|
296
|
+
|
297
|
+
|
298
|
+
def wait(futures: List[PrefectFuture], timeout=None) -> DoneAndNotDoneFutures:
|
299
|
+
"""
|
300
|
+
Wait for the futures in the given sequence to complete.
|
301
|
+
|
302
|
+
Args:
|
303
|
+
futures: The sequence of Futures to wait upon.
|
304
|
+
timeout: The maximum number of seconds to wait. If None, then there
|
305
|
+
is no limit on the wait time.
|
306
|
+
|
307
|
+
Returns:
|
308
|
+
A named 2-tuple of sets. The first set, named 'done', contains the
|
309
|
+
futures that completed (is finished or cancelled) before the wait
|
310
|
+
completed. The second set, named 'not_done', contains uncompleted
|
311
|
+
futures. Duplicate futures given to *futures* are removed and will be
|
312
|
+
returned only once.
|
313
|
+
|
314
|
+
Examples:
|
315
|
+
```python
|
316
|
+
@task
|
317
|
+
def sleep_task(seconds):
|
318
|
+
sleep(seconds)
|
319
|
+
return 42
|
320
|
+
|
321
|
+
@flow
|
322
|
+
def flow():
|
323
|
+
futures = random_task.map(range(10))
|
324
|
+
done, not_done = wait(futures, timeout=5)
|
325
|
+
print(f"Done: {len(done)}")
|
326
|
+
print(f"Not Done: {len(not_done)}")
|
327
|
+
```
|
328
|
+
"""
|
329
|
+
futures = set(futures)
|
330
|
+
done = {f for f in futures if f._final_state}
|
331
|
+
not_done = futures - done
|
332
|
+
if len(done) == len(futures):
|
333
|
+
return DoneAndNotDoneFutures(done, not_done)
|
334
|
+
try:
|
335
|
+
with timeout_context(timeout):
|
336
|
+
for future in not_done.copy():
|
337
|
+
future.wait()
|
338
|
+
done.add(future)
|
339
|
+
not_done.remove(future)
|
340
|
+
return DoneAndNotDoneFutures(done, not_done)
|
341
|
+
except TimeoutError:
|
342
|
+
logger.debug("Timed out waiting for all futures to complete.")
|
343
|
+
return DoneAndNotDoneFutures(done, not_done)
|
344
|
+
|
345
|
+
|
300
346
|
def resolve_futures_to_states(
|
301
347
|
expr: Union[PrefectFuture, Any],
|
302
348
|
) -> Union[State, Any]:
|
prefect/logging/loggers.py
CHANGED
@@ -97,7 +97,7 @@ def get_logger(name: Optional[str] = None) -> logging.Logger:
|
|
97
97
|
|
98
98
|
|
99
99
|
def get_run_logger(
|
100
|
-
context: "RunContext" = None, **kwargs: str
|
100
|
+
context: Optional["RunContext"] = None, **kwargs: str
|
101
101
|
) -> Union[logging.Logger, logging.LoggerAdapter]:
|
102
102
|
"""
|
103
103
|
Get a Prefect logger for the current task run or flow run.
|
prefect/results.py
CHANGED
@@ -25,7 +25,7 @@ from typing_extensions import ParamSpec, Self
|
|
25
25
|
import prefect
|
26
26
|
from prefect.blocks.core import Block
|
27
27
|
from prefect.client.utilities import inject_client
|
28
|
-
from prefect.exceptions import MissingResult
|
28
|
+
from prefect.exceptions import MissingResult
|
29
29
|
from prefect.filesystems import (
|
30
30
|
LocalFileSystem,
|
31
31
|
WritableFileSystem,
|
@@ -64,51 +64,6 @@ R = TypeVar("R")
|
|
64
64
|
_default_storages: Dict[Tuple[str, str], WritableFileSystem] = {}
|
65
65
|
|
66
66
|
|
67
|
-
async def _get_or_create_default_storage(block_document_slug: str) -> ResultStorage:
|
68
|
-
"""
|
69
|
-
Generate a default file system for storage.
|
70
|
-
"""
|
71
|
-
default_storage_name, storage_path = cache_key = (
|
72
|
-
block_document_slug,
|
73
|
-
PREFECT_LOCAL_STORAGE_PATH.value(),
|
74
|
-
)
|
75
|
-
|
76
|
-
async def get_storage() -> WritableFileSystem:
|
77
|
-
try:
|
78
|
-
return await Block.load(default_storage_name)
|
79
|
-
except ValueError as e:
|
80
|
-
if "Unable to find" not in str(e):
|
81
|
-
raise e
|
82
|
-
|
83
|
-
block_type_slug, name = default_storage_name.split("/")
|
84
|
-
if block_type_slug == "local-file-system":
|
85
|
-
block = LocalFileSystem(basepath=storage_path)
|
86
|
-
else:
|
87
|
-
raise ValueError(
|
88
|
-
"The default storage block does not exist, but it is of type "
|
89
|
-
f"'{block_type_slug}' which cannot be created implicitly. Please create "
|
90
|
-
"the block manually."
|
91
|
-
)
|
92
|
-
|
93
|
-
try:
|
94
|
-
await block.save(name, overwrite=False)
|
95
|
-
except ValueError as e:
|
96
|
-
if "already in use" not in str(e):
|
97
|
-
raise e
|
98
|
-
except ObjectAlreadyExists:
|
99
|
-
# Another client created the block before we reached this line
|
100
|
-
block = await Block.load(default_storage_name)
|
101
|
-
|
102
|
-
return block
|
103
|
-
|
104
|
-
try:
|
105
|
-
return _default_storages[cache_key]
|
106
|
-
except KeyError:
|
107
|
-
storage = await get_storage()
|
108
|
-
_default_storages[cache_key] = storage
|
109
|
-
return storage
|
110
|
-
|
111
|
-
|
112
67
|
@sync_compatible
|
113
68
|
async def get_default_result_storage() -> ResultStorage:
|
114
69
|
"""
|