prefect-client 3.0.0rc13__py3-none-any.whl → 3.0.0rc14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
prefect/flows.py CHANGED
@@ -95,7 +95,7 @@ from prefect.utilities.callables import (
95
95
  parameters_to_args_kwargs,
96
96
  raise_for_reserved_arguments,
97
97
  )
98
- from prefect.utilities.collections import listrepr
98
+ from prefect.utilities.collections import listrepr, visit_collection
99
99
  from prefect.utilities.filesystem import relative_path_to_current_platform
100
100
  from prefect.utilities.hashing import file_hash
101
101
  from prefect.utilities.importtools import import_object, safe_load_namespace
@@ -535,6 +535,21 @@ class Flow(Generic[P, R]):
535
535
  Raises:
536
536
  ParameterTypeError: if the provided parameters are not valid
537
537
  """
538
+
539
+ def resolve_block_reference(data: Any) -> Any:
540
+ if isinstance(data, dict) and "$ref" in data:
541
+ return Block.load_from_ref(data["$ref"])
542
+ return data
543
+
544
+ try:
545
+ parameters = visit_collection(
546
+ parameters, resolve_block_reference, return_data=True
547
+ )
548
+ except (ValueError, RuntimeError) as exc:
549
+ raise ParameterTypeError(
550
+ "Failed to resolve block references in parameters."
551
+ ) from exc
552
+
538
553
  args, kwargs = parameters_to_args_kwargs(self.fn, parameters)
539
554
 
540
555
  with warnings.catch_warnings():
@@ -1734,14 +1749,13 @@ def load_flow_from_entrypoint(
1734
1749
  raise MissingFlowError(
1735
1750
  f"Flow function with name {func_name!r} not found in {path!r}. "
1736
1751
  ) from exc
1737
- except ScriptError as exc:
1752
+ except ScriptError:
1738
1753
  # If the flow has dependencies that are not installed in the current
1739
- # environment, fallback to loading the flow via AST parsing. The
1740
- # drawback of this approach is that we're unable to actually load the
1741
- # function, so we create a placeholder flow that will re-raise this
1742
- # exception when called.
1754
+ # environment, fallback to loading the flow via AST parsing.
1743
1755
  if use_placeholder_flow:
1744
- flow = load_placeholder_flow(entrypoint=entrypoint, raises=exc)
1756
+ flow = safe_load_flow_from_entrypoint(entrypoint)
1757
+ if flow is None:
1758
+ raise
1745
1759
  else:
1746
1760
  raise
1747
1761
 
@@ -1976,6 +1990,147 @@ def load_placeholder_flow(entrypoint: str, raises: Exception):
1976
1990
  return Flow(**arguments)
1977
1991
 
1978
1992
 
1993
+ def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow]:
1994
+ """
1995
+ Load a flow from an entrypoint and return None if an exception is raised.
1996
+
1997
+ Args:
1998
+ entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
1999
+ or a module path to a flow function
2000
+ """
2001
+ func_def, source_code = _entrypoint_definition_and_source(entrypoint)
2002
+ path = None
2003
+ if ":" in entrypoint:
2004
+ path = entrypoint.rsplit(":")[0]
2005
+ namespace = safe_load_namespace(source_code, filepath=path)
2006
+ if func_def.name in namespace:
2007
+ return namespace[func_def.name]
2008
+ else:
2009
+ # If the function is not in the namespace, if may be due to missing dependencies
2010
+ # for the function. We will attempt to compile each annotation and default value
2011
+ # and remove them from the function definition to see if the function can be
2012
+ # compiled without them.
2013
+
2014
+ return _sanitize_and_load_flow(func_def, namespace)
2015
+
2016
+
2017
+ def _sanitize_and_load_flow(
2018
+ func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], namespace: Dict[str, Any]
2019
+ ) -> Optional[Flow]:
2020
+ """
2021
+ Attempt to load a flow from the function definition after sanitizing the annotations
2022
+ and defaults that can't be compiled.
2023
+
2024
+ Args:
2025
+ func_def: the function definition
2026
+ namespace: the namespace to load the function into
2027
+
2028
+ Returns:
2029
+ The loaded function or None if the function can't be loaded
2030
+ after sanitizing the annotations and defaults.
2031
+ """
2032
+ args = func_def.args.posonlyargs + func_def.args.args + func_def.args.kwonlyargs
2033
+ if func_def.args.vararg:
2034
+ args.append(func_def.args.vararg)
2035
+ if func_def.args.kwarg:
2036
+ args.append(func_def.args.kwarg)
2037
+ # Remove annotations that can't be compiled
2038
+ for arg in args:
2039
+ if arg.annotation is not None:
2040
+ try:
2041
+ code = compile(
2042
+ ast.Expression(arg.annotation),
2043
+ filename="<ast>",
2044
+ mode="eval",
2045
+ )
2046
+ exec(code, namespace)
2047
+ except Exception as e:
2048
+ logger.debug(
2049
+ "Failed to evaluate annotation for argument %s due to the following error. Ignoring annotation.",
2050
+ arg.arg,
2051
+ exc_info=e,
2052
+ )
2053
+ arg.annotation = None
2054
+
2055
+ # Remove defaults that can't be compiled
2056
+ new_defaults = []
2057
+ for default in func_def.args.defaults:
2058
+ try:
2059
+ code = compile(ast.Expression(default), "<ast>", "eval")
2060
+ exec(code, namespace)
2061
+ new_defaults.append(default)
2062
+ except Exception as e:
2063
+ logger.debug(
2064
+ "Failed to evaluate default value %s due to the following error. Ignoring default.",
2065
+ default,
2066
+ exc_info=e,
2067
+ )
2068
+ new_defaults.append(
2069
+ ast.Constant(
2070
+ value=None, lineno=default.lineno, col_offset=default.col_offset
2071
+ )
2072
+ )
2073
+ func_def.args.defaults = new_defaults
2074
+
2075
+ # Remove kw_defaults that can't be compiled
2076
+ new_kw_defaults = []
2077
+ for default in func_def.args.kw_defaults:
2078
+ if default is not None:
2079
+ try:
2080
+ code = compile(ast.Expression(default), "<ast>", "eval")
2081
+ exec(code, namespace)
2082
+ new_kw_defaults.append(default)
2083
+ except Exception as e:
2084
+ logger.debug(
2085
+ "Failed to evaluate default value %s due to the following error. Ignoring default.",
2086
+ default,
2087
+ exc_info=e,
2088
+ )
2089
+ new_kw_defaults.append(
2090
+ ast.Constant(
2091
+ value=None,
2092
+ lineno=default.lineno,
2093
+ col_offset=default.col_offset,
2094
+ )
2095
+ )
2096
+ else:
2097
+ new_kw_defaults.append(
2098
+ ast.Constant(
2099
+ value=None,
2100
+ lineno=func_def.lineno,
2101
+ col_offset=func_def.col_offset,
2102
+ )
2103
+ )
2104
+ func_def.args.kw_defaults = new_kw_defaults
2105
+
2106
+ if func_def.returns is not None:
2107
+ try:
2108
+ code = compile(
2109
+ ast.Expression(func_def.returns), filename="<ast>", mode="eval"
2110
+ )
2111
+ exec(code, namespace)
2112
+ except Exception as e:
2113
+ logger.debug(
2114
+ "Failed to evaluate return annotation due to the following error. Ignoring annotation.",
2115
+ exc_info=e,
2116
+ )
2117
+ func_def.returns = None
2118
+
2119
+ # Attempt to compile the function without annotations and defaults that
2120
+ # can't be compiled
2121
+ try:
2122
+ code = compile(
2123
+ ast.Module(body=[func_def], type_ignores=[]),
2124
+ filename="<ast>",
2125
+ mode="exec",
2126
+ )
2127
+ exec(code, namespace)
2128
+ except Exception as e:
2129
+ logger.debug("Failed to compile: %s", e)
2130
+ else:
2131
+ return namespace.get(func_def.name)
2132
+
2133
+
1979
2134
  def load_flow_arguments_from_entrypoint(
1980
2135
  entrypoint: str, arguments: Optional[Union[List[str], Set[str]]] = None
1981
2136
  ) -> dict[str, Any]:
@@ -1991,6 +2146,9 @@ def load_flow_arguments_from_entrypoint(
1991
2146
  """
1992
2147
 
1993
2148
  func_def, source_code = _entrypoint_definition_and_source(entrypoint)
2149
+ path = None
2150
+ if ":" in entrypoint:
2151
+ path = entrypoint.rsplit(":")[0]
1994
2152
 
1995
2153
  if arguments is None:
1996
2154
  # If no arguments are provided default to known arguments that are of
@@ -2026,7 +2184,7 @@ def load_flow_arguments_from_entrypoint(
2026
2184
 
2027
2185
  # if the arg value is not a raw str (i.e. a variable or expression),
2028
2186
  # then attempt to evaluate it
2029
- namespace = safe_load_namespace(source_code)
2187
+ namespace = safe_load_namespace(source_code, filepath=path)
2030
2188
  literal_arg_value = ast.get_source_segment(source_code, keyword.value)
2031
2189
  cleaned_value = (
2032
2190
  literal_arg_value.replace("\n", "") if literal_arg_value else ""
prefect/futures.py CHANGED
@@ -2,10 +2,11 @@ import abc
2
2
  import collections
3
3
  import concurrent.futures
4
4
  import inspect
5
+ import threading
5
6
  import uuid
6
- from collections.abc import Iterator
7
+ from collections.abc import Generator, Iterator
7
8
  from functools import partial
8
- from typing import Any, Generic, List, Optional, Set, Union, cast
9
+ from typing import Any, Callable, Generic, List, Optional, Set, Union, cast
9
10
 
10
11
  from typing_extensions import TypeVar
11
12
 
@@ -91,6 +92,16 @@ class PrefectFuture(abc.ABC, Generic[R]):
91
92
  The result of the task run.
92
93
  """
93
94
 
95
+ @abc.abstractmethod
96
+ def add_done_callback(self, fn):
97
+ """
98
+ Add a callback to be run when the future completes or is cancelled.
99
+
100
+ Args:
101
+ fn: A callable that will be called with this future as its only argument when the future completes or is cancelled.
102
+ """
103
+ ...
104
+
94
105
 
95
106
  class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[R, F]):
96
107
  """
@@ -106,6 +117,17 @@ class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[R, F]):
106
117
  """The underlying future object wrapped by this Prefect future"""
107
118
  return self._wrapped_future
108
119
 
120
+ def add_done_callback(self, fn: Callable[[PrefectFuture], None]):
121
+ if not self._final_state:
122
+
123
+ def call_with_self(future):
124
+ """Call the callback with self as the argument, this is necessary to ensure we remove the future from the pending set"""
125
+ fn(self)
126
+
127
+ self._wrapped_future.add_done_callback(call_with_self)
128
+ return
129
+ fn(self)
130
+
109
131
 
110
132
  class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]):
111
133
  """
@@ -138,6 +160,7 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
138
160
 
139
161
  if isinstance(future_result, State):
140
162
  self._final_state = future_result
163
+
141
164
  else:
142
165
  return future_result
143
166
 
@@ -172,6 +195,9 @@ class PrefectDistributedFuture(PrefectFuture[R]):
172
195
  any task run scheduled in Prefect's API.
173
196
  """
174
197
 
198
+ done_callbacks: List[Callable[[PrefectFuture], None]] = []
199
+ waiter = None
200
+
175
201
  @deprecated_async_method
176
202
  def wait(self, timeout: Optional[float] = None) -> None:
177
203
  return run_coro_as_sync(self.wait_async(timeout=timeout))
@@ -235,11 +261,27 @@ class PrefectDistributedFuture(PrefectFuture[R]):
235
261
  raise_on_failure=raise_on_failure, fetch=True
236
262
  )
237
263
 
264
+ def add_done_callback(self, fn: Callable[[PrefectFuture], None]):
265
+ if self._final_state:
266
+ fn(self)
267
+ return
268
+ TaskRunWaiter.instance()
269
+ with get_client(sync_client=True) as client:
270
+ task_run = client.read_task_run(task_run_id=self._task_run_id)
271
+ if task_run.state.is_final():
272
+ self._final_state = task_run.state
273
+ fn(self)
274
+ return
275
+ TaskRunWaiter.add_done_callback(self._task_run_id, partial(fn, self))
276
+
238
277
  def __eq__(self, other):
239
278
  if not isinstance(other, PrefectDistributedFuture):
240
279
  return False
241
280
  return self.task_run_id == other.task_run_id
242
281
 
282
+ def __hash__(self):
283
+ return hash(self.task_run_id)
284
+
243
285
 
244
286
  class PrefectFutureList(list, Iterator, Generic[F]):
245
287
  """
@@ -292,6 +334,46 @@ class PrefectFutureList(list, Iterator, Generic[F]):
292
334
  ) from exc
293
335
 
294
336
 
337
+ def as_completed(
338
+ futures: List[PrefectFuture], timeout: Optional[float] = None
339
+ ) -> Generator[PrefectFuture, None]:
340
+ unique_futures: Set[PrefectFuture] = set(futures)
341
+ total_futures = len(unique_futures)
342
+ try:
343
+ with timeout_context(timeout):
344
+ done = {f for f in unique_futures if f._final_state}
345
+ pending = unique_futures - done
346
+ yield from done
347
+
348
+ finished_event = threading.Event()
349
+ finished_lock = threading.Lock()
350
+ finished_futures = []
351
+
352
+ def add_to_done(future):
353
+ with finished_lock:
354
+ finished_futures.append(future)
355
+ finished_event.set()
356
+
357
+ for future in pending:
358
+ future.add_done_callback(add_to_done)
359
+
360
+ while pending:
361
+ finished_event.wait()
362
+ with finished_lock:
363
+ done = finished_futures
364
+ finished_futures = []
365
+ finished_event.clear()
366
+
367
+ for future in done:
368
+ pending.remove(future)
369
+ yield future
370
+
371
+ except TimeoutError:
372
+ raise TimeoutError(
373
+ "%d (of %d) futures unfinished" % (len(pending), total_futures)
374
+ )
375
+
376
+
295
377
  DoneAndNotDoneFutures = collections.namedtuple("DoneAndNotDoneFutures", "done not_done")
296
378
 
297
379
 
prefect/profiles.toml CHANGED
@@ -1,3 +1,14 @@
1
- active = "default"
1
+ # This is a template for profile configuration for Prefect.
2
+ # You can modify these profiles or create new ones to suit your needs.
2
3
 
3
- [profiles.default]
4
+ active = "ephemeral"
5
+
6
+ [profiles.ephemeral]
7
+ PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = "true"
8
+
9
+ [profiles.local]
10
+ # You will need to set these values appropriately for your local development environment
11
+ PREFECT_API_URL = "http://127.0.0.1:4200/api"
12
+
13
+
14
+ [profiles.cloud]
prefect/runner/runner.py CHANGED
@@ -92,7 +92,10 @@ from prefect.utilities.asyncutils import (
92
92
  )
93
93
  from prefect.utilities.engine import propose_state
94
94
  from prefect.utilities.processutils import _register_signal, run_process
95
- from prefect.utilities.services import critical_service_loop
95
+ from prefect.utilities.services import (
96
+ critical_service_loop,
97
+ start_client_metrics_server,
98
+ )
96
99
  from prefect.utilities.slugify import slugify
97
100
 
98
101
  if TYPE_CHECKING:
@@ -380,6 +383,8 @@ class Runner:
380
383
  )
381
384
  server_thread.start()
382
385
 
386
+ start_client_metrics_server()
387
+
383
388
  async with self as runner:
384
389
  async with self._loops_task_group as tg:
385
390
  for storage in self._storage_objs:
prefect/settings.py CHANGED
@@ -1216,6 +1216,19 @@ compromise. Adjust this setting based on your specific security requirements
1216
1216
  and usage patterns.
1217
1217
  """
1218
1218
 
1219
+ PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = Setting(bool, default=False)
1220
+ """
1221
+ Controls whether or not a subprocess server can be started when no API URL is provided.
1222
+ """
1223
+
1224
+ PREFECT_SERVER_EPHEMERAL_STARTUP_TIMEOUT_SECONDS = Setting(
1225
+ int,
1226
+ default=10,
1227
+ )
1228
+ """
1229
+ The number of seconds to wait for an ephemeral server to respond on start up before erroring.
1230
+ """
1231
+
1219
1232
  PREFECT_UI_ENABLED = Setting(
1220
1233
  bool,
1221
1234
  default=True,
@@ -1561,6 +1574,26 @@ The page size for the queries to backfill events for websocket subscribers
1561
1574
  """
1562
1575
 
1563
1576
 
1577
+ # Metrics settings
1578
+
1579
+ PREFECT_API_ENABLE_METRICS = Setting(bool, default=False)
1580
+ """
1581
+ Whether or not to enable Prometheus metrics in the server application. Metrics are
1582
+ served at the path /api/metrics on the API server.
1583
+ """
1584
+
1585
+ PREFECT_CLIENT_ENABLE_METRICS = Setting(bool, default=False)
1586
+ """
1587
+ Whether or not to enable Prometheus metrics in the client SDK. Metrics are served
1588
+ at the path /metrics.
1589
+ """
1590
+
1591
+ PREFECT_CLIENT_METRICS_PORT = Setting(int, default=4201)
1592
+ """
1593
+ The port to expose the client Prometheus metrics on.
1594
+ """
1595
+
1596
+
1564
1597
  # Deprecated settings ------------------------------------------------------------------
1565
1598
 
1566
1599
 
@@ -2125,10 +2158,10 @@ def load_current_profile():
2125
2158
  This will _not_ include settings from the current settings context. Only settings
2126
2159
  that have been persisted to the profiles file will be saved.
2127
2160
  """
2128
- from prefect.context import SettingsContext
2161
+ import prefect.context
2129
2162
 
2130
2163
  profiles = load_profiles()
2131
- context = SettingsContext.get()
2164
+ context = prefect.context.get_settings_context()
2132
2165
 
2133
2166
  if context:
2134
2167
  profiles.set_active(context.profile.name)