prefect-client 3.1.5__py3-none-any.whl → 3.1.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. prefect/__init__.py +3 -0
  2. prefect/_experimental/__init__.py +0 -0
  3. prefect/_experimental/lineage.py +181 -0
  4. prefect/_internal/compatibility/async_dispatch.py +38 -9
  5. prefect/_internal/compatibility/migration.py +1 -1
  6. prefect/_internal/concurrency/api.py +52 -52
  7. prefect/_internal/concurrency/calls.py +59 -35
  8. prefect/_internal/concurrency/cancellation.py +34 -18
  9. prefect/_internal/concurrency/event_loop.py +7 -6
  10. prefect/_internal/concurrency/threads.py +41 -33
  11. prefect/_internal/concurrency/waiters.py +28 -21
  12. prefect/_internal/pydantic/v1_schema.py +2 -2
  13. prefect/_internal/pydantic/v2_schema.py +10 -9
  14. prefect/_internal/pydantic/v2_validated_func.py +15 -10
  15. prefect/_internal/retries.py +15 -6
  16. prefect/_internal/schemas/bases.py +11 -8
  17. prefect/_internal/schemas/validators.py +7 -5
  18. prefect/_version.py +3 -3
  19. prefect/automations.py +53 -47
  20. prefect/blocks/abstract.py +12 -10
  21. prefect/blocks/core.py +148 -19
  22. prefect/blocks/system.py +2 -1
  23. prefect/cache_policies.py +11 -11
  24. prefect/client/__init__.py +3 -1
  25. prefect/client/base.py +36 -37
  26. prefect/client/cloud.py +26 -19
  27. prefect/client/collections.py +2 -2
  28. prefect/client/orchestration.py +430 -273
  29. prefect/client/schemas/__init__.py +24 -0
  30. prefect/client/schemas/actions.py +128 -121
  31. prefect/client/schemas/filters.py +1 -1
  32. prefect/client/schemas/objects.py +114 -85
  33. prefect/client/schemas/responses.py +19 -20
  34. prefect/client/schemas/schedules.py +136 -93
  35. prefect/client/subscriptions.py +30 -15
  36. prefect/client/utilities.py +46 -36
  37. prefect/concurrency/asyncio.py +6 -9
  38. prefect/concurrency/sync.py +35 -5
  39. prefect/context.py +40 -32
  40. prefect/deployments/flow_runs.py +6 -8
  41. prefect/deployments/runner.py +14 -14
  42. prefect/deployments/steps/core.py +3 -1
  43. prefect/deployments/steps/pull.py +60 -12
  44. prefect/docker/__init__.py +1 -1
  45. prefect/events/clients.py +55 -4
  46. prefect/events/filters.py +1 -1
  47. prefect/events/related.py +2 -1
  48. prefect/events/schemas/events.py +26 -21
  49. prefect/events/utilities.py +3 -2
  50. prefect/events/worker.py +8 -0
  51. prefect/filesystems.py +3 -3
  52. prefect/flow_engine.py +87 -87
  53. prefect/flow_runs.py +7 -5
  54. prefect/flows.py +218 -176
  55. prefect/logging/configuration.py +1 -1
  56. prefect/logging/highlighters.py +1 -2
  57. prefect/logging/loggers.py +30 -20
  58. prefect/main.py +17 -24
  59. prefect/results.py +43 -22
  60. prefect/runner/runner.py +43 -21
  61. prefect/runner/server.py +30 -32
  62. prefect/runner/storage.py +3 -3
  63. prefect/runner/submit.py +3 -6
  64. prefect/runner/utils.py +6 -6
  65. prefect/runtime/flow_run.py +7 -0
  66. prefect/serializers.py +28 -24
  67. prefect/settings/constants.py +2 -2
  68. prefect/settings/legacy.py +1 -1
  69. prefect/settings/models/experiments.py +5 -0
  70. prefect/settings/models/server/events.py +10 -0
  71. prefect/task_engine.py +87 -26
  72. prefect/task_runners.py +2 -2
  73. prefect/task_worker.py +43 -25
  74. prefect/tasks.py +148 -142
  75. prefect/telemetry/bootstrap.py +15 -2
  76. prefect/telemetry/instrumentation.py +1 -1
  77. prefect/telemetry/processors.py +10 -7
  78. prefect/telemetry/run_telemetry.py +231 -0
  79. prefect/transactions.py +14 -14
  80. prefect/types/__init__.py +5 -5
  81. prefect/utilities/_engine.py +96 -0
  82. prefect/utilities/annotations.py +25 -18
  83. prefect/utilities/asyncutils.py +126 -140
  84. prefect/utilities/callables.py +87 -78
  85. prefect/utilities/collections.py +278 -117
  86. prefect/utilities/compat.py +13 -21
  87. prefect/utilities/context.py +6 -5
  88. prefect/utilities/dispatch.py +23 -12
  89. prefect/utilities/dockerutils.py +33 -32
  90. prefect/utilities/engine.py +126 -239
  91. prefect/utilities/filesystem.py +18 -15
  92. prefect/utilities/hashing.py +10 -11
  93. prefect/utilities/importtools.py +40 -27
  94. prefect/utilities/math.py +9 -5
  95. prefect/utilities/names.py +3 -3
  96. prefect/utilities/processutils.py +121 -57
  97. prefect/utilities/pydantic.py +41 -36
  98. prefect/utilities/render_swagger.py +22 -12
  99. prefect/utilities/schema_tools/__init__.py +2 -1
  100. prefect/utilities/schema_tools/hydration.py +50 -43
  101. prefect/utilities/schema_tools/validation.py +52 -42
  102. prefect/utilities/services.py +13 -12
  103. prefect/utilities/templating.py +45 -45
  104. prefect/utilities/text.py +2 -1
  105. prefect/utilities/timeout.py +4 -4
  106. prefect/utilities/urls.py +9 -4
  107. prefect/utilities/visualization.py +46 -24
  108. prefect/variables.py +136 -27
  109. prefect/workers/base.py +15 -8
  110. {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/METADATA +5 -2
  111. {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/RECORD +114 -110
  112. {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/LICENSE +0 -0
  113. {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/WHEEL +0 -0
  114. {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/top_level.txt +0 -0
prefect/variables.py CHANGED
@@ -1,13 +1,14 @@
1
- from typing import List, Optional
1
+ from typing import Optional
2
2
 
3
3
  from pydantic import BaseModel, Field
4
4
 
5
+ from prefect._internal.compatibility.async_dispatch import async_dispatch
5
6
  from prefect._internal.compatibility.migration import getattr_migration
7
+ from prefect.client.orchestration import get_client
6
8
  from prefect.client.schemas.actions import VariableCreate, VariableUpdate
7
9
  from prefect.client.utilities import get_or_create_client
8
10
  from prefect.exceptions import ObjectNotFound
9
11
  from prefect.types import MAX_VARIABLE_NAME_LENGTH, StrictVariableValue
10
- from prefect.utilities.asyncutils import sync_compatible
11
12
 
12
13
 
13
14
  class Variable(BaseModel):
@@ -31,19 +32,18 @@ class Variable(BaseModel):
31
32
  description="The value of the variable",
32
33
  examples=["my-value"],
33
34
  )
34
- tags: Optional[List[str]] = Field(default=None)
35
+ tags: Optional[list[str]] = Field(default=None)
35
36
 
36
37
  @classmethod
37
- @sync_compatible
38
- async def set(
38
+ async def aset(
39
39
  cls,
40
40
  name: str,
41
41
  value: StrictVariableValue,
42
- tags: Optional[List[str]] = None,
42
+ tags: Optional[list[str]] = None,
43
43
  overwrite: bool = False,
44
44
  ) -> "Variable":
45
45
  """
46
- Sets a new variable. If one exists with the same name, must pass `overwrite=True`
46
+ Asynchronously sets a new variable. If one exists with the same name, must pass `overwrite=True`
47
47
 
48
48
  Returns the newly set variable object.
49
49
 
@@ -60,8 +60,8 @@ class Variable(BaseModel):
60
60
  from prefect.variables import Variable
61
61
 
62
62
  @flow
63
- def my_flow():
64
- Variable.set(name="my_var",value="test_value", tags=["hi", "there"], overwrite=True)
63
+ async def my_flow():
64
+ await Variable.aset(name="my_var",value="test_value", tags=["hi", "there"], overwrite=True)
65
65
  ```
66
66
  """
67
67
  client, _ = get_or_create_client()
@@ -73,27 +73,76 @@ class Variable(BaseModel):
73
73
  raise ValueError(
74
74
  f"Variable {name!r} already exists. Use `overwrite=True` to update it."
75
75
  )
76
- await client.update_variable(variable=VariableUpdate(**var_dict))
76
+ await client.update_variable(
77
+ variable=VariableUpdate.model_validate(var_dict)
78
+ )
77
79
  variable = await client.read_variable_by_name(name)
78
- var_dict = {
79
- "name": variable.name,
80
- "value": variable.value,
81
- "tags": variable.tags or [],
82
- }
80
+ for key in var_dict.keys():
81
+ var_dict.update({key: getattr(variable, key)})
83
82
  else:
84
- await client.create_variable(variable=VariableCreate(**var_dict))
83
+ await client.create_variable(
84
+ variable=VariableCreate.model_validate(var_dict)
85
+ )
86
+
87
+ return cls.model_validate(var_dict)
88
+
89
+ @classmethod
90
+ @async_dispatch(aset)
91
+ def set(
92
+ cls,
93
+ name: str,
94
+ value: StrictVariableValue,
95
+ tags: Optional[list[str]] = None,
96
+ overwrite: bool = False,
97
+ ) -> "Variable":
98
+ """
99
+ Sets a new variable. If one exists with the same name, must pass `overwrite=True`
100
+
101
+ Returns the newly set variable object.
102
+
103
+ Args:
104
+ - name: The name of the variable to set.
105
+ - value: The value of the variable to set.
106
+ - tags: An optional list of strings to associate with the variable.
107
+ - overwrite: Whether to overwrite the variable if it already exists.
108
+
109
+ Example:
110
+ Set a new variable and overwrite it if it already exists.
111
+
112
+ ```
113
+ from prefect.variables import Variable
114
+
115
+ @flow
116
+ def my_flow():
117
+ Variable.set(name="my_var",value="test_value", tags=["hi", "there"], overwrite=True)
118
+ ```
119
+ """
120
+ with get_client(sync_client=True) as client:
121
+ variable_exists = client.read_variable_by_name(name)
122
+ var_dict = {"name": name, "value": value, "tags": tags or []}
85
123
 
86
- return cls(**var_dict)
124
+ if variable_exists:
125
+ if not overwrite:
126
+ raise ValueError(
127
+ f"Variable {name!r} already exists. Use `overwrite=True` to update it."
128
+ )
129
+ client.update_variable(variable=VariableUpdate.model_validate(var_dict))
130
+ variable = client.read_variable_by_name(name)
131
+ for key in var_dict.keys():
132
+ var_dict.update({key: getattr(variable, key)})
133
+ else:
134
+ client.create_variable(variable=VariableCreate.model_validate(var_dict))
135
+
136
+ return cls.model_validate(var_dict)
87
137
 
88
138
  @classmethod
89
- @sync_compatible
90
- async def get(
139
+ async def aget(
91
140
  cls,
92
141
  name: str,
93
142
  default: StrictVariableValue = None,
94
143
  ) -> StrictVariableValue:
95
144
  """
96
- Get a variable's value by name.
145
+ Asynchronously get a variable's value by name.
97
146
 
98
147
  If the variable does not exist, return the default value.
99
148
 
@@ -108,8 +157,8 @@ class Variable(BaseModel):
108
157
  from prefect.variables import Variable
109
158
 
110
159
  @flow
111
- def my_flow():
112
- var = Variable.get("my_var")
160
+ async def my_flow():
161
+ var = await Variable.aget("my_var")
113
162
  ```
114
163
  """
115
164
  client, _ = get_or_create_client()
@@ -118,10 +167,41 @@ class Variable(BaseModel):
118
167
  return variable.value if variable else default
119
168
 
120
169
  @classmethod
121
- @sync_compatible
122
- async def unset(cls, name: str) -> bool:
170
+ @async_dispatch(aget)
171
+ def get(
172
+ cls,
173
+ name: str,
174
+ default: StrictVariableValue = None,
175
+ ) -> StrictVariableValue:
123
176
  """
124
- Unset a variable by name.
177
+ Get a variable's value by name.
178
+
179
+ If the variable does not exist, return the default value.
180
+
181
+ Args:
182
+ - name: The name of the variable value to get.
183
+ - default: The default value to return if the variable does not exist.
184
+
185
+ Example:
186
+ Get a variable's value by name.
187
+ ```python
188
+ from prefect import flow
189
+ from prefect.variables import Variable
190
+
191
+ @flow
192
+ def my_flow():
193
+ var = Variable.get("my_var")
194
+ ```
195
+ """
196
+ with get_client(sync_client=True) as client:
197
+ variable = client.read_variable_by_name(name)
198
+
199
+ return variable.value if variable else default
200
+
201
+ @classmethod
202
+ async def aunset(cls, name: str) -> bool:
203
+ """
204
+ Asynchronously unset a variable by name.
125
205
 
126
206
  Args:
127
207
  - name: The name of the variable to unset.
@@ -135,8 +215,8 @@ class Variable(BaseModel):
135
215
  from prefect.variables import Variable
136
216
 
137
217
  @flow
138
- def my_flow():
139
- Variable.unset("my_var")
218
+ async def my_flow():
219
+ await Variable.aunset("my_var")
140
220
  ```
141
221
  """
142
222
  client, _ = get_or_create_client()
@@ -146,5 +226,34 @@ class Variable(BaseModel):
146
226
  except ObjectNotFound:
147
227
  return False
148
228
 
229
+ @classmethod
230
+ @async_dispatch(aunset)
231
+ def unset(cls, name: str) -> bool:
232
+ """
233
+ Unset a variable by name.
234
+
235
+ Args:
236
+ - name: The name of the variable to unset.
237
+
238
+ Returns `True` if the variable was deleted, `False` if the variable did not exist.
239
+
240
+ Example:
241
+ Unset a variable by name.
242
+ ```python
243
+ from prefect import flow
244
+ from prefect.variables import Variable
245
+
246
+ @flow
247
+ def my_flow():
248
+ Variable.unset("my_var")
249
+ ```
250
+ """
251
+ with get_client(sync_client=True) as client:
252
+ try:
253
+ client.delete_variable_by_name(name=name)
254
+ return True
255
+ except ObjectNotFound:
256
+ return False
257
+
149
258
 
150
259
  __getattr__ = getattr_migration(__name__)
prefect/workers/base.py CHANGED
@@ -53,6 +53,7 @@ from prefect.states import (
53
53
  Pending,
54
54
  exception_to_failed_state,
55
55
  )
56
+ from prefect.types import KeyValueLabels
56
57
  from prefect.utilities.dispatch import get_registry_for_type, register_base_type
57
58
  from prefect.utilities.engine import propose_state
58
59
  from prefect.utilities.services import critical_service_loop
@@ -988,7 +989,6 @@ class BaseWorker(abc.ABC):
988
989
  try:
989
990
  configuration = await self._get_configuration(flow_run)
990
991
  submitted_event = self._emit_flow_run_submitted_event(configuration)
991
- await self._give_worker_labels_to_flow_run(flow_run.id)
992
992
  result = await self.run(
993
993
  flow_run=flow_run,
994
994
  task_status=task_status,
@@ -1222,13 +1222,20 @@ class BaseWorker(abc.ABC):
1222
1222
  Give this worker's identifying labels to the specified flow run.
1223
1223
  """
1224
1224
  if self._cloud_client:
1225
- await self._cloud_client.update_flow_run_labels(
1226
- flow_run_id,
1227
- {
1228
- "prefect.worker.name": self.name,
1229
- "prefect.worker.type": self.type,
1230
- },
1231
- )
1225
+ labels: KeyValueLabels = {
1226
+ "prefect.worker.name": self.name,
1227
+ "prefect.worker.type": self.type,
1228
+ }
1229
+
1230
+ if self._work_pool:
1231
+ labels.update(
1232
+ {
1233
+ "prefect.work-pool.name": self._work_pool.name,
1234
+ "prefect.work-pool.id": str(self._work_pool.id),
1235
+ }
1236
+ )
1237
+
1238
+ await self._cloud_client.update_flow_run_labels(flow_run_id, labels)
1232
1239
 
1233
1240
  async def __aenter__(self):
1234
1241
  self._logger.debug("Entering worker context...")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: prefect-client
3
- Version: 3.1.5
3
+ Version: 3.1.7
4
4
  Summary: Workflow orchestration and management.
5
5
  Home-page: https://www.prefect.io
6
6
  Author: Prefect Technologies, Inc.
@@ -51,6 +51,7 @@ Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
51
51
  Requires-Dist: pydantic-settings>2.2.1
52
52
  Requires-Dist: python-dateutil<3.0.0,>=2.8.2
53
53
  Requires-Dist: python-slugify<9.0,>=5.0
54
+ Requires-Dist: python-socks[asyncio]<3.0,>=2.5.3
54
55
  Requires-Dist: pyyaml<7.0.0,>=5.4.1
55
56
  Requires-Dist: rfc3339-validator<0.2.0,>=0.1.4
56
57
  Requires-Dist: rich<14.0,>=11.0
@@ -165,7 +166,9 @@ Start with our [friendly tutorial](https://docs.prefect.io/tutorials) or explore
165
166
 
166
167
  ## Join the community
167
168
 
168
- Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system. The [Prefect Slack community](https://prefect.io/slack) is a fantastic place to learn more about Prefect, ask questions, or get help with workflow design. All community forums, including code contributions, issue discussions, and slack messages are subject to our [Code of Conduct](https://discourse.prefect.io/faq).
169
+ Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system.
170
+ The [Prefect Slack community](https://prefect.io/slack) is a fantastic place to learn more about Prefect, ask questions, or get help with workflow design.
171
+ All community forums, including code contributions, issue discussions, and Slack messages are subject to our [Code of Conduct](https://github.com/PrefectHQ/prefect/blob/main/CODE_OF_CONDUCT.md).
169
172
 
170
173
  ## Contribute
171
174