prefect-client 3.3.5.dev4__py3-none-any.whl → 3.3.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_build_info.py +3 -3
- prefect/_experimental/{bundles.py → bundles/__init__.py} +25 -9
- prefect/_experimental/bundles/execute.py +34 -0
- prefect/_versioning.py +180 -0
- prefect/cache_policies.py +4 -4
- prefect/client/orchestration/_deployments/client.py +49 -22
- prefect/concurrency/context.py +3 -1
- prefect/context.py +1 -3
- prefect/deployments/runner.py +22 -6
- prefect/events/clients.py +1 -1
- prefect/flows.py +101 -1
- prefect/futures.py +54 -21
- prefect/runner/runner.py +1 -1
- prefect/server/api/deployments.py +41 -3
- prefect/utilities/_ast.py +117 -0
- prefect/workers/base.py +28 -1
- {prefect_client-3.3.5.dev4.dist-info → prefect_client-3.3.6.dist-info}/METADATA +2 -2
- {prefect_client-3.3.5.dev4.dist-info → prefect_client-3.3.6.dist-info}/RECORD +20 -17
- {prefect_client-3.3.5.dev4.dist-info → prefect_client-3.3.6.dist-info}/WHEEL +0 -0
- {prefect_client-3.3.5.dev4.dist-info → prefect_client-3.3.6.dist-info}/licenses/LICENSE +0 -0
prefect/_build_info.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# Generated by versioningit
|
2
|
-
__version__ = "3.3.
|
3
|
-
__build_date__ = "2025-04-
|
4
|
-
__git_commit__ = "
|
2
|
+
__version__ = "3.3.6"
|
3
|
+
__build_date__ = "2025-04-24 19:25:40.923968+00:00"
|
4
|
+
__git_commit__ = "01441afa228e34271ccd1ec7cacc03a23a20ec30"
|
5
5
|
__dirty__ = False
|
@@ -11,7 +11,7 @@ import sys
|
|
11
11
|
from pathlib import Path
|
12
12
|
from typing import Any, TypedDict
|
13
13
|
|
14
|
-
import cloudpickle
|
14
|
+
import cloudpickle # pyright: ignore[reportMissingTypeStubs]
|
15
15
|
|
16
16
|
from prefect.client.schemas.objects import FlowRun
|
17
17
|
from prefect.context import SettingsContext, get_settings_context, serialize_context
|
@@ -22,12 +22,18 @@ from prefect.settings.context import get_current_settings
|
|
22
22
|
from prefect.settings.models.root import Settings
|
23
23
|
from prefect.utilities.slugify import slugify
|
24
24
|
|
25
|
-
|
26
|
-
import uv
|
25
|
+
from .execute import execute_bundle_from_file
|
27
26
|
|
28
|
-
|
29
|
-
|
30
|
-
|
27
|
+
|
28
|
+
def _get_uv_path() -> str:
|
29
|
+
try:
|
30
|
+
import uv
|
31
|
+
|
32
|
+
uv_path = uv.find_uv_bin()
|
33
|
+
except (ImportError, ModuleNotFoundError):
|
34
|
+
uv_path = "uv"
|
35
|
+
|
36
|
+
return uv_path
|
31
37
|
|
32
38
|
|
33
39
|
class SerializedBundle(TypedDict):
|
@@ -46,7 +52,7 @@ def _serialize_bundle_object(obj: Any) -> str:
|
|
46
52
|
"""
|
47
53
|
Serializes an object to a string.
|
48
54
|
"""
|
49
|
-
return base64.b64encode(gzip.compress(cloudpickle.dumps(obj))).decode()
|
55
|
+
return base64.b64encode(gzip.compress(cloudpickle.dumps(obj))).decode() # pyright: ignore[reportUnknownMemberType]
|
50
56
|
|
51
57
|
|
52
58
|
def _deserialize_bundle_object(serialized_obj: str) -> Any:
|
@@ -80,7 +86,7 @@ def create_bundle_for_flow_run(
|
|
80
86
|
"flow_run": flow_run.model_dump(mode="json"),
|
81
87
|
"dependencies": subprocess.check_output(
|
82
88
|
[
|
83
|
-
|
89
|
+
_get_uv_path(),
|
84
90
|
"pip",
|
85
91
|
"freeze",
|
86
92
|
# Exclude editable installs because we won't be able to install them in the execution environment
|
@@ -164,7 +170,7 @@ def execute_bundle_in_subprocess(
|
|
164
170
|
# Install dependencies if necessary
|
165
171
|
if dependencies := bundle.get("dependencies"):
|
166
172
|
subprocess.check_call(
|
167
|
-
[
|
173
|
+
[_get_uv_path(), "pip", "install", *dependencies.split("\n")],
|
168
174
|
# Copy the current environment to ensure we install into the correct venv
|
169
175
|
env=os.environ,
|
170
176
|
)
|
@@ -238,3 +244,13 @@ def convert_step_to_command(
|
|
238
244
|
command.extend(["--key", key])
|
239
245
|
|
240
246
|
return command
|
247
|
+
|
248
|
+
|
249
|
+
__all__ = [
|
250
|
+
"execute_bundle_from_file",
|
251
|
+
"convert_step_to_command",
|
252
|
+
"create_bundle_for_flow_run",
|
253
|
+
"extract_flow_from_bundle",
|
254
|
+
"execute_bundle_in_subprocess",
|
255
|
+
"SerializedBundle",
|
256
|
+
]
|
@@ -0,0 +1,34 @@
|
|
1
|
+
import json
|
2
|
+
|
3
|
+
import typer
|
4
|
+
|
5
|
+
from prefect.utilities.asyncutils import run_coro_as_sync
|
6
|
+
|
7
|
+
|
8
|
+
def execute_bundle_from_file(key: str):
|
9
|
+
"""
|
10
|
+
Loads a bundle from a file and executes it.
|
11
|
+
|
12
|
+
Args:
|
13
|
+
key: The key of the bundle to execute.
|
14
|
+
"""
|
15
|
+
with open(key, "r") as f:
|
16
|
+
bundle = json.load(f)
|
17
|
+
|
18
|
+
from prefect.runner.runner import Runner
|
19
|
+
|
20
|
+
run_coro_as_sync(Runner().execute_bundle(bundle))
|
21
|
+
|
22
|
+
|
23
|
+
def _execute_bundle_from_file(key: str = typer.Option(...)):
|
24
|
+
"""
|
25
|
+
Loads a bundle from a file and executes it.
|
26
|
+
|
27
|
+
Args:
|
28
|
+
key: The key of the bundle to execute.
|
29
|
+
"""
|
30
|
+
execute_bundle_from_file(key)
|
31
|
+
|
32
|
+
|
33
|
+
if __name__ == "__main__":
|
34
|
+
typer.run(_execute_bundle_from_file)
|
prefect/_versioning.py
ADDED
@@ -0,0 +1,180 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import os
|
4
|
+
from enum import Enum
|
5
|
+
from typing import Any, Callable, Coroutine, Dict, Literal, Optional
|
6
|
+
from urllib.parse import urlparse
|
7
|
+
|
8
|
+
from anyio import run_process
|
9
|
+
from pydantic import Field, model_validator
|
10
|
+
|
11
|
+
from prefect.client.schemas.objects import VersionInfo
|
12
|
+
|
13
|
+
|
14
|
+
class SimpleVersionInfo(VersionInfo):
|
15
|
+
type: Literal["prefect:simple"] = "prefect:simple"
|
16
|
+
version: str = Field(default="")
|
17
|
+
branch: Optional[str] = Field(default=None)
|
18
|
+
url: Optional[str] = Field(default=None)
|
19
|
+
|
20
|
+
|
21
|
+
class GithubVersionInfo(VersionInfo):
|
22
|
+
type: Literal["vcs:github"] = "vcs:github"
|
23
|
+
version: str
|
24
|
+
branch: str
|
25
|
+
url: str
|
26
|
+
repository: str
|
27
|
+
|
28
|
+
@model_validator(mode="after")
|
29
|
+
def validate_branch(self):
|
30
|
+
if not self.branch:
|
31
|
+
raise ValueError("branch is required when type is 'vcs:github'")
|
32
|
+
return self
|
33
|
+
|
34
|
+
|
35
|
+
class GitVersionInfo(VersionInfo):
|
36
|
+
type: Literal["vcs:git"] = "vcs:git"
|
37
|
+
version: str
|
38
|
+
branch: str
|
39
|
+
url: str
|
40
|
+
repository: str
|
41
|
+
|
42
|
+
|
43
|
+
async def get_github_version_info(
|
44
|
+
version: Optional[str] = None,
|
45
|
+
branch: Optional[str] = None,
|
46
|
+
repository: Optional[str] = None,
|
47
|
+
url: Optional[str] = None,
|
48
|
+
) -> GithubVersionInfo:
|
49
|
+
"""Create a GithubVersionInfo object from provided values or environment variables.
|
50
|
+
|
51
|
+
Args:
|
52
|
+
version: The commit SHA, falls back to GITHUB_SHA env var
|
53
|
+
branch: The git branch, falls back to GITHUB_REF_NAME env var
|
54
|
+
repository: The repository name, falls back to GITHUB_REPOSITORY env var
|
55
|
+
url: The repository URL, constructed from GITHUB_SERVER_URL/GITHUB_REPOSITORY if not provided
|
56
|
+
|
57
|
+
Returns:
|
58
|
+
A GithubVersionInfo
|
59
|
+
|
60
|
+
Raises:
|
61
|
+
ValueError: If any required fields cannot be determined
|
62
|
+
"""
|
63
|
+
version = version or os.getenv("GITHUB_SHA")
|
64
|
+
branch = branch or os.getenv("GITHUB_REF_NAME")
|
65
|
+
repository = repository or os.getenv("GITHUB_REPOSITORY")
|
66
|
+
url = url or f"{os.getenv('GITHUB_SERVER_URL')}/{repository}"
|
67
|
+
|
68
|
+
if not version:
|
69
|
+
raise ValueError("version is required - must be provided or set in GITHUB_SHA")
|
70
|
+
if not branch:
|
71
|
+
raise ValueError(
|
72
|
+
"branch is required - must be provided or set in GITHUB_REF_NAME"
|
73
|
+
)
|
74
|
+
if not repository:
|
75
|
+
raise ValueError(
|
76
|
+
"repository is required - must be provided or set in GITHUB_REPOSITORY"
|
77
|
+
)
|
78
|
+
|
79
|
+
return GithubVersionInfo(
|
80
|
+
type="vcs:github",
|
81
|
+
version=version,
|
82
|
+
branch=branch,
|
83
|
+
repository=repository,
|
84
|
+
url=url,
|
85
|
+
)
|
86
|
+
|
87
|
+
|
88
|
+
async def get_git_version_info(
|
89
|
+
version: Optional[str] = None,
|
90
|
+
branch: Optional[str] = None,
|
91
|
+
url: Optional[str] = None,
|
92
|
+
repository: Optional[str] = None,
|
93
|
+
) -> GitVersionInfo:
|
94
|
+
try:
|
95
|
+
if not version:
|
96
|
+
# Run git command and get stdout
|
97
|
+
result = await run_process(["git", "rev-parse", "HEAD"])
|
98
|
+
# Decode bytes to string and strip whitespace
|
99
|
+
version = result.stdout.decode().strip()
|
100
|
+
|
101
|
+
if not branch:
|
102
|
+
result = await run_process(["git", "rev-parse", "--abbrev-ref", "HEAD"])
|
103
|
+
branch = result.stdout.decode().strip()
|
104
|
+
|
105
|
+
if not repository:
|
106
|
+
result = await run_process(["git", "config", "--get", "remote.origin.url"])
|
107
|
+
remote_url = result.stdout.decode().strip()
|
108
|
+
|
109
|
+
# Extract just the repository name (last part of the path)
|
110
|
+
repo_url = urlparse(remote_url)
|
111
|
+
repository = repo_url.path.strip("/")
|
112
|
+
if repository.endswith(".git"):
|
113
|
+
repository = repository[:-4]
|
114
|
+
|
115
|
+
if not url and repository:
|
116
|
+
# Use the full remote URL as the URL
|
117
|
+
result = await run_process(["git", "config", "--get", "remote.origin.url"])
|
118
|
+
url = result.stdout.decode().strip()
|
119
|
+
except Exception as e:
|
120
|
+
raise ValueError(
|
121
|
+
f"Error getting git version info: {e}. You may not be in a git repository."
|
122
|
+
)
|
123
|
+
|
124
|
+
if not url:
|
125
|
+
raise ValueError("Could not determine git repository URL")
|
126
|
+
|
127
|
+
return GitVersionInfo(
|
128
|
+
type="vcs:git", version=version, branch=branch, url=url, repository=repository
|
129
|
+
)
|
130
|
+
|
131
|
+
|
132
|
+
class VersionType(str, Enum):
|
133
|
+
SIMPLE = "prefect:simple"
|
134
|
+
GITHUB = "vcs:github"
|
135
|
+
GIT = "vcs:git"
|
136
|
+
DOCKER = "container:docker"
|
137
|
+
|
138
|
+
|
139
|
+
async def get_inferred_version_info(
|
140
|
+
version_type: Optional[str] = None,
|
141
|
+
) -> VersionInfo | None:
|
142
|
+
"""
|
143
|
+
Attempts to infer version information from the environment.
|
144
|
+
|
145
|
+
Args:
|
146
|
+
version_type: Optional type of version info to get. If provided, only that
|
147
|
+
type will be attempted.
|
148
|
+
|
149
|
+
Returns:
|
150
|
+
VersionInfo: The inferred version information
|
151
|
+
|
152
|
+
Raises:
|
153
|
+
ValueError: If unable to infer version info from any source
|
154
|
+
"""
|
155
|
+
# Map version types to their getter functions
|
156
|
+
type_to_getter: Dict[str, Callable[..., Coroutine[Any, Any, Any]]] = {
|
157
|
+
VersionType.GITHUB: get_github_version_info,
|
158
|
+
VersionType.GIT: get_git_version_info,
|
159
|
+
}
|
160
|
+
|
161
|
+
# Default order of getters to try
|
162
|
+
default_getters = [
|
163
|
+
get_github_version_info,
|
164
|
+
get_git_version_info,
|
165
|
+
]
|
166
|
+
|
167
|
+
if version_type:
|
168
|
+
if version_type not in type_to_getter:
|
169
|
+
raise ValueError(f"Unknown version type: {version_type}")
|
170
|
+
getters = [type_to_getter[version_type]]
|
171
|
+
else:
|
172
|
+
getters = default_getters
|
173
|
+
|
174
|
+
for getter in getters:
|
175
|
+
try:
|
176
|
+
return await getter()
|
177
|
+
except ValueError:
|
178
|
+
continue
|
179
|
+
|
180
|
+
return None
|
prefect/cache_policies.py
CHANGED
@@ -33,9 +33,9 @@ def _register_stable_transforms() -> None:
|
|
33
33
|
so that cache keys that utilize them are deterministic across invocations.
|
34
34
|
"""
|
35
35
|
try:
|
36
|
-
import pandas as pd
|
36
|
+
import pandas as pd # pyright: ignore
|
37
37
|
|
38
|
-
STABLE_TRANSFORMS[pd.DataFrame] = lambda df: [
|
38
|
+
STABLE_TRANSFORMS[pd.DataFrame] = lambda df: [ # pyright: ignore
|
39
39
|
df[col] for col in sorted(df.columns)
|
40
40
|
]
|
41
41
|
except (ImportError, ModuleNotFoundError):
|
@@ -183,7 +183,7 @@ class CompoundCachePolicy(CachePolicy):
|
|
183
183
|
Any keys that return `None` will be ignored.
|
184
184
|
"""
|
185
185
|
|
186
|
-
policies: list[CachePolicy] = field(default_factory=
|
186
|
+
policies: list[CachePolicy] = field(default_factory=lambda: [])
|
187
187
|
|
188
188
|
def __post_init__(self) -> None:
|
189
189
|
# flatten any CompoundCachePolicies
|
@@ -349,7 +349,7 @@ class Inputs(CachePolicy):
|
|
349
349
|
Policy that computes a cache key based on a hash of the runtime inputs provided to the task..
|
350
350
|
"""
|
351
351
|
|
352
|
-
exclude: list[str] = field(default_factory=
|
352
|
+
exclude: list[str] = field(default_factory=lambda: [])
|
353
353
|
|
354
354
|
def compute_key(
|
355
355
|
self,
|
@@ -153,13 +153,14 @@ class DeploymentClient(BaseClient):
|
|
153
153
|
if getattr(deployment_create, field) is None:
|
154
154
|
exclude.add(field)
|
155
155
|
|
156
|
-
|
156
|
+
payload = deployment_create.model_dump(mode="json", exclude=exclude)
|
157
|
+
if deployment_create.version_info:
|
158
|
+
payload["version_info"] = deployment_create.version_info.model_dump(
|
159
|
+
mode="json"
|
160
|
+
)
|
161
|
+
|
162
|
+
response = self.request("POST", "/deployments/", json=payload)
|
157
163
|
|
158
|
-
response = self.request(
|
159
|
-
"POST",
|
160
|
-
"/deployments/",
|
161
|
-
json=json,
|
162
|
-
)
|
163
164
|
deployment_id = response.json().get("id")
|
164
165
|
if not deployment_id:
|
165
166
|
raise RequestError(f"Malformed response: {response}")
|
@@ -179,15 +180,28 @@ class DeploymentClient(BaseClient):
|
|
179
180
|
deployment_id: UUID,
|
180
181
|
deployment: "DeploymentUpdate",
|
181
182
|
) -> None:
|
183
|
+
exclude_if_none = [
|
184
|
+
"version_info",
|
185
|
+
]
|
186
|
+
|
187
|
+
exclude = {"name", "flow_name", "triggers"}
|
188
|
+
for field in exclude_if_none:
|
189
|
+
if getattr(deployment, field) is None:
|
190
|
+
exclude.add(field)
|
191
|
+
|
192
|
+
payload = deployment.model_dump(
|
193
|
+
mode="json",
|
194
|
+
exclude_unset=True,
|
195
|
+
exclude=exclude,
|
196
|
+
)
|
197
|
+
if deployment.version_info:
|
198
|
+
payload["version_info"] = deployment.version_info.model_dump(mode="json")
|
199
|
+
|
182
200
|
self.request(
|
183
201
|
"PATCH",
|
184
202
|
"/deployments/{id}",
|
185
203
|
path_params={"id": deployment_id},
|
186
|
-
json=
|
187
|
-
mode="json",
|
188
|
-
exclude_unset=True,
|
189
|
-
exclude={"name", "flow_name", "triggers"},
|
190
|
-
),
|
204
|
+
json=payload,
|
191
205
|
)
|
192
206
|
|
193
207
|
def _create_deployment_from_schema(self, schema: "DeploymentCreate") -> UUID:
|
@@ -733,13 +747,13 @@ class DeploymentAsyncClient(BaseAsyncClient):
|
|
733
747
|
if getattr(deployment_create, field) is None:
|
734
748
|
exclude.add(field)
|
735
749
|
|
736
|
-
|
750
|
+
payload = deployment_create.model_dump(mode="json", exclude=exclude)
|
751
|
+
if deployment_create.version_info:
|
752
|
+
payload["version_info"] = deployment_create.version_info.model_dump(
|
753
|
+
mode="json"
|
754
|
+
)
|
737
755
|
|
738
|
-
response = await self.request(
|
739
|
-
"POST",
|
740
|
-
"/deployments/",
|
741
|
-
json=json,
|
742
|
-
)
|
756
|
+
response = await self.request("POST", "/deployments/", json=payload)
|
743
757
|
deployment_id = response.json().get("id")
|
744
758
|
if not deployment_id:
|
745
759
|
raise RequestError(f"Malformed response: {response}")
|
@@ -761,15 +775,28 @@ class DeploymentAsyncClient(BaseAsyncClient):
|
|
761
775
|
deployment_id: UUID,
|
762
776
|
deployment: "DeploymentUpdate",
|
763
777
|
) -> None:
|
778
|
+
exclude_if_none = [
|
779
|
+
"version_info",
|
780
|
+
]
|
781
|
+
|
782
|
+
exclude = {"name", "flow_name", "triggers"}
|
783
|
+
for field in exclude_if_none:
|
784
|
+
if getattr(deployment, field) is None:
|
785
|
+
exclude.add(field)
|
786
|
+
|
787
|
+
payload = deployment.model_dump(
|
788
|
+
mode="json",
|
789
|
+
exclude_unset=True,
|
790
|
+
exclude=exclude,
|
791
|
+
)
|
792
|
+
if deployment.version_info:
|
793
|
+
payload["version_info"] = deployment.version_info.model_dump(mode="json")
|
794
|
+
|
764
795
|
await self.request(
|
765
796
|
"PATCH",
|
766
797
|
"/deployments/{id}",
|
767
798
|
path_params={"id": deployment_id},
|
768
|
-
json=
|
769
|
-
mode="json",
|
770
|
-
exclude_unset=True,
|
771
|
-
exclude={"name", "flow_name", "triggers"},
|
772
|
-
),
|
799
|
+
json=payload,
|
773
800
|
)
|
774
801
|
|
775
802
|
async def _create_deployment_from_schema(self, schema: "DeploymentCreate") -> UUID:
|
prefect/concurrency/context.py
CHANGED
@@ -13,7 +13,9 @@ class ConcurrencyContext(ContextModel):
|
|
13
13
|
# Track the slots that have been acquired but were not able to be released
|
14
14
|
# due to cancellation or some other error. These slots are released when
|
15
15
|
# the context manager exits.
|
16
|
-
cleanup_slots: list[tuple[list[str], int, float]] = Field(
|
16
|
+
cleanup_slots: list[tuple[list[str], int, float]] = Field(
|
17
|
+
default_factory=lambda: []
|
18
|
+
)
|
17
19
|
|
18
20
|
def __exit__(self, *exc_info: Any) -> None:
|
19
21
|
if self.cleanup_slots:
|
prefect/context.py
CHANGED
@@ -18,8 +18,6 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Un
|
|
18
18
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
19
19
|
from typing_extensions import Self
|
20
20
|
|
21
|
-
import prefect.logging
|
22
|
-
import prefect.logging.configuration
|
23
21
|
import prefect.settings
|
24
22
|
import prefect.types._datetime
|
25
23
|
from prefect._internal.compatibility.migration import getattr_migration
|
@@ -128,7 +126,7 @@ class ContextModel(BaseModel):
|
|
128
126
|
def __init__(self, **kwargs: Any) -> None: ...
|
129
127
|
|
130
128
|
# The context variable for storing data must be defined by the child class
|
131
|
-
__var__: ClassVar[ContextVar[
|
129
|
+
__var__: ClassVar[ContextVar[Any]]
|
132
130
|
_token: Optional[Token[Self]] = PrivateAttr(None)
|
133
131
|
model_config: ClassVar[ConfigDict] = ConfigDict(
|
134
132
|
arbitrary_types_allowed=True,
|
prefect/deployments/runner.py
CHANGED
@@ -29,6 +29,8 @@ Example:
|
|
29
29
|
|
30
30
|
"""
|
31
31
|
|
32
|
+
from __future__ import annotations
|
33
|
+
|
32
34
|
import importlib
|
33
35
|
import tempfile
|
34
36
|
from datetime import datetime, timedelta
|
@@ -63,6 +65,7 @@ from prefect.client.schemas.filters import WorkerFilter, WorkerFilterStatus
|
|
63
65
|
from prefect.client.schemas.objects import (
|
64
66
|
ConcurrencyLimitConfig,
|
65
67
|
ConcurrencyOptions,
|
68
|
+
VersionInfo,
|
66
69
|
)
|
67
70
|
from prefect.client.schemas.schedules import (
|
68
71
|
SCHEDULE_TYPES,
|
@@ -281,7 +284,10 @@ class RunnerDeployment(BaseModel):
|
|
281
284
|
return reconcile_schedules_runner(values)
|
282
285
|
|
283
286
|
async def _create(
|
284
|
-
self,
|
287
|
+
self,
|
288
|
+
work_pool_name: Optional[str] = None,
|
289
|
+
image: Optional[str] = None,
|
290
|
+
version_info: VersionInfo | None = None,
|
285
291
|
) -> UUID:
|
286
292
|
work_pool_name = work_pool_name or self.work_pool_name
|
287
293
|
|
@@ -312,6 +318,7 @@ class RunnerDeployment(BaseModel):
|
|
312
318
|
work_queue_name=self.work_queue_name,
|
313
319
|
work_pool_name=work_pool_name,
|
314
320
|
version=self.version,
|
321
|
+
version_info=version_info,
|
315
322
|
paused=self.paused,
|
316
323
|
schedules=self.schedules,
|
317
324
|
concurrency_limit=self.concurrency_limit,
|
@@ -367,7 +374,12 @@ class RunnerDeployment(BaseModel):
|
|
367
374
|
|
368
375
|
return deployment_id
|
369
376
|
|
370
|
-
async def _update(
|
377
|
+
async def _update(
|
378
|
+
self,
|
379
|
+
deployment_id: UUID,
|
380
|
+
client: PrefectClient,
|
381
|
+
version_info: VersionInfo | None,
|
382
|
+
):
|
371
383
|
parameter_openapi_schema = self._parameter_openapi_schema.model_dump(
|
372
384
|
exclude_unset=True
|
373
385
|
)
|
@@ -388,6 +400,7 @@ class RunnerDeployment(BaseModel):
|
|
388
400
|
deployment_id,
|
389
401
|
deployment=DeploymentUpdate(
|
390
402
|
**update_payload,
|
403
|
+
version_info=version_info,
|
391
404
|
parameter_openapi_schema=parameter_openapi_schema,
|
392
405
|
),
|
393
406
|
)
|
@@ -428,7 +441,10 @@ class RunnerDeployment(BaseModel):
|
|
428
441
|
|
429
442
|
@sync_compatible
|
430
443
|
async def apply(
|
431
|
-
self,
|
444
|
+
self,
|
445
|
+
work_pool_name: Optional[str] = None,
|
446
|
+
image: Optional[str] = None,
|
447
|
+
version_info: VersionInfo | None = None,
|
432
448
|
) -> UUID:
|
433
449
|
"""
|
434
450
|
Registers this deployment with the API and returns the deployment's ID.
|
@@ -439,7 +455,7 @@ class RunnerDeployment(BaseModel):
|
|
439
455
|
image: The registry, name, and tag of the Docker image to
|
440
456
|
use for this deployment. Only used when the deployment is
|
441
457
|
deployed to a work pool.
|
442
|
-
|
458
|
+
version_info: Version information for the deployment.
|
443
459
|
Returns:
|
444
460
|
The ID of the created deployment.
|
445
461
|
"""
|
@@ -448,13 +464,13 @@ class RunnerDeployment(BaseModel):
|
|
448
464
|
try:
|
449
465
|
deployment = await client.read_deployment_by_name(self.full_name)
|
450
466
|
except ObjectNotFound:
|
451
|
-
return await self._create(work_pool_name, image)
|
467
|
+
return await self._create(work_pool_name, image, version_info)
|
452
468
|
else:
|
453
469
|
if image:
|
454
470
|
self.job_variables["image"] = image
|
455
471
|
if work_pool_name:
|
456
472
|
self.work_pool_name = work_pool_name
|
457
|
-
return await self._update(deployment.id, client)
|
473
|
+
return await self._update(deployment.id, client, version_info)
|
458
474
|
|
459
475
|
async def _create_slas(self, deployment_id: UUID, client: PrefectClient):
|
460
476
|
if not isinstance(self._sla, list):
|
prefect/events/clients.py
CHANGED
prefect/flows.py
CHANGED
@@ -65,7 +65,7 @@ from prefect.exceptions import (
|
|
65
65
|
UnspecifiedFlowError,
|
66
66
|
)
|
67
67
|
from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
|
68
|
-
from prefect.futures import PrefectFuture
|
68
|
+
from prefect.futures import PrefectFlowRunFuture, PrefectFuture
|
69
69
|
from prefect.logging import get_logger
|
70
70
|
from prefect.logging.loggers import flow_run_logger
|
71
71
|
from prefect.results import ResultSerializer, ResultStorage
|
@@ -2104,6 +2104,106 @@ class InfrastructureBoundFlow(Flow[P, R]):
|
|
2104
2104
|
)
|
2105
2105
|
)
|
2106
2106
|
|
2107
|
+
def submit(self, *args: P.args, **kwargs: P.kwargs) -> PrefectFlowRunFuture[R]:
|
2108
|
+
"""
|
2109
|
+
EXPERIMENTAL: This method is experimental and may be removed or changed in future
|
2110
|
+
releases.
|
2111
|
+
|
2112
|
+
Submit the flow to run on remote infrastructure.
|
2113
|
+
|
2114
|
+
Args:
|
2115
|
+
*args: Positional arguments to pass to the flow.
|
2116
|
+
**kwargs: Keyword arguments to pass to the flow.
|
2117
|
+
|
2118
|
+
Returns:
|
2119
|
+
A `PrefectFlowRunFuture` that can be used to retrieve the result of the flow run.
|
2120
|
+
|
2121
|
+
Examples:
|
2122
|
+
Submit a flow to run on Kubernetes:
|
2123
|
+
|
2124
|
+
```python
|
2125
|
+
from prefect import flow
|
2126
|
+
from prefect_kubernetes.experimental import kubernetes
|
2127
|
+
|
2128
|
+
@kubernetes(work_pool="my-kubernetes-work-pool")
|
2129
|
+
@flow
|
2130
|
+
def my_flow(x: int, y: int):
|
2131
|
+
return x + y
|
2132
|
+
|
2133
|
+
future = my_flow.submit(x=1, y=2)
|
2134
|
+
result = future.result()
|
2135
|
+
print(result)
|
2136
|
+
```
|
2137
|
+
"""
|
2138
|
+
|
2139
|
+
async def submit_func():
|
2140
|
+
async with self.worker_cls(work_pool_name=self.work_pool) as worker:
|
2141
|
+
parameters = get_call_parameters(self, args, kwargs)
|
2142
|
+
return await worker.submit(
|
2143
|
+
flow=self,
|
2144
|
+
parameters=parameters,
|
2145
|
+
job_variables=self.job_variables,
|
2146
|
+
)
|
2147
|
+
|
2148
|
+
return run_coro_as_sync(submit_func())
|
2149
|
+
|
2150
|
+
def with_options(
|
2151
|
+
self,
|
2152
|
+
*,
|
2153
|
+
name: Optional[str] = None,
|
2154
|
+
version: Optional[str] = None,
|
2155
|
+
retries: Optional[int] = None,
|
2156
|
+
retry_delay_seconds: Optional[Union[int, float]] = None,
|
2157
|
+
description: Optional[str] = None,
|
2158
|
+
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
2159
|
+
task_runner: Union[
|
2160
|
+
Type[TaskRunner[PrefectFuture[Any]]], TaskRunner[PrefectFuture[Any]], None
|
2161
|
+
] = None,
|
2162
|
+
timeout_seconds: Union[int, float, None] = None,
|
2163
|
+
validate_parameters: Optional[bool] = None,
|
2164
|
+
persist_result: Optional[bool] = NotSet, # type: ignore
|
2165
|
+
result_storage: Optional[ResultStorage] = NotSet, # type: ignore
|
2166
|
+
result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
|
2167
|
+
cache_result_in_memory: Optional[bool] = None,
|
2168
|
+
log_prints: Optional[bool] = NotSet, # type: ignore
|
2169
|
+
on_completion: Optional[list[FlowStateHook[P, R]]] = None,
|
2170
|
+
on_failure: Optional[list[FlowStateHook[P, R]]] = None,
|
2171
|
+
on_cancellation: Optional[list[FlowStateHook[P, R]]] = None,
|
2172
|
+
on_crashed: Optional[list[FlowStateHook[P, R]]] = None,
|
2173
|
+
on_running: Optional[list[FlowStateHook[P, R]]] = None,
|
2174
|
+
job_variables: Optional[dict[str, Any]] = None,
|
2175
|
+
) -> "InfrastructureBoundFlow[P, R]":
|
2176
|
+
new_flow = super().with_options(
|
2177
|
+
name=name,
|
2178
|
+
version=version,
|
2179
|
+
retries=retries,
|
2180
|
+
retry_delay_seconds=retry_delay_seconds,
|
2181
|
+
description=description,
|
2182
|
+
flow_run_name=flow_run_name,
|
2183
|
+
task_runner=task_runner,
|
2184
|
+
timeout_seconds=timeout_seconds,
|
2185
|
+
validate_parameters=validate_parameters,
|
2186
|
+
persist_result=persist_result,
|
2187
|
+
result_storage=result_storage,
|
2188
|
+
result_serializer=result_serializer,
|
2189
|
+
cache_result_in_memory=cache_result_in_memory,
|
2190
|
+
log_prints=log_prints,
|
2191
|
+
on_completion=on_completion,
|
2192
|
+
on_failure=on_failure,
|
2193
|
+
on_cancellation=on_cancellation,
|
2194
|
+
on_crashed=on_crashed,
|
2195
|
+
on_running=on_running,
|
2196
|
+
)
|
2197
|
+
new_infrastructure_bound_flow = bind_flow_to_infrastructure(
|
2198
|
+
new_flow,
|
2199
|
+
self.work_pool,
|
2200
|
+
self.worker_cls,
|
2201
|
+
job_variables=job_variables
|
2202
|
+
if job_variables is not None
|
2203
|
+
else self.job_variables,
|
2204
|
+
)
|
2205
|
+
return new_infrastructure_bound_flow
|
2206
|
+
|
2107
2207
|
|
2108
2208
|
def bind_flow_to_infrastructure(
|
2109
2209
|
flow: Flow[P, R],
|
prefect/futures.py
CHANGED
@@ -612,19 +612,44 @@ def resolve_futures_to_states(
|
|
612
612
|
|
613
613
|
Unsupported object types will be returned without modification.
|
614
614
|
"""
|
615
|
-
futures: set[PrefectFuture[R]] = set()
|
616
615
|
|
617
|
-
def
|
618
|
-
|
619
|
-
|
620
|
-
# Expressions inside quotes should not be traversed
|
621
|
-
if isinstance(context.get("annotation"), quote):
|
622
|
-
raise StopVisiting()
|
616
|
+
def _resolve_state(future: PrefectFuture[R]):
|
617
|
+
future.wait()
|
618
|
+
return future.state
|
623
619
|
|
624
|
-
|
625
|
-
|
620
|
+
return _resolve_futures(
|
621
|
+
expr,
|
622
|
+
resolve_fn=_resolve_state,
|
623
|
+
)
|
626
624
|
|
627
|
-
|
625
|
+
|
626
|
+
def resolve_futures_to_results(
|
627
|
+
expr: PrefectFuture[R] | Any,
|
628
|
+
) -> Any:
|
629
|
+
"""
|
630
|
+
Given a Python built-in collection, recursively find `PrefectFutures` and build a
|
631
|
+
new collection with the same structure with futures resolved to their final results.
|
632
|
+
Resolving futures to their final result may wait for execution to complete.
|
633
|
+
|
634
|
+
Unsupported object types will be returned without modification.
|
635
|
+
"""
|
636
|
+
|
637
|
+
def _resolve_result(future: PrefectFuture[R]) -> Any:
|
638
|
+
future.wait()
|
639
|
+
if future.state.is_completed():
|
640
|
+
return future.result()
|
641
|
+
else:
|
642
|
+
raise Exception("At least one result did not complete successfully")
|
643
|
+
|
644
|
+
return _resolve_futures(expr, resolve_fn=_resolve_result)
|
645
|
+
|
646
|
+
|
647
|
+
def _resolve_futures(
|
648
|
+
expr: PrefectFuture[R] | Any,
|
649
|
+
resolve_fn: Callable[[PrefectFuture[R]], Any],
|
650
|
+
) -> Any:
|
651
|
+
"""Helper function to resolve PrefectFutures in a collection."""
|
652
|
+
futures: set[PrefectFuture[R]] = set()
|
628
653
|
|
629
654
|
visit_collection(
|
630
655
|
expr,
|
@@ -633,31 +658,39 @@ def resolve_futures_to_states(
|
|
633
658
|
context={},
|
634
659
|
)
|
635
660
|
|
636
|
-
#
|
661
|
+
# If no futures were found, return the original expression
|
637
662
|
if not futures:
|
638
663
|
return expr
|
639
664
|
|
640
|
-
#
|
641
|
-
|
642
|
-
for future in futures:
|
643
|
-
future.wait()
|
644
|
-
states.append(future.state)
|
645
|
-
|
646
|
-
states_by_future = dict(zip(futures, states))
|
665
|
+
# Resolve each future using the provided resolve function
|
666
|
+
resolved_values = {future: resolve_fn(future) for future in futures}
|
647
667
|
|
648
|
-
def
|
668
|
+
def replace_futures(expr: Any, context: Any) -> Any:
|
649
669
|
# Expressions inside quotes should not be modified
|
650
670
|
if isinstance(context.get("annotation"), quote):
|
651
671
|
raise StopVisiting()
|
652
672
|
|
653
673
|
if isinstance(expr, PrefectFuture):
|
654
|
-
return
|
674
|
+
return resolved_values[expr]
|
655
675
|
else:
|
656
676
|
return expr
|
657
677
|
|
658
678
|
return visit_collection(
|
659
679
|
expr,
|
660
|
-
visit_fn=
|
680
|
+
visit_fn=replace_futures,
|
661
681
|
return_data=True,
|
662
682
|
context={},
|
663
683
|
)
|
684
|
+
|
685
|
+
|
686
|
+
def _collect_futures(
|
687
|
+
futures: set[PrefectFuture[R]], expr: Any | PrefectFuture[R], context: Any
|
688
|
+
) -> Any | PrefectFuture[R]:
|
689
|
+
# Expressions inside quotes should not be traversed
|
690
|
+
if isinstance(context.get("annotation"), quote):
|
691
|
+
raise StopVisiting()
|
692
|
+
|
693
|
+
if isinstance(expr, PrefectFuture):
|
694
|
+
futures.add(expr)
|
695
|
+
|
696
|
+
return expr
|
prefect/runner/runner.py
CHANGED
@@ -781,7 +781,7 @@ class Runner:
|
|
781
781
|
if command is None:
|
782
782
|
runner_command = [get_sys_executable(), "-m", "prefect.engine"]
|
783
783
|
else:
|
784
|
-
runner_command = shlex.split(command)
|
784
|
+
runner_command = shlex.split(command, posix=(os.name != "nt"))
|
785
785
|
|
786
786
|
flow_run_logger = self._get_flow_run_logger(flow_run)
|
787
787
|
|
@@ -98,9 +98,31 @@ async def create_deployment(
|
|
98
98
|
)
|
99
99
|
|
100
100
|
# hydrate the input model into a full model
|
101
|
-
deployment_dict = deployment.model_dump(
|
102
|
-
exclude={"work_pool_name"},
|
101
|
+
deployment_dict: dict = deployment.model_dump(
|
102
|
+
exclude={"work_pool_name"},
|
103
|
+
exclude_unset=True,
|
103
104
|
)
|
105
|
+
|
106
|
+
requested_concurrency_limit = deployment_dict.pop(
|
107
|
+
"global_concurrency_limit_id", "unset"
|
108
|
+
)
|
109
|
+
if requested_concurrency_limit != "unset":
|
110
|
+
if requested_concurrency_limit:
|
111
|
+
concurrency_limit = (
|
112
|
+
await models.concurrency_limits_v2.read_concurrency_limit(
|
113
|
+
session=session,
|
114
|
+
concurrency_limit_id=requested_concurrency_limit,
|
115
|
+
)
|
116
|
+
)
|
117
|
+
|
118
|
+
if not concurrency_limit:
|
119
|
+
raise HTTPException(
|
120
|
+
status_code=status.HTTP_404_NOT_FOUND,
|
121
|
+
detail="Concurrency limit not found",
|
122
|
+
)
|
123
|
+
|
124
|
+
deployment_dict["concurrency_limit_id"] = requested_concurrency_limit
|
125
|
+
|
104
126
|
if deployment.work_pool_name and deployment.work_queue_name:
|
105
127
|
# If a specific pool name/queue name combination was provided, get the
|
106
128
|
# ID for that work pool queue.
|
@@ -300,8 +322,24 @@ async def update_deployment(
|
|
300
322
|
detail="Invalid schema: Unable to validate schema with circular references.",
|
301
323
|
)
|
302
324
|
|
325
|
+
if deployment.global_concurrency_limit_id:
|
326
|
+
concurrency_limit = (
|
327
|
+
await models.concurrency_limits_v2.read_concurrency_limit(
|
328
|
+
session=session,
|
329
|
+
concurrency_limit_id=deployment.global_concurrency_limit_id,
|
330
|
+
)
|
331
|
+
)
|
332
|
+
|
333
|
+
if not concurrency_limit:
|
334
|
+
raise HTTPException(
|
335
|
+
status_code=status.HTTP_404_NOT_FOUND,
|
336
|
+
detail="Concurrency limit not found",
|
337
|
+
)
|
338
|
+
|
303
339
|
result = await models.deployments.update_deployment(
|
304
|
-
session=session,
|
340
|
+
session=session,
|
341
|
+
deployment_id=deployment_id,
|
342
|
+
deployment=deployment,
|
305
343
|
)
|
306
344
|
|
307
345
|
for schedule in schedules_to_patch:
|
@@ -0,0 +1,117 @@
|
|
1
|
+
import ast
|
2
|
+
import math
|
3
|
+
from typing import TYPE_CHECKING, Literal
|
4
|
+
|
5
|
+
import anyio
|
6
|
+
from typing_extensions import TypeAlias
|
7
|
+
|
8
|
+
from prefect.logging.loggers import get_logger
|
9
|
+
from prefect.settings import get_current_settings
|
10
|
+
from prefect.utilities.asyncutils import LazySemaphore
|
11
|
+
from prefect.utilities.filesystem import get_open_file_limit
|
12
|
+
|
13
|
+
# Only allow half of the open file limit to be open at once to allow for other
|
14
|
+
# actors to open files.
|
15
|
+
OPEN_FILE_SEMAPHORE = LazySemaphore(lambda: math.floor(get_open_file_limit() * 0.5))
|
16
|
+
|
17
|
+
# this potentially could be a TypedDict, but you
|
18
|
+
# need some way to convince the type checker that
|
19
|
+
# Literal["flow_name", "task_name"] are being provided
|
20
|
+
DecoratedFnMetadata: TypeAlias = dict[str, str]
|
21
|
+
|
22
|
+
|
23
|
+
async def find_prefect_decorated_functions_in_file(
|
24
|
+
path: anyio.Path, decorator_module: str, decorator_name: Literal["flow", "task"]
|
25
|
+
) -> list[DecoratedFnMetadata]:
|
26
|
+
logger = get_logger()
|
27
|
+
decorator_name_key = f"{decorator_name}_name"
|
28
|
+
decorated_functions: list[DecoratedFnMetadata] = []
|
29
|
+
|
30
|
+
async with OPEN_FILE_SEMAPHORE:
|
31
|
+
try:
|
32
|
+
async with await anyio.open_file(path) as f:
|
33
|
+
try:
|
34
|
+
tree = ast.parse(await f.read())
|
35
|
+
except SyntaxError:
|
36
|
+
if get_current_settings().debug_mode:
|
37
|
+
logger.debug(
|
38
|
+
f"Could not parse {path} as a Python file. Skipping."
|
39
|
+
)
|
40
|
+
return decorated_functions
|
41
|
+
except Exception as exc:
|
42
|
+
if get_current_settings().debug_mode:
|
43
|
+
logger.debug(f"Could not open {path}: {exc}. Skipping.")
|
44
|
+
return decorated_functions
|
45
|
+
|
46
|
+
for node in ast.walk(tree):
|
47
|
+
if isinstance(
|
48
|
+
node,
|
49
|
+
(
|
50
|
+
ast.FunctionDef,
|
51
|
+
ast.AsyncFunctionDef,
|
52
|
+
),
|
53
|
+
):
|
54
|
+
for decorator in node.decorator_list:
|
55
|
+
# handles @decorator_name
|
56
|
+
is_name_match = (
|
57
|
+
isinstance(decorator, ast.Name) and decorator.id == decorator_name
|
58
|
+
)
|
59
|
+
# handles @decorator_name()
|
60
|
+
is_func_name_match = (
|
61
|
+
isinstance(decorator, ast.Call)
|
62
|
+
and isinstance(decorator.func, ast.Name)
|
63
|
+
and decorator.func.id == decorator_name
|
64
|
+
)
|
65
|
+
# handles @decorator_module.decorator_name
|
66
|
+
is_module_attribute_match = (
|
67
|
+
isinstance(decorator, ast.Attribute)
|
68
|
+
and isinstance(decorator.value, ast.Name)
|
69
|
+
and decorator.value.id == decorator_module
|
70
|
+
and decorator.attr == decorator_name
|
71
|
+
)
|
72
|
+
# handles @decorator_module.decorator_name()
|
73
|
+
is_module_attribute_func_match = (
|
74
|
+
isinstance(decorator, ast.Call)
|
75
|
+
and isinstance(decorator.func, ast.Attribute)
|
76
|
+
and decorator.func.attr == decorator_name
|
77
|
+
and isinstance(decorator.func.value, ast.Name)
|
78
|
+
and decorator.func.value.id == decorator_module
|
79
|
+
)
|
80
|
+
if is_name_match or is_module_attribute_match:
|
81
|
+
decorated_functions.append(
|
82
|
+
{
|
83
|
+
decorator_name_key: node.name,
|
84
|
+
"function_name": node.name,
|
85
|
+
"filepath": str(path),
|
86
|
+
}
|
87
|
+
)
|
88
|
+
if is_func_name_match or is_module_attribute_func_match:
|
89
|
+
name_kwarg_node = None
|
90
|
+
if TYPE_CHECKING:
|
91
|
+
assert isinstance(decorator, ast.Call)
|
92
|
+
for kw in decorator.keywords:
|
93
|
+
if kw.arg == "name":
|
94
|
+
name_kwarg_node = kw
|
95
|
+
break
|
96
|
+
if name_kwarg_node is not None and isinstance(
|
97
|
+
name_kwarg_node.value, ast.Constant
|
98
|
+
):
|
99
|
+
decorated_fn_name = name_kwarg_node.value.value
|
100
|
+
else:
|
101
|
+
decorated_fn_name = node.name
|
102
|
+
decorated_functions.append(
|
103
|
+
{
|
104
|
+
decorator_name_key: decorated_fn_name,
|
105
|
+
"function_name": node.name,
|
106
|
+
"filepath": str(path),
|
107
|
+
}
|
108
|
+
)
|
109
|
+
return decorated_functions
|
110
|
+
|
111
|
+
|
112
|
+
async def find_flow_functions_in_file(path: anyio.Path) -> list[DecoratedFnMetadata]:
|
113
|
+
return await find_prefect_decorated_functions_in_file(path, "prefect", "flow")
|
114
|
+
|
115
|
+
|
116
|
+
async def find_task_functions_in_file(path: anyio.Path) -> list[DecoratedFnMetadata]:
|
117
|
+
return await find_prefect_decorated_functions_in_file(path, "prefect", "task")
|
prefect/workers/base.py
CHANGED
@@ -48,6 +48,7 @@ from prefect.client.schemas.objects import (
|
|
48
48
|
WorkPool,
|
49
49
|
)
|
50
50
|
from prefect.client.utilities import inject_client
|
51
|
+
from prefect.context import FlowRunContext, TagsContext
|
51
52
|
from prefect.events import Event, RelatedResource, emit_event
|
52
53
|
from prefect.events.related import object_as_related_resource, tags_as_related_resources
|
53
54
|
from prefect.exceptions import (
|
@@ -75,6 +76,7 @@ from prefect.states import (
|
|
75
76
|
Pending,
|
76
77
|
exception_to_failed_state,
|
77
78
|
)
|
79
|
+
from prefect.tasks import Task
|
78
80
|
from prefect.types import KeyValueLabels
|
79
81
|
from prefect.utilities.dispatch import get_registry_for_type, register_base_type
|
80
82
|
from prefect.utilities.engine import propose_state
|
@@ -720,6 +722,15 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
|
|
720
722
|
if self._runs_task_group is None:
|
721
723
|
raise RuntimeError("Worker not properly initialized")
|
722
724
|
|
725
|
+
from prefect.results import get_result_store
|
726
|
+
|
727
|
+
current_result_store = get_result_store()
|
728
|
+
if current_result_store.result_storage is None and flow.result_storage is None:
|
729
|
+
self._logger.warning(
|
730
|
+
f"Flow {flow.name!r} has no result storage configured. Please configure "
|
731
|
+
"result storage for the flow if you want to retrieve the result for the flow run."
|
732
|
+
)
|
733
|
+
|
723
734
|
flow_run = await self._runs_task_group.start(
|
724
735
|
partial(
|
725
736
|
self._submit_adhoc_run,
|
@@ -766,12 +777,28 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
|
|
766
777
|
)
|
767
778
|
|
768
779
|
job_variables = (job_variables or {}) | {"command": " ".join(execute_command)}
|
780
|
+
parameters = parameters or {}
|
781
|
+
parent_task_run = None
|
782
|
+
|
783
|
+
if flow_run_ctx := FlowRunContext.get():
|
784
|
+
parent_task = Task[Any, Any](
|
785
|
+
name=flow.name,
|
786
|
+
fn=flow.fn,
|
787
|
+
version=flow.version,
|
788
|
+
)
|
789
|
+
parent_task_run = await parent_task.create_run(
|
790
|
+
flow_run_context=flow_run_ctx,
|
791
|
+
parameters=parameters,
|
792
|
+
)
|
793
|
+
|
769
794
|
flow_run = await self.client.create_flow_run(
|
770
795
|
flow,
|
771
|
-
parameters=parameters,
|
796
|
+
parameters=flow.serialize_parameters(parameters),
|
772
797
|
state=Pending(),
|
773
798
|
job_variables=job_variables,
|
774
799
|
work_pool_name=self.work_pool.name,
|
800
|
+
tags=TagsContext.get().current_tags,
|
801
|
+
parent_task_run_id=getattr(parent_task_run, "id", None),
|
775
802
|
)
|
776
803
|
if task_status is not None:
|
777
804
|
# Emit the flow run object to .submit to allow it to return a future as soon as possible
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: prefect-client
|
3
|
-
Version: 3.3.
|
3
|
+
Version: 3.3.6
|
4
4
|
Summary: Workflow orchestration and management.
|
5
5
|
Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
|
6
6
|
Project-URL: Documentation, https://docs.prefect.io
|
@@ -47,7 +47,7 @@ Requires-Dist: prometheus-client>=0.20.0
|
|
47
47
|
Requires-Dist: pydantic!=2.10.0,<3.0.0,>=2.9
|
48
48
|
Requires-Dist: pydantic-core<3.0.0,>=2.12.0
|
49
49
|
Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
|
50
|
-
Requires-Dist: pydantic-settings
|
50
|
+
Requires-Dist: pydantic-settings!=2.9.0,<3.0.0,>2.2.1
|
51
51
|
Requires-Dist: python-dateutil<3.0.0,>=2.8.2
|
52
52
|
Requires-Dist: python-slugify<9.0,>=5.0
|
53
53
|
Requires-Dist: python-socks[asyncio]<3.0,>=2.5.3
|
@@ -1,21 +1,22 @@
|
|
1
1
|
prefect/.prefectignore,sha256=awSprvKT0vI8a64mEOLrMxhxqcO-b0ERQeYpA2rNKVQ,390
|
2
2
|
prefect/__init__.py,sha256=iCdcC5ZmeewikCdnPEP6YBAjPNV5dvfxpYCTpw30Hkw,3685
|
3
3
|
prefect/__main__.py,sha256=WFjw3kaYJY6pOTA7WDOgqjsz8zUEUZHCcj3P5wyVa-g,66
|
4
|
-
prefect/_build_info.py,sha256=
|
4
|
+
prefect/_build_info.py,sha256=OD-rnD9CpXpPYNn3bk6Wp8dfxfSjgkDcwGroH4wbFic,180
|
5
5
|
prefect/_result_records.py,sha256=S6QmsODkehGVSzbMm6ig022PYbI6gNKz671p_8kBYx4,7789
|
6
|
+
prefect/_versioning.py,sha256=Bm2EwEODvMe_kLkeVXy32BaTA_4ijBZl9eFbdtXEV4w,5498
|
6
7
|
prefect/_waiters.py,sha256=Ia2ITaXdHzevtyWIgJoOg95lrEXQqNEOquHvw3T33UQ,9026
|
7
8
|
prefect/agent.py,sha256=dPvG1jDGD5HSH7aM2utwtk6RaJ9qg13XjkA0lAIgQmY,287
|
8
9
|
prefect/artifacts.py,sha256=dMBUOAWnUamzjb5HSqwB5-GR2Qb-Gxee26XG5NDCUuw,22720
|
9
10
|
prefect/automations.py,sha256=ZzPxn2tINdlXTQo805V4rIlbXuNWxd7cdb3gTJxZIeY,12567
|
10
|
-
prefect/cache_policies.py,sha256=
|
11
|
-
prefect/context.py,sha256=
|
11
|
+
prefect/cache_policies.py,sha256=jH1aDW6vItTcsEytuTCrNYyjbq87IQPwdOgF0yxiUts,12749
|
12
|
+
prefect/context.py,sha256=IXS_ddSkQVFKFCQhjWk7Fwgfstfu6ITCmNeZ1beablY,23737
|
12
13
|
prefect/engine.py,sha256=uB5JN4l045i5JTlRQNT1x7MwlSiGQ5Bop2Q6jHHOgxY,3699
|
13
14
|
prefect/exceptions.py,sha256=wZLQQMRB_DyiYkeEdIC5OKwbba5A94Dlnics-lrWI7A,11581
|
14
15
|
prefect/filesystems.py,sha256=v5YqGB4uXf9Ew2VuB9VCSkawvYMMVvEtZf7w1VmAmr8,18036
|
15
16
|
prefect/flow_engine.py,sha256=hZpTYEtwTPMtwVoTCrfD93igN7rlKeG_0kyCvdU4aYE,58876
|
16
17
|
prefect/flow_runs.py,sha256=dbHcXsOq1UsNM7vyJV9gboCTylmdUwQ_-W4NQt4R4ds,17267
|
17
|
-
prefect/flows.py,sha256=
|
18
|
-
prefect/futures.py,sha256=
|
18
|
+
prefect/flows.py,sha256=UCBwsb99wtPTGPu2PneKCfAMlMBA2GhXJb5rzMBxw1s,118041
|
19
|
+
prefect/futures.py,sha256=F4eplqRcqw5-aMNKu6-lOFOWdDNr0RGrPso4C4G02bU,24248
|
19
20
|
prefect/main.py,sha256=8V-qLB4GjEVCkGRgGXeaIk-JIXY8Z9FozcNluj4Sm9E,2589
|
20
21
|
prefect/plugins.py,sha256=FPRLR2mWVBMuOnlzeiTD9krlHONZH2rtYLD753JQDNQ,2516
|
21
22
|
prefect/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -31,8 +32,9 @@ prefect/tasks.py,sha256=EpMw5O1B9pAFVraC0KzytMOKi8iy7ZYnKWRs7WtvogU,74742
|
|
31
32
|
prefect/transactions.py,sha256=uIoPNudzJzH6NrMJhrgr5lyh6JxOJQqT1GvrXt69yNw,26068
|
32
33
|
prefect/variables.py,sha256=dCK3vX7TbkqXZhnNT_v7rcGh3ISRqoR6pJVLpoll3Js,8342
|
33
34
|
prefect/_experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
34
|
-
prefect/_experimental/bundles.py,sha256=E5nRaLVTbYCrACXZcRJCd4ssOcQU-Z26ewCb_7tPeTM,6687
|
35
35
|
prefect/_experimental/lineage.py,sha256=8LssReoq7eLtQScUCu-7FCtrWoRZstXKRdpO0PxgbKg,9958
|
36
|
+
prefect/_experimental/bundles/__init__.py,sha256=9e7L7drTpHG82fnr6kuABkpk3SdqUNF-8HB2y6vD5U4,7108
|
37
|
+
prefect/_experimental/bundles/execute.py,sha256=xBwk1H1Fui1lt2G7Fgd6yDArqSmqZiYEA7Y7R4f7SDM,699
|
36
38
|
prefect/_experimental/sla/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
37
39
|
prefect/_experimental/sla/client.py,sha256=XTkYHFZiBy_O7RgUyGEdl9MxaHP-6fEAKBk3ksNQobU,3611
|
38
40
|
prefect/_experimental/sla/objects.py,sha256=Ja1z2XUgkklvtNTumKWWjojEM5I0L_RjdGv61sRbVP0,2834
|
@@ -98,7 +100,7 @@ prefect/client/orchestration/_blocks_types/client.py,sha256=alA4xD-yp3mycAbzMyRu
|
|
98
100
|
prefect/client/orchestration/_concurrency_limits/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
99
101
|
prefect/client/orchestration/_concurrency_limits/client.py,sha256=r_oyY7hQbgyG1rntwe7WWcsraQHBKhk6MOPFUAHWiVc,23678
|
100
102
|
prefect/client/orchestration/_deployments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
101
|
-
prefect/client/orchestration/_deployments/client.py,sha256=
|
103
|
+
prefect/client/orchestration/_deployments/client.py,sha256=a_sUmznQC3lBLxXfu4hIIbszbUc28sWXtCioQoNgZPk,43876
|
102
104
|
prefect/client/orchestration/_flow_runs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
103
105
|
prefect/client/orchestration/_flow_runs/client.py,sha256=fjh5J-LG8tsny7BGYEvynbuGuHDAudYHpx-PamL0GYQ,32220
|
104
106
|
prefect/client/orchestration/_flows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -122,7 +124,7 @@ prefect/concurrency/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
|
|
122
124
|
prefect/concurrency/_asyncio.py,sha256=uHjC3vQAiznRz_ueZE1RQ4x28zTcPJPoO2MMi0J41vU,2575
|
123
125
|
prefect/concurrency/_events.py,sha256=KWHDldCWE3b5AH9eZ7kfmajvp36lRFCjCXIEx77jtKk,1825
|
124
126
|
prefect/concurrency/asyncio.py,sha256=SUnRfqwBdBGwQll7SvywugVQnVbEzePqPFcUfIcTNMs,4505
|
125
|
-
prefect/concurrency/context.py,sha256=
|
127
|
+
prefect/concurrency/context.py,sha256=kJWE2zGuoel9qiGOqHW5qnSyzV1INlsicTmeEEexoFo,1029
|
126
128
|
prefect/concurrency/services.py,sha256=U_1Y8Mm-Fd4Nvn0gxiWc_UdacdqT-vKjzex-oJpUt50,2288
|
127
129
|
prefect/concurrency/sync.py,sha256=MMRJvxK-Yzyt0WEEu95C2RaMwfLdYgYH6vejCqfSUmw,4687
|
128
130
|
prefect/concurrency/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -136,7 +138,7 @@ prefect/deployments/__init__.py,sha256=_wb7NxDKhq11z9MjYsPckmT3o6MRhGLRgCV9TmvYt
|
|
136
138
|
prefect/deployments/base.py,sha256=YY7g8MN6qzjNEjEA8wQXPxCrd47WnACIUeSRtI4nrEk,11849
|
137
139
|
prefect/deployments/deployments.py,sha256=K3Rgnpjxo_T8I8LMwlq24OKqZiZBTE8-YnPg-YGUStM,171
|
138
140
|
prefect/deployments/flow_runs.py,sha256=NYe-Bphsy6ENLqSSfywQuX5cRZt-uVgzqGmOsf3Sqw4,7643
|
139
|
-
prefect/deployments/runner.py,sha256=
|
141
|
+
prefect/deployments/runner.py,sha256=_VqbkXvPVvdyFVkRsr5emi26cJmu5-2uhtVUoE0EkNA,54805
|
140
142
|
prefect/deployments/schedules.py,sha256=2eL1-w8qXtwKVkgfUK7cuamwpKK3X6tN1QYTDa_gWxU,2190
|
141
143
|
prefect/deployments/steps/__init__.py,sha256=Dlz9VqMRyG1Gal8dj8vfGpPr0LyQhZdvcciozkK8WoY,206
|
142
144
|
prefect/deployments/steps/core.py,sha256=ulSgBFSx1lhBt1fP-UxebrernkumBDlympR6IPffV1g,6900
|
@@ -146,7 +148,7 @@ prefect/docker/__init__.py,sha256=z6wdc6UFfiBG2jb9Jk64uCWVM04JKVWeVyDWwuuon8M,52
|
|
146
148
|
prefect/docker/docker_image.py,sha256=bR_pEq5-FDxlwTj8CP_7nwZ_MiGK6KxIi8v7DRjy1Kg,3138
|
147
149
|
prefect/events/__init__.py,sha256=GtKl2bE--pJduTxelH2xy7SadlLJmmis8WR1EYixhuA,2094
|
148
150
|
prefect/events/actions.py,sha256=A7jS8bo4zWGnrt3QfSoQs0uYC1xfKXio3IfU0XtTb5s,9129
|
149
|
-
prefect/events/clients.py,sha256=
|
151
|
+
prefect/events/clients.py,sha256=c5ZTt-ZVslvDr6-OrbsWb_7XUocWptGyAuVAVtAzokY,27589
|
150
152
|
prefect/events/filters.py,sha256=2hVfzc3Rdgy0mBHDutWxT__LJY0zpVM8greWX3y6kjM,8233
|
151
153
|
prefect/events/related.py,sha256=CTeexYUmmA93V4gsR33GIFmw-SS-X_ouOpRg-oeq-BU,6672
|
152
154
|
prefect/events/utilities.py,sha256=ww34bTMENCNwcp6RhhgzG0KgXOvKGe0MKmBdSJ8NpZY,3043
|
@@ -182,7 +184,7 @@ prefect/logging/highlighters.py,sha256=BCf_LNhFInIfGPqwuu8YVrGa4wVxNc4YXo2pYgftp
|
|
182
184
|
prefect/logging/loggers.py,sha256=rwFJv0i3dhdKr25XX-xUkQy4Vv4dy18bTy366jrC0OQ,12741
|
183
185
|
prefect/logging/logging.yml,sha256=tT7gTyC4NmngFSqFkCdHaw7R0GPNPDDsTCGZQByiJAQ,3169
|
184
186
|
prefect/runner/__init__.py,sha256=pQBd9wVrUVUDUFJlgiweKSnbahoBZwqnd2O2jkhrULY,158
|
185
|
-
prefect/runner/runner.py,sha256=
|
187
|
+
prefect/runner/runner.py,sha256=jv87XyaJ89uK0VzKpMzL3HfXgKZky8JlRs-gW04no5Y,65117
|
186
188
|
prefect/runner/server.py,sha256=YRYFNoYddA9XfiTIYtudxrnD1vCX-PaOLhvyGUOb9AQ,11966
|
187
189
|
prefect/runner/storage.py,sha256=L7aSjie5L6qbXYCDqYDX3ouQ_NsNMlmfjPeaWOC-ncs,28043
|
188
190
|
prefect/runner/submit.py,sha256=qOEj-NChQ6RYFV35hHEVMTklrNmKwaGs2mR78ku9H0o,9474
|
@@ -205,7 +207,7 @@ prefect/server/api/concurrency_limits.py,sha256=E5TB2cJPIZjnxnm1pGxUJnwMDz5CS58g
|
|
205
207
|
prefect/server/api/concurrency_limits_v2.py,sha256=PGjG7W2Z65OojNTP0ezFu2z69plXo1N8paqwHlHAPj0,10183
|
206
208
|
prefect/server/api/csrf_token.py,sha256=BwysSjQAhre7O0OY_LF3ZcIiO53FdMQroNT11Q6OcOM,1344
|
207
209
|
prefect/server/api/dependencies.py,sha256=VujfcIGn41TGJxUunFHVabY5hE-6nY6uSHyhNFj8PdI,6634
|
208
|
-
prefect/server/api/deployments.py,sha256=
|
210
|
+
prefect/server/api/deployments.py,sha256=ppYA3b2csnw32-SbOXz5Dm_IsnmPKczNiSbqCzusFKI,39332
|
209
211
|
prefect/server/api/events.py,sha256=3-Qdt6ORxFv3nLoogQqvd72zEulJSoAmcqZto2OULuk,9907
|
210
212
|
prefect/server/api/flow_run_notification_policies.py,sha256=F8xNm6bgZTC3nFe9xCUJS4NlU9tLXZ8fShtJqmhT2m4,4828
|
211
213
|
prefect/server/api/flow_run_states.py,sha256=lIdxVE9CqLgtDCuH9bTaKkzHNL81FPrr11liPzvONrw,1661
|
@@ -278,6 +280,7 @@ prefect/types/__init__.py,sha256=yBjKxiQmSC7jXoo0UNmM3KZil1NBFS-BWGPfwSEaoJo,462
|
|
278
280
|
prefect/types/_datetime.py,sha256=Cy6z7MxPDV_-jH2vxqC3PNA2G74IdUDIB07Jaakdj5w,7294
|
279
281
|
prefect/types/entrypoint.py,sha256=2FF03-wLPgtnqR_bKJDB2BsXXINPdu8ptY9ZYEZnXg8,328
|
280
282
|
prefect/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
283
|
+
prefect/utilities/_ast.py,sha256=sgEPUWElih-3cp4PoAy1IOyPtu8E27lL0Dldf3ijnYY,4905
|
281
284
|
prefect/utilities/_deprecated.py,sha256=b3pqRSoFANdVJAc8TJkygBcP-VjZtLJUxVIWC7kwspI,1303
|
282
285
|
prefect/utilities/_engine.py,sha256=9GW4X1lyAbmPwCuXXIubVJ7Z0DMT3dykkEUtp9tm5hI,3356
|
283
286
|
prefect/utilities/_git.py,sha256=bPYWQdr9xvH0BqxR1ll1RkaSb3x0vhwylhYD5EilkKU,863
|
@@ -310,13 +313,13 @@ prefect/utilities/schema_tools/__init__.py,sha256=At3rMHd2g_Em2P3_dFQlFgqR_EpBwr
|
|
310
313
|
prefect/utilities/schema_tools/hydration.py,sha256=NkRhWkNfxxFmVGhNDfmxdK_xeKaEhs3a42q83Sg9cT4,9436
|
311
314
|
prefect/utilities/schema_tools/validation.py,sha256=Wix26IVR-ZJ32-6MX2pHhrwm3reB-Q4iB6_phn85OKE,10743
|
312
315
|
prefect/workers/__init__.py,sha256=EaM1F0RZ-XIJaGeTKLsXDnfOPHzVWk5bk0_c4BVS44M,64
|
313
|
-
prefect/workers/base.py,sha256=
|
316
|
+
prefect/workers/base.py,sha256=B3K80V-bZ1oI-5iwM2jw93is9srTSCLNN2lvVtlmB7g,60267
|
314
317
|
prefect/workers/block.py,sha256=dPvG1jDGD5HSH7aM2utwtk6RaJ9qg13XjkA0lAIgQmY,287
|
315
318
|
prefect/workers/cloud.py,sha256=dPvG1jDGD5HSH7aM2utwtk6RaJ9qg13XjkA0lAIgQmY,287
|
316
319
|
prefect/workers/process.py,sha256=Yi5D0U5AQ51wHT86GdwtImXSefe0gJf3LGq4r4z9zwM,11090
|
317
320
|
prefect/workers/server.py,sha256=2pmVeJZiVbEK02SO6BEZaBIvHMsn6G8LzjW8BXyiTtk,1952
|
318
321
|
prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
|
319
|
-
prefect_client-3.3.
|
320
|
-
prefect_client-3.3.
|
321
|
-
prefect_client-3.3.
|
322
|
-
prefect_client-3.3.
|
322
|
+
prefect_client-3.3.6.dist-info/METADATA,sha256=P3Idrvm26XdduiE-Fo-WN0zyLG4_2wgdDV4tD9DXqio,7466
|
323
|
+
prefect_client-3.3.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
324
|
+
prefect_client-3.3.6.dist-info/licenses/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
|
325
|
+
prefect_client-3.3.6.dist-info/RECORD,,
|
File without changes
|
File without changes
|