flyte 2.0.0b21__py3-none-any.whl → 2.0.0b22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flyte/_bin/runtime.py +1 -1
- flyte/_deploy.py +16 -8
- flyte/_image.py +5 -1
- flyte/_initialize.py +7 -1
- flyte/_interface.py +3 -1
- flyte/_internal/imagebuild/docker_builder.py +3 -0
- flyte/_internal/imagebuild/image_builder.py +8 -7
- flyte/_internal/runtime/task_serde.py +16 -6
- flyte/_keyring/file.py +2 -2
- flyte/_task_environment.py +1 -1
- flyte/_version.py +3 -3
- flyte/cli/_common.py +2 -2
- flyte/cli/_deploy.py +11 -1
- flyte/cli/_run.py +13 -3
- flyte/remote/_action.py +9 -8
- flyte/types/_pickle.py +18 -4
- flyte/types/_type_engine.py +6 -0
- {flyte-2.0.0b21.data → flyte-2.0.0b22.data}/scripts/runtime.py +1 -1
- {flyte-2.0.0b21.dist-info → flyte-2.0.0b22.dist-info}/METADATA +1 -1
- {flyte-2.0.0b21.dist-info → flyte-2.0.0b22.dist-info}/RECORD +25 -25
- {flyte-2.0.0b21.data → flyte-2.0.0b22.data}/scripts/debug.py +0 -0
- {flyte-2.0.0b21.dist-info → flyte-2.0.0b22.dist-info}/WHEEL +0 -0
- {flyte-2.0.0b21.dist-info → flyte-2.0.0b22.dist-info}/entry_points.txt +0 -0
- {flyte-2.0.0b21.dist-info → flyte-2.0.0b22.dist-info}/licenses/LICENSE +0 -0
- {flyte-2.0.0b21.dist-info → flyte-2.0.0b22.dist-info}/top_level.txt +0 -0
flyte/_bin/runtime.py
CHANGED
|
@@ -137,7 +137,7 @@ def main(
|
|
|
137
137
|
logger.debug(f"Using controller endpoint: {ep} with kwargs: {controller_kwargs}")
|
|
138
138
|
|
|
139
139
|
bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
|
|
140
|
-
init(org=org, project=project, domain=domain, **controller_kwargs)
|
|
140
|
+
init(org=org, project=project, domain=domain, image_builder="remote", **controller_kwargs)
|
|
141
141
|
# Controller is created with the same kwargs as init, so that it can be used to run tasks
|
|
142
142
|
controller = create_controller(ct="remote", **controller_kwargs)
|
|
143
143
|
|
flyte/_deploy.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import hashlib
|
|
5
|
+
import sys
|
|
4
6
|
import typing
|
|
5
7
|
from dataclasses import dataclass
|
|
6
8
|
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
|
|
7
9
|
|
|
10
|
+
import cloudpickle
|
|
8
11
|
import rich.repr
|
|
9
12
|
|
|
10
13
|
import flyte.errors
|
|
@@ -160,9 +163,11 @@ async def _build_images(deployment: DeploymentPlan) -> ImageCache:
|
|
|
160
163
|
logger.warning(f"Built Image for environment {env_name}, image: {image_uri}")
|
|
161
164
|
env = deployment.envs[env_name]
|
|
162
165
|
if isinstance(env.image, Image):
|
|
163
|
-
|
|
166
|
+
py_version = "{}.{}".format(*env.image.python_version)
|
|
167
|
+
image_identifier_map[env.image.identifier] = {py_version: image_uri}
|
|
164
168
|
elif env.image == "auto":
|
|
165
|
-
|
|
169
|
+
py_version = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
|
|
170
|
+
image_identifier_map["auto"] = {py_version: image_uri}
|
|
166
171
|
|
|
167
172
|
return ImageCache(image_lookup=image_identifier_map)
|
|
168
173
|
|
|
@@ -175,15 +180,18 @@ async def apply(deployment_plan: DeploymentPlan, copy_style: CopyFiles, dryrun:
|
|
|
175
180
|
|
|
176
181
|
image_cache = await _build_images(deployment_plan)
|
|
177
182
|
|
|
178
|
-
|
|
179
|
-
if copy_style == "none" and not version:
|
|
183
|
+
if copy_style == "none" and not deployment_plan.version:
|
|
180
184
|
raise flyte.errors.DeploymentError("Version must be set when copy_style is none")
|
|
181
185
|
else:
|
|
182
186
|
code_bundle = await build_code_bundle(from_dir=cfg.root_dir, dryrun=dryrun, copy_style=copy_style)
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
+
if deployment_plan.version:
|
|
188
|
+
version = deployment_plan.version
|
|
189
|
+
else:
|
|
190
|
+
h = hashlib.md5()
|
|
191
|
+
h.update(cloudpickle.dumps(deployment_plan.envs))
|
|
192
|
+
h.update(code_bundle.computed_version.encode("utf-8"))
|
|
193
|
+
h.update(cloudpickle.dumps(image_cache))
|
|
194
|
+
version = h.hexdigest()
|
|
187
195
|
|
|
188
196
|
sc = SerializationContext(
|
|
189
197
|
project=cfg.project,
|
flyte/_image.py
CHANGED
|
@@ -203,6 +203,7 @@ class UVProject(PipOption, Layer):
|
|
|
203
203
|
|
|
204
204
|
super().update_hash(hasher)
|
|
205
205
|
filehash_update(self.uvlock, hasher)
|
|
206
|
+
filehash_update(self.pyproject, hasher)
|
|
206
207
|
|
|
207
208
|
|
|
208
209
|
@rich.repr.auto
|
|
@@ -435,7 +436,10 @@ class Image:
|
|
|
435
436
|
# Only get the non-None values in the Image to ensure the hash is consistent
|
|
436
437
|
# across different SDK versions.
|
|
437
438
|
# Layers can specify a _compute_identifier optionally, but the default will just stringify
|
|
438
|
-
image_dict = asdict(
|
|
439
|
+
image_dict = asdict(
|
|
440
|
+
self,
|
|
441
|
+
dict_factory=lambda x: {k: v for (k, v) in x if v is not None and k not in ("_layers", "python_version")},
|
|
442
|
+
)
|
|
439
443
|
layers_str_repr = "".join([layer.identifier() for layer in self._layers])
|
|
440
444
|
image_dict["layers"] = layers_str_repr
|
|
441
445
|
spec_bytes = image_dict.__str__().encode("utf-8")
|
flyte/_initialize.py
CHANGED
|
@@ -33,6 +33,7 @@ class CommonInit:
|
|
|
33
33
|
project: str | None = None
|
|
34
34
|
domain: str | None = None
|
|
35
35
|
batch_size: int = 1000
|
|
36
|
+
source_config_path: Optional[Path] = None # Only used for documentation
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
@dataclass(init=True, kw_only=True, repr=True, eq=True, frozen=True)
|
|
@@ -140,6 +141,7 @@ async def init(
|
|
|
140
141
|
storage: Storage | None = None,
|
|
141
142
|
batch_size: int = 1000,
|
|
142
143
|
image_builder: ImageBuildEngine.ImageBuilderType = "local",
|
|
144
|
+
source_config_path: Optional[Path] = None,
|
|
143
145
|
) -> None:
|
|
144
146
|
"""
|
|
145
147
|
Initialize the Flyte system with the given configuration. This method should be called before any other Flyte
|
|
@@ -175,7 +177,7 @@ async def init(
|
|
|
175
177
|
:param batch_size: Optional batch size for operations that use listings, defaults to 1000, so limit larger than
|
|
176
178
|
batch_size will be split into multiple requests.
|
|
177
179
|
:param image_builder: Optional image builder configuration, if not provided, the default image builder will be used.
|
|
178
|
-
|
|
180
|
+
:param source_config_path: Optional path to the source configuration file (This is only used for documentation)
|
|
179
181
|
:return: None
|
|
180
182
|
"""
|
|
181
183
|
from flyte._utils import get_cwd_editable_install, org_from_endpoint, sanitize_endpoint
|
|
@@ -224,6 +226,7 @@ async def init(
|
|
|
224
226
|
org=org or org_from_endpoint(endpoint),
|
|
225
227
|
batch_size=batch_size,
|
|
226
228
|
image_builder=image_builder,
|
|
229
|
+
source_config_path=source_config_path,
|
|
227
230
|
)
|
|
228
231
|
|
|
229
232
|
|
|
@@ -245,6 +248,7 @@ async def init_from_config(
|
|
|
245
248
|
if not available, the current working directory.
|
|
246
249
|
:param log_level: Optional logging level for the framework logger,
|
|
247
250
|
default is set using the default initialization policies
|
|
251
|
+
:param storage: Optional blob store (S3, GCS, Azure) configuration if needed to access (i.e. using Minio)
|
|
248
252
|
:return: None
|
|
249
253
|
"""
|
|
250
254
|
from rich.highlighter import ReprHighlighter
|
|
@@ -252,6 +256,7 @@ async def init_from_config(
|
|
|
252
256
|
import flyte.config as config
|
|
253
257
|
|
|
254
258
|
cfg: config.Config
|
|
259
|
+
cfg_path: Optional[Path] = None
|
|
255
260
|
if path_or_config is None:
|
|
256
261
|
# If no path is provided, use the default config file
|
|
257
262
|
cfg = config.auto()
|
|
@@ -290,6 +295,7 @@ async def init_from_config(
|
|
|
290
295
|
log_level=log_level,
|
|
291
296
|
image_builder=cfg.image.builder,
|
|
292
297
|
storage=storage,
|
|
298
|
+
source_config_path=cfg_path,
|
|
293
299
|
)
|
|
294
300
|
|
|
295
301
|
|
flyte/_interface.py
CHANGED
|
@@ -7,6 +7,8 @@ from typing import Dict, Generator, Literal, Tuple, Type, TypeVar, Union, cast,
|
|
|
7
7
|
|
|
8
8
|
from flyte._logging import logger
|
|
9
9
|
|
|
10
|
+
LITERAL_ENUM = "LiteralEnum"
|
|
11
|
+
|
|
10
12
|
|
|
11
13
|
def default_output_name(index: int = 0) -> str:
|
|
12
14
|
return f"o{index}"
|
|
@@ -110,6 +112,6 @@ def literal_to_enum(literal_type: Type) -> Type[Enum | typing.Any]:
|
|
|
110
112
|
enum_dict = {str(v).upper(): v for v in values}
|
|
111
113
|
|
|
112
114
|
# Dynamically create an Enum
|
|
113
|
-
literal_enum = Enum(
|
|
115
|
+
literal_enum = Enum(LITERAL_ENUM, enum_dict) # type: ignore
|
|
114
116
|
|
|
115
117
|
return literal_enum # type: ignore
|
|
@@ -245,6 +245,9 @@ class UVProjectHandler:
|
|
|
245
245
|
else:
|
|
246
246
|
# Copy the entire project.
|
|
247
247
|
pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path)
|
|
248
|
+
if layer.uvlock:
|
|
249
|
+
# Sometimes the uv.lock file is in a different folder, if it's specified, let's copy it there explicitly
|
|
250
|
+
shutil.copy(layer.uvlock, pyproject_dst)
|
|
248
251
|
delta = UV_LOCK_INSTALL_TEMPLATE.substitute(
|
|
249
252
|
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
250
253
|
PIP_INSTALL_ARGS=" ".join(layer.get_pip_install_args()),
|
|
@@ -235,7 +235,7 @@ class ImageBuildEngine:
|
|
|
235
235
|
|
|
236
236
|
|
|
237
237
|
class ImageCache(BaseModel):
|
|
238
|
-
image_lookup: Dict[str, str]
|
|
238
|
+
image_lookup: Dict[str, Dict[str, str]]
|
|
239
239
|
serialized_form: str | None = None
|
|
240
240
|
|
|
241
241
|
@property
|
|
@@ -273,10 +273,11 @@ class ImageCache(BaseModel):
|
|
|
273
273
|
"""
|
|
274
274
|
tuples = []
|
|
275
275
|
for k, v in self.image_lookup.items():
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
276
|
+
for py_version, image_uri in v.items():
|
|
277
|
+
tuples.append(
|
|
278
|
+
[
|
|
279
|
+
("Name", f"{k} (py{py_version})"),
|
|
280
|
+
("image", image_uri),
|
|
281
|
+
]
|
|
282
|
+
)
|
|
282
283
|
return tuples
|
|
@@ -4,6 +4,7 @@ It includes a Resolver interface for loading tasks, and functions to load classe
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
import copy
|
|
7
|
+
import sys
|
|
7
8
|
import typing
|
|
8
9
|
from datetime import timedelta
|
|
9
10
|
from typing import Optional, cast
|
|
@@ -217,16 +218,25 @@ def _get_urun_container(
|
|
|
217
218
|
img_uri = task_template.image.uri
|
|
218
219
|
elif serialize_context.image_cache and image_id not in serialize_context.image_cache.image_lookup:
|
|
219
220
|
img_uri = task_template.image.uri
|
|
220
|
-
from flyte._version import __version__
|
|
221
221
|
|
|
222
222
|
logger.warning(
|
|
223
|
-
f"Image {task_template.image} not found in the image cache: {serialize_context.image_cache.image_lookup}
|
|
224
|
-
f"This typically occurs when the Flyte SDK version (`{__version__}`) used in the task environment "
|
|
225
|
-
f"differs from the version used to compile or deploy it.\n"
|
|
226
|
-
f"Ensure both environments use the same Flyte SDK version to avoid inconsistencies in image resolution."
|
|
223
|
+
f"Image {task_template.image} not found in the image cache: {serialize_context.image_cache.image_lookup}."
|
|
227
224
|
)
|
|
228
225
|
else:
|
|
229
|
-
|
|
226
|
+
python_version_str = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
|
|
227
|
+
version_lookup = serialize_context.image_cache.image_lookup[image_id]
|
|
228
|
+
if python_version_str in version_lookup:
|
|
229
|
+
img_uri = version_lookup[python_version_str]
|
|
230
|
+
elif version_lookup:
|
|
231
|
+
# Fallback: try to get any available version
|
|
232
|
+
fallback_py_version, img_uri = next(iter(version_lookup.items()))
|
|
233
|
+
logger.warning(
|
|
234
|
+
f"Image {task_template.image} for python version {python_version_str} "
|
|
235
|
+
f"not found in the image cache: {serialize_context.image_cache.image_lookup}.\n"
|
|
236
|
+
f"Fall back using image {img_uri} for python version {fallback_py_version} ."
|
|
237
|
+
)
|
|
238
|
+
else:
|
|
239
|
+
img_uri = task_template.image.uri
|
|
230
240
|
|
|
231
241
|
return tasks_pb2.Container(
|
|
232
242
|
image=img_uri,
|
flyte/_keyring/file.py
CHANGED
|
@@ -72,9 +72,9 @@ class SimplePlainTextKeyring(KeyringBackend):
|
|
|
72
72
|
|
|
73
73
|
@property
|
|
74
74
|
def file_path(self) -> Path:
|
|
75
|
-
from flyte.
|
|
75
|
+
from flyte._initialize import get_common_config
|
|
76
76
|
|
|
77
|
-
config_path =
|
|
77
|
+
config_path = get_common_config().source_config_path
|
|
78
78
|
if config_path and str(config_path.parent) == ".flyte":
|
|
79
79
|
# if the config is in a .flyte directory, use that as the path
|
|
80
80
|
return config_path.parent / "keyring.cfg"
|
flyte/_task_environment.py
CHANGED
flyte/_version.py
CHANGED
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '2.0.
|
|
32
|
-
__version_tuple__ = version_tuple = (2, 0, 0, '
|
|
31
|
+
__version__ = version = '2.0.0b22'
|
|
32
|
+
__version_tuple__ = version_tuple = (2, 0, 0, 'b22')
|
|
33
33
|
|
|
34
|
-
__commit_id__ = commit_id = '
|
|
34
|
+
__commit_id__ = commit_id = 'gce9a6bede'
|
flyte/cli/_common.py
CHANGED
|
@@ -111,7 +111,7 @@ class CLIConfig:
|
|
|
111
111
|
"""
|
|
112
112
|
return replace(self, **kwargs)
|
|
113
113
|
|
|
114
|
-
def init(self, project: str | None = None, domain: str | None = None):
|
|
114
|
+
def init(self, project: str | None = None, domain: str | None = None, root_dir: str | None = None):
|
|
115
115
|
from flyte.config._config import TaskConfig
|
|
116
116
|
|
|
117
117
|
task_cfg = TaskConfig(
|
|
@@ -131,7 +131,7 @@ class CLIConfig:
|
|
|
131
131
|
|
|
132
132
|
updated_config = self.config.with_params(platform_cfg, task_cfg)
|
|
133
133
|
|
|
134
|
-
flyte.init_from_config(updated_config, log_level=self.log_level)
|
|
134
|
+
flyte.init_from_config(updated_config, log_level=self.log_level, root_dir=root_dir)
|
|
135
135
|
|
|
136
136
|
|
|
137
137
|
class InvokeBaseMixin:
|
flyte/cli/_deploy.py
CHANGED
|
@@ -43,6 +43,16 @@ class DeployArguments:
|
|
|
43
43
|
)
|
|
44
44
|
},
|
|
45
45
|
)
|
|
46
|
+
root_dir: str | None = field(
|
|
47
|
+
default=None,
|
|
48
|
+
metadata={
|
|
49
|
+
"click.option": click.Option(
|
|
50
|
+
["--root-dir"],
|
|
51
|
+
type=str,
|
|
52
|
+
help="Override the root source directory, helpful when working with monorepos.",
|
|
53
|
+
)
|
|
54
|
+
},
|
|
55
|
+
)
|
|
46
56
|
recursive: bool = field(
|
|
47
57
|
default=False,
|
|
48
58
|
metadata={
|
|
@@ -99,7 +109,7 @@ class DeployEnvCommand(click.RichCommand):
|
|
|
99
109
|
console = Console()
|
|
100
110
|
console.print(f"Deploying root - environment: {self.env_name}")
|
|
101
111
|
obj: CLIConfig = ctx.obj
|
|
102
|
-
obj.init(self.deploy_args.project, self.deploy_args.domain)
|
|
112
|
+
obj.init(self.deploy_args.project, self.deploy_args.domain, root_dir=self.deploy_args.root_dir)
|
|
103
113
|
with console.status("Deploying...", spinner="dots"):
|
|
104
114
|
deployment = flyte.deploy(
|
|
105
115
|
self.env,
|
flyte/cli/_run.py
CHANGED
|
@@ -23,14 +23,14 @@ RUN_REMOTE_CMD = "deployed-task"
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
@lru_cache()
|
|
26
|
-
def _initialize_config(ctx: click.Context, project: str, domain: str):
|
|
26
|
+
def _initialize_config(ctx: click.Context, project: str, domain: str, root_dir: str | None = None):
|
|
27
27
|
obj: CLIConfig | None = ctx.obj
|
|
28
28
|
if obj is None:
|
|
29
29
|
import flyte.config
|
|
30
30
|
|
|
31
31
|
obj = CLIConfig(flyte.config.auto(), ctx)
|
|
32
32
|
|
|
33
|
-
obj.init(project, domain)
|
|
33
|
+
obj.init(project, domain, root_dir)
|
|
34
34
|
return obj
|
|
35
35
|
|
|
36
36
|
|
|
@@ -77,6 +77,16 @@ class RunArguments:
|
|
|
77
77
|
)
|
|
78
78
|
},
|
|
79
79
|
)
|
|
80
|
+
root_dir: str | None = field(
|
|
81
|
+
default=None,
|
|
82
|
+
metadata={
|
|
83
|
+
"click.option": click.Option(
|
|
84
|
+
["--root-dir"],
|
|
85
|
+
type=str,
|
|
86
|
+
help="Override the root source directory, helpful when working with monorepos.",
|
|
87
|
+
)
|
|
88
|
+
},
|
|
89
|
+
)
|
|
80
90
|
name: str | None = field(
|
|
81
91
|
default=None,
|
|
82
92
|
metadata={
|
|
@@ -121,7 +131,7 @@ class RunTaskCommand(click.RichCommand):
|
|
|
121
131
|
super().__init__(obj_name, *args, **kwargs)
|
|
122
132
|
|
|
123
133
|
def invoke(self, ctx: click.Context):
|
|
124
|
-
obj: CLIConfig = _initialize_config(ctx, self.run_args.project, self.run_args.domain)
|
|
134
|
+
obj: CLIConfig = _initialize_config(ctx, self.run_args.project, self.run_args.domain, self.run_args.root_dir)
|
|
125
135
|
|
|
126
136
|
async def _run():
|
|
127
137
|
import flyte
|
flyte/remote/_action.py
CHANGED
|
@@ -369,14 +369,15 @@ class Action(ToJSONMixin):
|
|
|
369
369
|
# If the action is done, handle the final state
|
|
370
370
|
if ad.done():
|
|
371
371
|
progress.stop_task(task_id)
|
|
372
|
-
if
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
372
|
+
if not quiet:
|
|
373
|
+
if ad.pb2.status.phase == run_definition_pb2.PHASE_SUCCEEDED:
|
|
374
|
+
console.print(f"[bold green]Run '{self.run_name}' completed successfully.[/bold green]")
|
|
375
|
+
else:
|
|
376
|
+
error_message = ad.error_info.message if ad.error_info else ""
|
|
377
|
+
console.print(
|
|
378
|
+
f"[bold red]Run '{self.run_name}' exited unsuccessfully in state {ad.phase}"
|
|
379
|
+
f" with error: {error_message}[/bold red]"
|
|
380
|
+
)
|
|
380
381
|
break
|
|
381
382
|
except asyncio.CancelledError:
|
|
382
383
|
# Handle cancellation gracefully
|
flyte/types/_pickle.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
import os
|
|
3
|
+
import sys
|
|
3
4
|
import typing
|
|
4
5
|
from typing import Type
|
|
5
6
|
|
|
@@ -12,6 +13,7 @@ import flyte.storage as storage
|
|
|
12
13
|
from ._type_engine import TypeEngine, TypeTransformer
|
|
13
14
|
|
|
14
15
|
T = typing.TypeVar("T")
|
|
16
|
+
DEFAULT_PICKLE_BYTES_LIMIT = 2**10 * 10 # 10KB
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
class FlytePickle(typing.Generic[T]):
|
|
@@ -76,8 +78,13 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
|
|
|
76
78
|
...
|
|
77
79
|
|
|
78
80
|
async def to_python_value(self, lv: literals_pb2.Literal, expected_python_type: Type[T]) -> T:
|
|
79
|
-
|
|
80
|
-
|
|
81
|
+
if lv.scalar.blob and lv.scalar.blob.uri:
|
|
82
|
+
uri = lv.scalar.blob.uri
|
|
83
|
+
return await FlytePickle.from_pickle(uri)
|
|
84
|
+
elif lv.scalar.binary and lv.scalar.binary.value:
|
|
85
|
+
return cloudpickle.loads(lv.scalar.binary.value)
|
|
86
|
+
else:
|
|
87
|
+
raise ValueError(f"Cannot convert {lv} to {expected_python_type}")
|
|
81
88
|
|
|
82
89
|
async def to_literal(
|
|
83
90
|
self,
|
|
@@ -92,8 +99,15 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
|
|
|
92
99
|
format=self.PYTHON_PICKLE_FORMAT, dimensionality=types_pb2.BlobType.BlobDimensionality.SINGLE
|
|
93
100
|
)
|
|
94
101
|
)
|
|
95
|
-
|
|
96
|
-
|
|
102
|
+
if sys.getsizeof(python_val) > DEFAULT_PICKLE_BYTES_LIMIT:
|
|
103
|
+
remote_path = await FlytePickle.to_pickle(python_val)
|
|
104
|
+
return literals_pb2.Literal(
|
|
105
|
+
scalar=literals_pb2.Scalar(blob=literals_pb2.Blob(metadata=meta, uri=remote_path))
|
|
106
|
+
)
|
|
107
|
+
else:
|
|
108
|
+
return literals_pb2.Literal(
|
|
109
|
+
scalar=literals_pb2.Scalar(binary=literals_pb2.Binary(value=cloudpickle.dumps(python_val)))
|
|
110
|
+
)
|
|
97
111
|
|
|
98
112
|
def guess_python_type(self, literal_type: types_pb2.LiteralType) -> typing.Type[FlytePickle[typing.Any]]:
|
|
99
113
|
if (
|
flyte/types/_type_engine.py
CHANGED
|
@@ -803,6 +803,12 @@ class EnumTransformer(TypeTransformer[enum.Enum]):
|
|
|
803
803
|
async def to_python_value(self, lv: Literal, expected_python_type: Type[T]) -> T:
|
|
804
804
|
if lv.HasField("scalar") and lv.scalar.HasField("binary"):
|
|
805
805
|
return self.from_binary_idl(lv.scalar.binary, expected_python_type) # type: ignore
|
|
806
|
+
from flyte._interface import LITERAL_ENUM
|
|
807
|
+
|
|
808
|
+
if expected_python_type.__name__ is LITERAL_ENUM:
|
|
809
|
+
# This is the case when python Literal types are used as enums. The class name is always LiteralEnum an
|
|
810
|
+
# hardcoded in flyte.models
|
|
811
|
+
return lv.scalar.primitive.string_value
|
|
806
812
|
return expected_python_type(lv.scalar.primitive.string_value) # type: ignore
|
|
807
813
|
|
|
808
814
|
def guess_python_type(self, literal_type: LiteralType) -> Type[enum.Enum]:
|
|
@@ -137,7 +137,7 @@ def main(
|
|
|
137
137
|
logger.debug(f"Using controller endpoint: {ep} with kwargs: {controller_kwargs}")
|
|
138
138
|
|
|
139
139
|
bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
|
|
140
|
-
init(org=org, project=project, domain=domain, **controller_kwargs)
|
|
140
|
+
init(org=org, project=project, domain=domain, image_builder="remote", **controller_kwargs)
|
|
141
141
|
# Controller is created with the same kwargs as init, so that it can be used to run tasks
|
|
142
142
|
controller = create_controller(ct="remote", **controller_kwargs)
|
|
143
143
|
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
flyte/__init__.py,sha256=RIvRNnyKc9TfF5FlvREETDuosgs66w6d_A-MPMbyy9E,2158
|
|
2
2
|
flyte/_build.py,sha256=MkgfLAPeL56YeVrGRNZUCZgbwzlEzVP3wLbl5Qru4yk,578
|
|
3
3
|
flyte/_context.py,sha256=NGl-tDoTOoIljsUzD1IPTESOdaG8vKeoAzs5271XuPM,5244
|
|
4
|
-
flyte/_deploy.py,sha256=
|
|
4
|
+
flyte/_deploy.py,sha256=Y-YjPXtlnFGlP3Yuy6N_DhVBwwcUGPEYWImJfqOWcyE,10816
|
|
5
5
|
flyte/_doc.py,sha256=_OPCf3t_git6UT7kSJISFaWO9cfNzJhhoe6JjVdyCJo,706
|
|
6
6
|
flyte/_docstring.py,sha256=SsG0Ab_YMAwy2ABJlEo3eBKlyC3kwPdnDJ1FIms-ZBQ,1127
|
|
7
7
|
flyte/_environment.py,sha256=XYGscZ_I4xw-52o1ZULCGCVBY-l2m98rp9wpiQO4tQw,4590
|
|
8
8
|
flyte/_excepthook.py,sha256=nXts84rzEg6-7RtFarbKzOsRZTQR4plnbWVIFMAEprs,1310
|
|
9
9
|
flyte/_group.py,sha256=7o1j16sZyUmYB50mOiq1ui4TBAKhRpDqLakV8Ya1kw4,803
|
|
10
10
|
flyte/_hash.py,sha256=KMKjoI7SaxXildb-xv6n5Vb32B0csvBiYc06iUe-BrI,137
|
|
11
|
-
flyte/_image.py,sha256=
|
|
12
|
-
flyte/_initialize.py,sha256=
|
|
13
|
-
flyte/_interface.py,sha256=
|
|
11
|
+
flyte/_image.py,sha256=fMVV7G13BrAm8S1TzubOlmneOeW6novLSGYchcBg6SQ,38709
|
|
12
|
+
flyte/_initialize.py,sha256=nIeZVVO0sQjQh6F9P5rOSq5I2SXumnLwbPYzavdzRhY,18396
|
|
13
|
+
flyte/_interface.py,sha256=krUJOv43dOsSUNGlt30ajfFUl8_HPhubPg6dRlGSa4k,4795
|
|
14
14
|
flyte/_logging.py,sha256=7e_rQC4gtBcKvJStNX8Af-ZJG0U0r-2Om_8JMvkvb5k,6283
|
|
15
15
|
flyte/_map.py,sha256=8u4jZsM26V-M-dhVbSnyqSrkzJnGgisN3kwwSNQZaa4,10697
|
|
16
16
|
flyte/_pod.py,sha256=MB5eP2WvTc5lD5ovdVlxoOpx-wJeutv0S64s75astLc,1105
|
|
@@ -20,19 +20,19 @@ flyte/_reusable_environment.py,sha256=qzmLJlHFiek8_k3EEqxew3837Pe2xjmz3mjGk_xqPE
|
|
|
20
20
|
flyte/_run.py,sha256=N5N2LPFIOgVBflGaL7WAErQyspGa3zlKuEQ1uoqVTuA,25901
|
|
21
21
|
flyte/_secret.py,sha256=wug5NbIYEkXO6FJkqnPRnPoc2nKDerQiemWtRGRi288,3576
|
|
22
22
|
flyte/_task.py,sha256=kYoguvcRXwkwCbFBVCl6MXXYKHgADl2H53qWcs4CEQ4,20201
|
|
23
|
-
flyte/_task_environment.py,sha256=
|
|
23
|
+
flyte/_task_environment.py,sha256=jwvi6ceYoqwnvEFEJR5KmTZHngsuhAkFfWmhtRcl1rk,10086
|
|
24
24
|
flyte/_task_plugins.py,sha256=9MH3nFPOH_e8_92BT4sFk4oyAnj6GJFvaPYWaraX7yE,1037
|
|
25
25
|
flyte/_timeout.py,sha256=zx5sFcbYmjJAJbZWSGzzX-BpC9HC7Jfs35T7vVhKwkk,1571
|
|
26
26
|
flyte/_tools.py,sha256=lB3OiJSAhxzSMCYjLUF6nZjlFsmNpaRXtr3_Fefcxbg,747
|
|
27
27
|
flyte/_trace.py,sha256=-BIprs2MbupWl3vsC_Pn33SV3fSVku1rUIsnwfmrIy0,5204
|
|
28
|
-
flyte/_version.py,sha256=
|
|
28
|
+
flyte/_version.py,sha256=kBIvRTcJoPn-XOZzko2zml9uPZwZAzL5typ9a4VENRE,722
|
|
29
29
|
flyte/errors.py,sha256=k1-pgz7xm4syViJmPwp7ZDusmqcrlM2uZKrl0com_9I,6707
|
|
30
30
|
flyte/extend.py,sha256=GB4ZedGzKa30vYWRVPOdxEeK62xnUVFY4z2tD6H9eEw,376
|
|
31
31
|
flyte/models.py,sha256=l7h-o6pLHGqFcbymAOagHdR4BGKBLJUs6Pmd4J1xbQU,16409
|
|
32
32
|
flyte/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
33
|
flyte/_bin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
34
34
|
flyte/_bin/debug.py,sha256=hnX2tlv9QbqckoT5CJ3c3apJj3tGDpsrdV7ZAsE7j34,911
|
|
35
|
-
flyte/_bin/runtime.py,sha256=
|
|
35
|
+
flyte/_bin/runtime.py,sha256=4MUSHHPznY6mcfy9nEr48KaM2pFIIScEei_J779r6KU,6227
|
|
36
36
|
flyte/_cache/__init__.py,sha256=zhdO5UuHQRdzn8GHmSN40nrxfAmI4ihDRuHZM11U84Y,305
|
|
37
37
|
flyte/_cache/cache.py,sha256=-oS5q6QpkxPqZ8gWkn8uVNgzx3JFuXaWLV-h0KVKW9c,5010
|
|
38
38
|
flyte/_cache/defaults.py,sha256=gzJZW0QJPUfd2OPnGpv3tzIfwPtgFjAKoie3NP1P97U,217
|
|
@@ -58,8 +58,8 @@ flyte/_internal/controllers/remote/_core.py,sha256=5qIGFNTilJwtYLQ_KQoawLQ172ih-
|
|
|
58
58
|
flyte/_internal/controllers/remote/_informer.py,sha256=M2uqfe0oz9ySkYK94UFLoTPRt9UoaYQKzmy4lrVOchY,14912
|
|
59
59
|
flyte/_internal/controllers/remote/_service_protocol.py,sha256=B9qbIg6DiGeac-iSccLmX_AL2xUgX4ezNUOiAbSy4V0,1357
|
|
60
60
|
flyte/_internal/imagebuild/__init__.py,sha256=dwXdJ1jMhw9RF8itF7jkPLanvX1yCviSns7hE5eoIts,102
|
|
61
|
-
flyte/_internal/imagebuild/docker_builder.py,sha256=
|
|
62
|
-
flyte/_internal/imagebuild/image_builder.py,sha256=
|
|
61
|
+
flyte/_internal/imagebuild/docker_builder.py,sha256=jaKs67PqPKTfQaUWN6VHF_TN9jWLSWJgJLw6Ltn9RoA,24047
|
|
62
|
+
flyte/_internal/imagebuild/image_builder.py,sha256=2nqcTX0ry2lxTQYmHZzyXZ9sIcfir6a9fOCNLZkAv6Y,11267
|
|
63
63
|
flyte/_internal/imagebuild/remote_builder.py,sha256=QOs0miveAnDEDxXCNRnkblw7DB-BJwQUOeL0Y_9ao2A,13120
|
|
64
64
|
flyte/_internal/imagebuild/utils.py,sha256=2MMCNyyMoPLO8sJ5J9TnrZwd1xWTty5AW2VVp8ZUd_M,1250
|
|
65
65
|
flyte/_internal/resolvers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -73,11 +73,11 @@ flyte/_internal/runtime/io.py,sha256=ysL7hMpfVumvsEYWOM-_VPa8MXn5_X_CZorKbOThyv4
|
|
|
73
73
|
flyte/_internal/runtime/resources_serde.py,sha256=TObMVsSjVcQhcY8-nY81pbvrz7TP-adDD5xV-LqAaxM,4813
|
|
74
74
|
flyte/_internal/runtime/reuse.py,sha256=uGjomQz4X4JvPv6M8eDW94g4YfHJugNYufwK6eVFlwI,4901
|
|
75
75
|
flyte/_internal/runtime/rusty.py,sha256=e2uSg9-Hooa65-BIuqXhIrEq86RHteFFs3W7dDuM3uo,6946
|
|
76
|
-
flyte/_internal/runtime/task_serde.py,sha256=
|
|
76
|
+
flyte/_internal/runtime/task_serde.py,sha256=ZDym2IujC70KzePrDnB0GteaCcP-87x_C44n3bLmr_Q,14504
|
|
77
77
|
flyte/_internal/runtime/taskrunner.py,sha256=T1dviDXYds18eSwT_J6AMvD4oY91t1r_31EAFA5grO0,7683
|
|
78
78
|
flyte/_internal/runtime/types_serde.py,sha256=EjRh9Yypx9-20XXQprtNgp766LeQVRoYWtY6XPGMZQg,1813
|
|
79
79
|
flyte/_keyring/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
80
|
-
flyte/_keyring/file.py,sha256=
|
|
80
|
+
flyte/_keyring/file.py,sha256=MKAHv7B6rFV0gJN509jSMSG7beu-wEpizvDaQ1uuwLY,2962
|
|
81
81
|
flyte/_protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
82
82
|
flyte/_protos/common/authorization_pb2.py,sha256=6G7CAfq_Vq1qrm8JFkAnMAj0AaEipiX7MkjA7nk91-M,6707
|
|
83
83
|
flyte/_protos/common/authorization_pb2.pyi,sha256=tdqc3wZo3Yc6lKfjVgJlUFUFzGv4GAaCknIv43RGd-8,4759
|
|
@@ -166,15 +166,15 @@ flyte/_utils/uv_script_parser.py,sha256=PxqD8lSMi6xv0uDd1s8LKB2IPZr4ttZJCUweqlyM
|
|
|
166
166
|
flyte/cli/__init__.py,sha256=aeCcumeP9xD_5aCmaRYUPCe2QRJSGCaxcUbTZ3co768,341
|
|
167
167
|
flyte/cli/_abort.py,sha256=Ty-63Gtd2PUn6lCuL5AaasfBoPu7TDSU5EQKVbkF4qw,661
|
|
168
168
|
flyte/cli/_build.py,sha256=S8I53ou-rt3m54A_Sjg2sPU_Cw7D1GpkS5DO766sfk4,3534
|
|
169
|
-
flyte/cli/_common.py,sha256=
|
|
169
|
+
flyte/cli/_common.py,sha256=36VI1dPaFBjzmvqUd-5Pq3VEg5Tp7NAlwtfRFrVYChA,13639
|
|
170
170
|
flyte/cli/_create.py,sha256=3nNf5XxsynITyJljvDYlj2la7ywARF9Tg7GbuIOo1Yw,5640
|
|
171
171
|
flyte/cli/_delete.py,sha256=VTmXv09PBjkdtyl23mbSjIQQlN7Y1AI_bO0GkHP-f9E,546
|
|
172
|
-
flyte/cli/_deploy.py,sha256=
|
|
172
|
+
flyte/cli/_deploy.py,sha256=EhdNaz1sQR1M16-hi2L7mpA0I_SeUgohirFFv4ogNwE,9220
|
|
173
173
|
flyte/cli/_gen.py,sha256=7K2eYQLGVr26I2OC3Xe_bzAn4ANYA5mPlBW5m1476PM,6079
|
|
174
174
|
flyte/cli/_get.py,sha256=E2HTekZcVbTtwy6LnM5WHzF41aT1uPRWqV1rYB4H5B8,10420
|
|
175
175
|
flyte/cli/_option.py,sha256=oC1Gs0u0UrOC1SsrFo-iCuAkqQvI1wJWCdjYXA9rW4Q,1445
|
|
176
176
|
flyte/cli/_params.py,sha256=oCSnBh6fDSS1HLFZvNRstfXlaJHTWaOVwMQ_R6icehs,20009
|
|
177
|
-
flyte/cli/_run.py,sha256=
|
|
177
|
+
flyte/cli/_run.py,sha256=yP8lh5Zgl2y1-tLCH9d7GWSNYJK03oSo1quO00GC_Sk,16742
|
|
178
178
|
flyte/cli/main.py,sha256=t5Ivjipd6bVHIGjRBGwkeP577j59ASq9c1wgoNf3h2c,5334
|
|
179
179
|
flyte/config/__init__.py,sha256=MiwEYK5Iv7MRR22z61nzbsbvZ9Q6MdmAU_g9If1Pmb8,144
|
|
180
180
|
flyte/config/_config.py,sha256=gsGtIZa9CUILfBmWm2CaEMRLHcSOeNUgwPHeIV807bs,10942
|
|
@@ -193,7 +193,7 @@ flyte/io/_dataframe/__init__.py,sha256=SDgNw45uf7m3cHhbUCOA3V3-5A2zSKgPcsWriRLwd
|
|
|
193
193
|
flyte/io/_dataframe/basic_dfs.py,sha256=tUJKiBbIJ30r9hjwMo-y3bxjdLmyf3WVe9n9tYskKWk,8043
|
|
194
194
|
flyte/io/_dataframe/dataframe.py,sha256=HOW6W12uAlN_1YH9fOgk-HaAse3ACgVyb18vsa83Uzo,49371
|
|
195
195
|
flyte/remote/__init__.py,sha256=y9eke9JzEJkygk8eKZjSj44fJGlyepuW4i-j6lbCAPY,617
|
|
196
|
-
flyte/remote/_action.py,sha256=
|
|
196
|
+
flyte/remote/_action.py,sha256=wYP_tD0tOnjhOAUP3CzyJfiE6tzE2IePXa48qf8Q3qs,24324
|
|
197
197
|
flyte/remote/_common.py,sha256=2XLLxWL1NjwfdPQUhOfYn3zMrg-yGNfi6NYIaqHutUA,862
|
|
198
198
|
flyte/remote/_console.py,sha256=avmELJPx8nQMAVPrHlh6jEIRPjrMwFpdZjJsWOOa9rE,660
|
|
199
199
|
flyte/remote/_data.py,sha256=WeRkhUrJlx7rU3CpvbPvikosv7r02UMveWrf70cFilQ,6156
|
|
@@ -235,16 +235,16 @@ flyte/syncify/__init__.py,sha256=WgTk-v-SntULnI55CsVy71cxGJ9Q6pxpTrhbPFuouJ0,197
|
|
|
235
235
|
flyte/syncify/_api.py,sha256=k4LQB8odJb5Fx2dabL340g0Tq1bKfSG_ZHsV5qRodE0,14858
|
|
236
236
|
flyte/types/__init__.py,sha256=9310PRtVrwJePwEPeoUO0HPyIkgaja7-Dar_QlE_MUI,1745
|
|
237
237
|
flyte/types/_interface.py,sha256=5y9EC5r897xz03Hh0ltF8QVGKMfMfAznws-hKSEO4Go,1677
|
|
238
|
-
flyte/types/_pickle.py,sha256=
|
|
238
|
+
flyte/types/_pickle.py,sha256=onpJDslzTugnicbqbkBkCFe7JW0YtU06xkp6Yd7ry70,4661
|
|
239
239
|
flyte/types/_renderer.py,sha256=ygcCo5l60lHufyQISFddZfWwLlQ8kJAKxUT_XnR_6dY,4818
|
|
240
240
|
flyte/types/_string_literals.py,sha256=NlG1xV8RSA-sZ-n-IFQCAsdB6jXJOAKkHWtnopxVVDk,4231
|
|
241
|
-
flyte/types/_type_engine.py,sha256=
|
|
241
|
+
flyte/types/_type_engine.py,sha256=xGxuNQK2kShdvxcAvN5v0yP704EXMraM_d7gBPJgHn4,95711
|
|
242
242
|
flyte/types/_utils.py,sha256=pbts9E1_2LTdLygAY0UYTLYJ8AsN3BZyviSXvrtcutc,2626
|
|
243
|
-
flyte-2.0.
|
|
244
|
-
flyte-2.0.
|
|
245
|
-
flyte-2.0.
|
|
246
|
-
flyte-2.0.
|
|
247
|
-
flyte-2.0.
|
|
248
|
-
flyte-2.0.
|
|
249
|
-
flyte-2.0.
|
|
250
|
-
flyte-2.0.
|
|
243
|
+
flyte-2.0.0b22.data/scripts/debug.py,sha256=hnX2tlv9QbqckoT5CJ3c3apJj3tGDpsrdV7ZAsE7j34,911
|
|
244
|
+
flyte-2.0.0b22.data/scripts/runtime.py,sha256=4MUSHHPznY6mcfy9nEr48KaM2pFIIScEei_J779r6KU,6227
|
|
245
|
+
flyte-2.0.0b22.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
246
|
+
flyte-2.0.0b22.dist-info/METADATA,sha256=GMmE_XSJ2wEFNqxe5C89PJeznQhVEmP531uc2HIHruk,10045
|
|
247
|
+
flyte-2.0.0b22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
248
|
+
flyte-2.0.0b22.dist-info/entry_points.txt,sha256=rb43Gfxw40iPH5B6EUs6Ter0ekLkGXsj7R890_MOTyk,136
|
|
249
|
+
flyte-2.0.0b22.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
|
|
250
|
+
flyte-2.0.0b22.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|