flyte 2.0.0b9__py3-none-any.whl → 2.0.0b14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/__init__.py +55 -31
- flyte/_bin/debug.py +38 -0
- flyte/_bin/runtime.py +13 -0
- flyte/_code_bundle/_utils.py +2 -0
- flyte/_code_bundle/bundle.py +4 -4
- flyte/_context.py +1 -1
- flyte/_debug/__init__.py +0 -0
- flyte/_debug/constants.py +39 -0
- flyte/_debug/utils.py +17 -0
- flyte/_debug/vscode.py +300 -0
- flyte/_environment.py +5 -5
- flyte/_image.py +34 -19
- flyte/_initialize.py +15 -29
- flyte/_internal/controllers/remote/_action.py +2 -2
- flyte/_internal/controllers/remote/_controller.py +1 -1
- flyte/_internal/imagebuild/docker_builder.py +11 -15
- flyte/_internal/imagebuild/remote_builder.py +71 -22
- flyte/_internal/runtime/entrypoints.py +3 -0
- flyte/_internal/runtime/reuse.py +7 -3
- flyte/_internal/runtime/task_serde.py +4 -3
- flyte/_internal/runtime/taskrunner.py +9 -3
- flyte/_logging.py +5 -2
- flyte/_protos/common/identifier_pb2.py +25 -19
- flyte/_protos/common/identifier_pb2.pyi +10 -0
- flyte/_protos/imagebuilder/definition_pb2.py +32 -31
- flyte/_protos/imagebuilder/definition_pb2.pyi +25 -12
- flyte/_protos/workflow/queue_service_pb2.py +24 -24
- flyte/_protos/workflow/queue_service_pb2.pyi +6 -6
- flyte/_protos/workflow/run_definition_pb2.py +48 -48
- flyte/_protos/workflow/run_definition_pb2.pyi +20 -10
- flyte/_reusable_environment.py +41 -19
- flyte/_run.py +9 -9
- flyte/_secret.py +9 -5
- flyte/_task.py +16 -11
- flyte/_task_environment.py +11 -13
- flyte/_tools.py +0 -13
- flyte/_version.py +16 -3
- flyte/cli/_build.py +2 -3
- flyte/cli/_common.py +16 -5
- flyte/cli/_gen.py +10 -1
- flyte/cli/_get.py +16 -14
- flyte/cli/_run.py +258 -25
- flyte/models.py +9 -0
- flyte/remote/_client/auth/_authenticators/base.py +8 -2
- flyte/remote/_client/auth/_authenticators/device_code.py +1 -1
- flyte/remote/_client/auth/_authenticators/pkce.py +1 -1
- flyte/remote/_client/auth/_channel.py +0 -6
- flyte/remote/_client/auth/_client_config.py +4 -2
- flyte/remote/_client/controlplane.py +14 -0
- flyte/remote/_task.py +18 -4
- flyte/storage/_storage.py +83 -7
- flyte/types/_type_engine.py +3 -33
- flyte-2.0.0b14.data/scripts/debug.py +38 -0
- {flyte-2.0.0b9.data → flyte-2.0.0b14.data}/scripts/runtime.py +13 -0
- {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/METADATA +2 -2
- {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/RECORD +60 -54
- {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/WHEEL +0 -0
- {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/entry_points.txt +0 -0
- {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/licenses/LICENSE +0 -0
- {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/top_level.txt +0 -0
flyte/_image.py
CHANGED
|
@@ -252,9 +252,15 @@ class AptPackages(Layer):
|
|
|
252
252
|
@dataclass(frozen=True, repr=True)
|
|
253
253
|
class Commands(Layer):
|
|
254
254
|
commands: Tuple[str, ...]
|
|
255
|
+
secret_mounts: Optional[Tuple[str | Secret, ...]] = None
|
|
255
256
|
|
|
256
257
|
def update_hash(self, hasher: hashlib._Hash):
|
|
257
|
-
|
|
258
|
+
hash_input = "".join(self.commands)
|
|
259
|
+
|
|
260
|
+
if self.secret_mounts:
|
|
261
|
+
for secret_mount in self.secret_mounts:
|
|
262
|
+
hash_input += str(secret_mount)
|
|
263
|
+
hasher.update(hash_input.encode("utf-8"))
|
|
258
264
|
|
|
259
265
|
|
|
260
266
|
@rich.repr.auto
|
|
@@ -279,7 +285,7 @@ class DockerIgnore(Layer):
|
|
|
279
285
|
@dataclass(frozen=True, repr=True)
|
|
280
286
|
class CopyConfig(Layer):
|
|
281
287
|
path_type: CopyConfigType = field(metadata={"identifier": True})
|
|
282
|
-
src: Path = field(metadata={"identifier":
|
|
288
|
+
src: Path = field(metadata={"identifier": False})
|
|
283
289
|
dst: str
|
|
284
290
|
src_name: str = field(init=False)
|
|
285
291
|
|
|
@@ -451,11 +457,7 @@ class Image:
|
|
|
451
457
|
# this default image definition may need to be updated once there is a released pypi version
|
|
452
458
|
from flyte._version import __version__
|
|
453
459
|
|
|
454
|
-
dev_mode = (
|
|
455
|
-
(cls._is_editable_install() or (__version__ and "dev" in __version__))
|
|
456
|
-
and not flyte_version
|
|
457
|
-
and install_flyte
|
|
458
|
-
)
|
|
460
|
+
dev_mode = (__version__ and "dev" in __version__) and not flyte_version and install_flyte
|
|
459
461
|
if install_flyte is False:
|
|
460
462
|
preset_tag = f"py{python_version[0]}.{python_version[1]}"
|
|
461
463
|
else:
|
|
@@ -507,13 +509,6 @@ class Image:
|
|
|
507
509
|
|
|
508
510
|
return image
|
|
509
511
|
|
|
510
|
-
@staticmethod
|
|
511
|
-
def _is_editable_install():
|
|
512
|
-
"""Internal hacky function to see if the current install is editable or not."""
|
|
513
|
-
curr = Path(__file__)
|
|
514
|
-
pyproject = curr.parent.parent.parent / "pyproject.toml"
|
|
515
|
-
return pyproject.exists()
|
|
516
|
-
|
|
517
512
|
@classmethod
|
|
518
513
|
def from_debian_base(
|
|
519
514
|
cls,
|
|
@@ -793,9 +788,24 @@ class Image:
|
|
|
793
788
|
|
|
794
789
|
Example:
|
|
795
790
|
```python
|
|
796
|
-
@flyte.task(image=(flyte.Image
|
|
797
|
-
|
|
798
|
-
|
|
791
|
+
@flyte.task(image=(flyte.Image.from_debian_base().with_pip_packages("requests", "numpy")))
|
|
792
|
+
def my_task(x: int) -> int:
|
|
793
|
+
import numpy as np
|
|
794
|
+
return np.sum([x, 1])
|
|
795
|
+
```
|
|
796
|
+
|
|
797
|
+
To mount secrets during the build process to download private packages, you can use the `secret_mounts`.
|
|
798
|
+
In the below example, "GITHUB_PAT" will be mounted as env var "GITHUB_PAT",
|
|
799
|
+
and "apt-secret" will be mounted at /etc/apt/apt-secret.
|
|
800
|
+
Example:
|
|
801
|
+
```python
|
|
802
|
+
private_package = "git+https://$GITHUB_PAT@github.com/flyteorg/flytex.git@2e20a2acebfc3877d84af643fdd768edea41d533"
|
|
803
|
+
@flyte.task(
|
|
804
|
+
image=(
|
|
805
|
+
flyte.Image.from_debian_base()
|
|
806
|
+
.with_pip_packages("private_package", secret_mounts=[Secret(key="GITHUB_PAT")])
|
|
807
|
+
.with_apt_packages("git", secret_mounts=[Secret(key="apt-secret", mount="/etc/apt/apt-secret")])
|
|
808
|
+
)
|
|
799
809
|
def my_task(x: int) -> int:
|
|
800
810
|
import numpy as np
|
|
801
811
|
return np.sum([x, 1])
|
|
@@ -920,16 +930,21 @@ class Image:
|
|
|
920
930
|
)
|
|
921
931
|
return new_image
|
|
922
932
|
|
|
923
|
-
def with_commands(self, commands: List[str]) -> Image:
|
|
933
|
+
def with_commands(self, commands: List[str], secret_mounts: Optional[SecretRequest] = None) -> Image:
|
|
924
934
|
"""
|
|
925
935
|
Use this method to create a new image with the specified commands layered on top of the current image
|
|
926
936
|
Be sure not to use RUN in your command.
|
|
927
937
|
|
|
928
938
|
:param commands: list of commands to run
|
|
939
|
+
:param secret_mounts: list of secret mounts to use for the build process.
|
|
929
940
|
:return: Image
|
|
930
941
|
"""
|
|
931
942
|
new_commands: Tuple = _ensure_tuple(commands)
|
|
932
|
-
new_image = self.clone(
|
|
943
|
+
new_image = self.clone(
|
|
944
|
+
addl_layer=Commands(
|
|
945
|
+
commands=new_commands, secret_mounts=_ensure_tuple(secret_mounts) if secret_mounts else None
|
|
946
|
+
)
|
|
947
|
+
)
|
|
933
948
|
return new_image
|
|
934
949
|
|
|
935
950
|
def with_local_v2(self) -> Image:
|
flyte/_initialize.py
CHANGED
|
@@ -11,7 +11,6 @@ from flyte.errors import InitializationError
|
|
|
11
11
|
from flyte.syncify import syncify
|
|
12
12
|
|
|
13
13
|
from ._logging import initialize_logger, logger
|
|
14
|
-
from ._tools import ipython_check
|
|
15
14
|
|
|
16
15
|
if TYPE_CHECKING:
|
|
17
16
|
from flyte._internal.imagebuild import ImageBuildEngine
|
|
@@ -173,6 +172,7 @@ async def init(
|
|
|
173
172
|
|
|
174
173
|
:return: None
|
|
175
174
|
"""
|
|
175
|
+
from flyte._tools import ipython_check
|
|
176
176
|
from flyte._utils import get_cwd_editable_install, org_from_endpoint, sanitize_endpoint
|
|
177
177
|
|
|
178
178
|
interactive_mode = ipython_check()
|
|
@@ -205,6 +205,14 @@ async def init(
|
|
|
205
205
|
http_proxy_url=http_proxy_url,
|
|
206
206
|
)
|
|
207
207
|
|
|
208
|
+
if not root_dir:
|
|
209
|
+
editable_root = get_cwd_editable_install()
|
|
210
|
+
if editable_root:
|
|
211
|
+
logger.info(f"Using editable install as root directory: {editable_root}")
|
|
212
|
+
root_dir = editable_root
|
|
213
|
+
else:
|
|
214
|
+
logger.info("No editable install found, using current working directory as root directory.")
|
|
215
|
+
root_dir = Path.cwd()
|
|
208
216
|
root_dir = root_dir or get_cwd_editable_install() or Path.cwd()
|
|
209
217
|
_init_config = _InitConfig(
|
|
210
218
|
root_dir=root_dir,
|
|
@@ -242,17 +250,19 @@ async def init_from_config(
|
|
|
242
250
|
cfg: config.Config
|
|
243
251
|
if path_or_config is None or isinstance(path_or_config, str):
|
|
244
252
|
# If a string is passed, treat it as a path to the config file
|
|
245
|
-
if path_or_config:
|
|
253
|
+
if root_dir and path_or_config:
|
|
254
|
+
cfg = config.auto(str(root_dir / path_or_config))
|
|
255
|
+
elif path_or_config:
|
|
246
256
|
if not Path(path_or_config).exists():
|
|
247
257
|
raise InitializationError(
|
|
248
258
|
"ConfigFileNotFoundError",
|
|
249
259
|
"user",
|
|
250
260
|
f"Configuration file '{path_or_config}' does not exist., current working directory is {Path.cwd()}",
|
|
251
261
|
)
|
|
252
|
-
if root_dir and path_or_config:
|
|
253
|
-
cfg = config.auto(str(root_dir / path_or_config))
|
|
254
|
-
else:
|
|
255
262
|
cfg = config.auto(path_or_config)
|
|
263
|
+
else:
|
|
264
|
+
# If no path is provided, use the default config file
|
|
265
|
+
cfg = config.auto()
|
|
256
266
|
else:
|
|
257
267
|
# If a Config object is passed, use it directly
|
|
258
268
|
cfg = path_or_config
|
|
@@ -374,30 +384,6 @@ def ensure_client():
|
|
|
374
384
|
)
|
|
375
385
|
|
|
376
386
|
|
|
377
|
-
def requires_client(func: T) -> T:
|
|
378
|
-
"""
|
|
379
|
-
Decorator that checks if the client has been initialized before executing the function.
|
|
380
|
-
Raises InitializationError if the client is not initialized.
|
|
381
|
-
|
|
382
|
-
:param func: Function to decorate
|
|
383
|
-
:return: Decorated function that checks for initialization
|
|
384
|
-
"""
|
|
385
|
-
|
|
386
|
-
@functools.wraps(func)
|
|
387
|
-
async def wrapper(*args, **kwargs) -> T:
|
|
388
|
-
init_config = _get_init_config()
|
|
389
|
-
if init_config is None or init_config.client is None:
|
|
390
|
-
raise InitializationError(
|
|
391
|
-
"ClientNotInitializedError",
|
|
392
|
-
"user",
|
|
393
|
-
f"Function '{func.__name__}' requires client to be initialized. "
|
|
394
|
-
f"Call flyte.init() with a valid endpoint or api-key before using this function.",
|
|
395
|
-
)
|
|
396
|
-
return func(*args, **kwargs)
|
|
397
|
-
|
|
398
|
-
return typing.cast(T, wrapper)
|
|
399
|
-
|
|
400
|
-
|
|
401
387
|
def requires_storage(func: T) -> T:
|
|
402
388
|
"""
|
|
403
389
|
Decorator that checks if the storage has been initialized before executing the function.
|
|
@@ -183,7 +183,7 @@ class Action:
|
|
|
183
183
|
et.FromSeconds(int(end_time))
|
|
184
184
|
et.nanos = int((end_time % 1) * 1e9)
|
|
185
185
|
|
|
186
|
-
|
|
186
|
+
_ = (
|
|
187
187
|
task_definition_pb2.TaskSpec(task_template=tasks_pb2.TaskTemplate(interface=typed_interface))
|
|
188
188
|
if typed_interface
|
|
189
189
|
else None
|
|
@@ -208,6 +208,6 @@ class Action:
|
|
|
208
208
|
output_uri=outputs_uri,
|
|
209
209
|
report_uri=report_uri,
|
|
210
210
|
),
|
|
211
|
-
spec=spec,
|
|
211
|
+
# spec=spec,
|
|
212
212
|
),
|
|
213
213
|
)
|
|
@@ -167,7 +167,7 @@ class RemoteController(Controller):
|
|
|
167
167
|
# It is not allowed to change the code bundle (for regular code bundles) in the middle of a run.
|
|
168
168
|
code_bundle = tctx.code_bundle
|
|
169
169
|
|
|
170
|
-
if code_bundle and code_bundle.pkl:
|
|
170
|
+
if tctx.interactive_mode or (code_bundle and code_bundle.pkl):
|
|
171
171
|
logger.debug(f"Building new pkl bundle for task {_task.name}")
|
|
172
172
|
code_bundle = await build_pkl_bundle(
|
|
173
173
|
_task,
|
|
@@ -302,11 +302,9 @@ def _get_secret_commands(layers: typing.Tuple[Layer, ...]) -> typing.List[str]:
|
|
|
302
302
|
commands = []
|
|
303
303
|
|
|
304
304
|
def _get_secret_command(secret: str | Secret) -> typing.List[str]:
|
|
305
|
-
secret_id = hash(secret)
|
|
306
305
|
if isinstance(secret, str):
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
return ["--secret", f"id={secret_id},src={secret}"]
|
|
306
|
+
secret = Secret(key=secret)
|
|
307
|
+
secret_id = hash(secret)
|
|
310
308
|
secret_env_key = "_".join([k.upper() for k in filter(None, (secret.group, secret.key))])
|
|
311
309
|
secret_env = os.getenv(secret_env_key)
|
|
312
310
|
if secret_env:
|
|
@@ -329,18 +327,16 @@ def _get_secret_mounts_layer(secrets: typing.Tuple[str | Secret, ...] | None) ->
|
|
|
329
327
|
if secrets is None:
|
|
330
328
|
return ""
|
|
331
329
|
secret_mounts_layer = ""
|
|
332
|
-
for
|
|
330
|
+
for s in secrets:
|
|
331
|
+
secret = Secret(key=s) if isinstance(s, str) else s
|
|
333
332
|
secret_id = hash(secret)
|
|
334
|
-
if
|
|
335
|
-
secret_mounts_layer += f"--mount=type=secret,id={secret_id},target
|
|
336
|
-
elif
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
else:
|
|
342
|
-
secret_file_name = "_".join(list(filter(None, (secret.group, secret.key))))
|
|
343
|
-
secret_mounts_layer += f"--mount=type=secret,id={secret_id},src=/run/secrets/{secret_file_name}"
|
|
333
|
+
if secret.mount:
|
|
334
|
+
secret_mounts_layer += f"--mount=type=secret,id={secret_id},target={secret.mount}"
|
|
335
|
+
elif secret.as_env_var:
|
|
336
|
+
secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret.as_env_var}"
|
|
337
|
+
else:
|
|
338
|
+
secret_default_env_key = "_".join(list(filter(None, (secret.group, secret.key))))
|
|
339
|
+
secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret_default_env_key}"
|
|
344
340
|
|
|
345
341
|
return secret_mounts_layer
|
|
346
342
|
|
|
@@ -19,21 +19,26 @@ from flyte._image import (
|
|
|
19
19
|
CopyConfig,
|
|
20
20
|
DockerIgnore,
|
|
21
21
|
Env,
|
|
22
|
+
PipOption,
|
|
22
23
|
PipPackages,
|
|
23
24
|
PythonWheels,
|
|
24
25
|
Requirements,
|
|
25
26
|
UVProject,
|
|
26
27
|
UVScript,
|
|
28
|
+
WorkDir,
|
|
27
29
|
)
|
|
28
30
|
from flyte._internal.imagebuild.image_builder import ImageBuilder, ImageChecker
|
|
29
31
|
from flyte._internal.imagebuild.utils import copy_files_to_context
|
|
32
|
+
from flyte._internal.runtime.task_serde import get_security_context
|
|
30
33
|
from flyte._logging import logger
|
|
34
|
+
from flyte._secret import Secret
|
|
31
35
|
from flyte.remote import ActionOutputs, Run
|
|
32
36
|
|
|
33
37
|
if TYPE_CHECKING:
|
|
34
38
|
from flyte._protos.imagebuilder import definition_pb2 as image_definition_pb2
|
|
35
39
|
|
|
36
40
|
IMAGE_TASK_NAME = "build-image"
|
|
41
|
+
OPTIMIZE_TASK_NAME = "optimize-task"
|
|
37
42
|
IMAGE_TASK_PROJECT = "system"
|
|
38
43
|
IMAGE_TASK_DOMAIN = "production"
|
|
39
44
|
|
|
@@ -95,12 +100,12 @@ class RemoteImageBuilder(ImageBuilder):
|
|
|
95
100
|
spec, context = await _validate_configuration(image)
|
|
96
101
|
|
|
97
102
|
start = datetime.now(timezone.utc)
|
|
98
|
-
entity = remote.Task.get(
|
|
103
|
+
entity = await remote.Task.get(
|
|
99
104
|
name=IMAGE_TASK_NAME,
|
|
100
105
|
project=IMAGE_TASK_PROJECT,
|
|
101
106
|
domain=IMAGE_TASK_DOMAIN,
|
|
102
107
|
auto_version="latest",
|
|
103
|
-
)
|
|
108
|
+
).override.aio(secrets=_get_build_secrets_from_image(image))
|
|
104
109
|
run = cast(
|
|
105
110
|
Run,
|
|
106
111
|
await flyte.with_runcontext(project=IMAGE_TASK_PROJECT, domain=IMAGE_TASK_DOMAIN).run.aio(
|
|
@@ -117,6 +122,19 @@ class RemoteImageBuilder(ImageBuilder):
|
|
|
117
122
|
|
|
118
123
|
if run_details.action_details.raw_phase == run_definition_pb2.PHASE_SUCCEEDED:
|
|
119
124
|
logger.warning(click.style(f"✅ Build completed in {elapsed}!", bold=True, fg="green"))
|
|
125
|
+
try:
|
|
126
|
+
entity = remote.Task.get(
|
|
127
|
+
name=OPTIMIZE_TASK_NAME,
|
|
128
|
+
project=IMAGE_TASK_PROJECT,
|
|
129
|
+
domain=IMAGE_TASK_DOMAIN,
|
|
130
|
+
auto_version="latest",
|
|
131
|
+
)
|
|
132
|
+
await flyte.with_runcontext(project=IMAGE_TASK_PROJECT, domain=IMAGE_TASK_DOMAIN).run.aio(
|
|
133
|
+
entity, spec=spec, context=context, target_image=image_name
|
|
134
|
+
)
|
|
135
|
+
except Exception as e:
|
|
136
|
+
# Ignore the error if optimize is not enabled in the backend.
|
|
137
|
+
logger.warning(f"Failed to run optimize task with error: {e}")
|
|
120
138
|
else:
|
|
121
139
|
raise flyte.errors.ImageBuildError(f"❌ Build failed in {elapsed} at {click.style(run.url, fg='cyan')}")
|
|
122
140
|
|
|
@@ -164,9 +182,27 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
|
|
|
164
182
|
|
|
165
183
|
layers = []
|
|
166
184
|
for layer in image._layers:
|
|
185
|
+
secret_mounts = None
|
|
186
|
+
pip_options = None
|
|
187
|
+
|
|
188
|
+
if isinstance(layer, PipOption):
|
|
189
|
+
pip_options = image_definition_pb2.PipOptions(
|
|
190
|
+
index_url=layer.index_url,
|
|
191
|
+
extra_index_urls=layer.extra_index_urls,
|
|
192
|
+
pre=layer.pre,
|
|
193
|
+
extra_args=layer.extra_args,
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
if hasattr(layer, "secret_mounts"):
|
|
197
|
+
sc = get_security_context(layer.secret_mounts)
|
|
198
|
+
secret_mounts = sc.secrets if sc else None
|
|
199
|
+
|
|
167
200
|
if isinstance(layer, AptPackages):
|
|
168
201
|
apt_layer = image_definition_pb2.Layer(
|
|
169
|
-
apt_packages=image_definition_pb2.AptPackages(
|
|
202
|
+
apt_packages=image_definition_pb2.AptPackages(
|
|
203
|
+
packages=layer.packages,
|
|
204
|
+
secret_mounts=secret_mounts,
|
|
205
|
+
),
|
|
170
206
|
)
|
|
171
207
|
layers.append(apt_layer)
|
|
172
208
|
elif isinstance(layer, PythonWheels):
|
|
@@ -174,12 +210,8 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
|
|
|
174
210
|
wheel_layer = image_definition_pb2.Layer(
|
|
175
211
|
python_wheels=image_definition_pb2.PythonWheels(
|
|
176
212
|
dir=str(dst_path.relative_to(context_path)),
|
|
177
|
-
options=
|
|
178
|
-
|
|
179
|
-
extra_index_urls=layer.extra_index_urls,
|
|
180
|
-
pre=layer.pre,
|
|
181
|
-
extra_args=layer.extra_args,
|
|
182
|
-
),
|
|
213
|
+
options=pip_options,
|
|
214
|
+
secret_mounts=secret_mounts,
|
|
183
215
|
)
|
|
184
216
|
)
|
|
185
217
|
layers.append(wheel_layer)
|
|
@@ -189,12 +221,8 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
|
|
|
189
221
|
requirements_layer = image_definition_pb2.Layer(
|
|
190
222
|
requirements=image_definition_pb2.Requirements(
|
|
191
223
|
file=str(dst_path.relative_to(context_path)),
|
|
192
|
-
options=
|
|
193
|
-
|
|
194
|
-
extra_index_urls=layer.extra_index_urls,
|
|
195
|
-
pre=layer.pre,
|
|
196
|
-
extra_args=layer.extra_args,
|
|
197
|
-
),
|
|
224
|
+
options=pip_options,
|
|
225
|
+
secret_mounts=secret_mounts,
|
|
198
226
|
)
|
|
199
227
|
)
|
|
200
228
|
layers.append(requirements_layer)
|
|
@@ -211,12 +239,8 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
|
|
|
211
239
|
pip_layer = image_definition_pb2.Layer(
|
|
212
240
|
pip_packages=image_definition_pb2.PipPackages(
|
|
213
241
|
packages=packages,
|
|
214
|
-
options=
|
|
215
|
-
|
|
216
|
-
extra_index_urls=layer.extra_index_urls,
|
|
217
|
-
pre=layer.pre,
|
|
218
|
-
extra_args=layer.extra_args,
|
|
219
|
-
),
|
|
242
|
+
options=pip_options,
|
|
243
|
+
secret_mounts=secret_mounts,
|
|
220
244
|
)
|
|
221
245
|
)
|
|
222
246
|
layers.append(pip_layer)
|
|
@@ -235,7 +259,10 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
|
|
|
235
259
|
layers.append(uv_layer)
|
|
236
260
|
elif isinstance(layer, Commands):
|
|
237
261
|
commands_layer = image_definition_pb2.Layer(
|
|
238
|
-
commands=image_definition_pb2.Commands(
|
|
262
|
+
commands=image_definition_pb2.Commands(
|
|
263
|
+
cmd=list(layer.commands),
|
|
264
|
+
secret_mounts=secret_mounts,
|
|
265
|
+
)
|
|
239
266
|
)
|
|
240
267
|
layers.append(commands_layer)
|
|
241
268
|
elif isinstance(layer, DockerIgnore):
|
|
@@ -257,6 +284,11 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
|
|
|
257
284
|
)
|
|
258
285
|
)
|
|
259
286
|
layers.append(env_layer)
|
|
287
|
+
elif isinstance(layer, WorkDir):
|
|
288
|
+
workdir_layer = image_definition_pb2.Layer(
|
|
289
|
+
workdir=image_definition_pb2.WorkDir(workdir=layer.workdir),
|
|
290
|
+
)
|
|
291
|
+
layers.append(workdir_layer)
|
|
260
292
|
|
|
261
293
|
return image_definition_pb2.ImageSpec(
|
|
262
294
|
base_image=image.base_image,
|
|
@@ -267,3 +299,20 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
|
|
|
267
299
|
|
|
268
300
|
def _get_fully_qualified_image_name(outputs: ActionOutputs) -> str:
|
|
269
301
|
return outputs.pb2.literals[0].value.scalar.primitive.string_value
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _get_build_secrets_from_image(image: Image) -> Optional[typing.List[Secret]]:
|
|
305
|
+
secrets = []
|
|
306
|
+
DEFAULT_SECRET_DIR = Path("etc/flyte/secrets")
|
|
307
|
+
for layer in image._layers:
|
|
308
|
+
if isinstance(layer, (PipOption, Commands, AptPackages)) and layer.secret_mounts is not None:
|
|
309
|
+
for secret_mount in layer.secret_mounts:
|
|
310
|
+
# Mount all the image secrets to a default directory that will be passed to the BuildKit server.
|
|
311
|
+
if isinstance(secret_mount, Secret):
|
|
312
|
+
secrets.append(Secret(key=secret_mount.key, group=secret_mount.group, mount=DEFAULT_SECRET_DIR))
|
|
313
|
+
elif isinstance(secret_mount, str):
|
|
314
|
+
secrets.append(Secret(key=secret_mount, mount=DEFAULT_SECRET_DIR))
|
|
315
|
+
else:
|
|
316
|
+
raise ValueError(f"Unsupported secret_mount type: {type(secret_mount)}")
|
|
317
|
+
|
|
318
|
+
return secrets
|
|
@@ -142,6 +142,7 @@ async def load_and_run_task(
|
|
|
142
142
|
code_bundle: CodeBundle | None = None,
|
|
143
143
|
input_path: str | None = None,
|
|
144
144
|
image_cache: ImageCache | None = None,
|
|
145
|
+
interactive_mode: bool = False,
|
|
145
146
|
):
|
|
146
147
|
"""
|
|
147
148
|
This method is invoked from the runtime/CLI and is used to run a task. This creates the context tree,
|
|
@@ -159,6 +160,7 @@ async def load_and_run_task(
|
|
|
159
160
|
:param code_bundle: The code bundle to use for the task.
|
|
160
161
|
:param input_path: The input path to use for the task.
|
|
161
162
|
:param image_cache: Mappings of Image identifiers to image URIs.
|
|
163
|
+
:param interactive_mode: Whether to run the task in interactive mode.
|
|
162
164
|
"""
|
|
163
165
|
task = await _download_and_load_task(code_bundle, resolver, resolver_args)
|
|
164
166
|
|
|
@@ -175,4 +177,5 @@ async def load_and_run_task(
|
|
|
175
177
|
code_bundle=code_bundle,
|
|
176
178
|
input_path=input_path,
|
|
177
179
|
image_cache=image_cache,
|
|
180
|
+
interactive_mode=interactive_mode,
|
|
178
181
|
)
|
flyte/_internal/runtime/reuse.py
CHANGED
|
@@ -54,10 +54,10 @@ def extract_unique_id_and_image(
|
|
|
54
54
|
components += f":{reuse_policy.replicas}"
|
|
55
55
|
if reuse_policy.ttl is not None:
|
|
56
56
|
components += f":{reuse_policy.ttl.total_seconds()}"
|
|
57
|
-
if reuse_policy.
|
|
57
|
+
if reuse_policy.get_scaledown_ttl() is not None:
|
|
58
|
+
components += f":{reuse_policy.get_scaledown_ttl()}"
|
|
59
|
+
if code_bundle is not None:
|
|
58
60
|
components += f":{code_bundle.computed_version}"
|
|
59
|
-
else:
|
|
60
|
-
components += f":{reuse_policy.reuse_salt}"
|
|
61
61
|
if task.security_context is not None:
|
|
62
62
|
security_ctx_str = task.security_context.SerializeToString(deterministic=True)
|
|
63
63
|
components += f":{security_ctx_str}"
|
|
@@ -102,6 +102,8 @@ def add_reusable(
|
|
|
102
102
|
env_name=name, code_bundle=code_bundle, task=task, reuse_policy=reuse_policy
|
|
103
103
|
)
|
|
104
104
|
|
|
105
|
+
scaledown_ttl = reuse_policy.get_scaledown_ttl()
|
|
106
|
+
|
|
105
107
|
task.custom = {
|
|
106
108
|
"name": name,
|
|
107
109
|
"version": version[:15], # Use only the first 15 characters for the version
|
|
@@ -110,8 +112,10 @@ def add_reusable(
|
|
|
110
112
|
"container_image": image_uri,
|
|
111
113
|
"backlog_length": None,
|
|
112
114
|
"parallelism": reuse_policy.concurrency,
|
|
115
|
+
"min_replica_count": reuse_policy.min_replicas,
|
|
113
116
|
"replica_count": reuse_policy.max_replicas,
|
|
114
117
|
"ttl_seconds": reuse_policy.ttl.total_seconds() if reuse_policy.ttl else None,
|
|
118
|
+
"scaledown_ttl_seconds": scaledown_ttl.total_seconds() if scaledown_ttl else None,
|
|
115
119
|
},
|
|
116
120
|
}
|
|
117
121
|
|
|
@@ -54,7 +54,7 @@ def translate_task_to_wire(
|
|
|
54
54
|
return task_definition_pb2.TaskSpec(
|
|
55
55
|
task_template=tt,
|
|
56
56
|
default_inputs=default_inputs,
|
|
57
|
-
short_name=task.
|
|
57
|
+
short_name=task.short_name[:_MAX_TASK_SHORT_NAME_LENGTH],
|
|
58
58
|
environment=env,
|
|
59
59
|
)
|
|
60
60
|
|
|
@@ -145,7 +145,6 @@ def get_proto_task(task: TaskTemplate, serialize_context: SerializationContext)
|
|
|
145
145
|
logger.debug(f"Detected pkl bundle for task {task.name}, using computed version as cache version")
|
|
146
146
|
cache_version = serialize_context.code_bundle.computed_version
|
|
147
147
|
else:
|
|
148
|
-
version_parameters = None
|
|
149
148
|
if isinstance(task, AsyncFunctionTaskTemplate):
|
|
150
149
|
version_parameters = VersionParameters(func=task.func, image=task.image)
|
|
151
150
|
else:
|
|
@@ -204,7 +203,9 @@ def _get_urun_container(
|
|
|
204
203
|
serialize_context: SerializationContext, task_template: TaskTemplate
|
|
205
204
|
) -> Optional[tasks_pb2.Container]:
|
|
206
205
|
env = (
|
|
207
|
-
[literals_pb2.KeyValuePair(key=k, value=v) for k, v in task_template.
|
|
206
|
+
[literals_pb2.KeyValuePair(key=k, value=v) for k, v in task_template.env_vars.items()]
|
|
207
|
+
if task_template.env_vars
|
|
208
|
+
else None
|
|
208
209
|
)
|
|
209
210
|
resources = get_proto_resources(task_template.resources)
|
|
210
211
|
# pr: under what conditions should this return None?
|
|
@@ -110,16 +110,18 @@ async def run_task(
|
|
|
110
110
|
async def convert_and_run(
|
|
111
111
|
*,
|
|
112
112
|
task: TaskTemplate,
|
|
113
|
-
inputs: Inputs,
|
|
114
113
|
action: ActionID,
|
|
115
114
|
controller: Controller,
|
|
116
115
|
raw_data_path: RawDataPath,
|
|
117
116
|
version: str,
|
|
118
117
|
output_path: str,
|
|
119
118
|
run_base_dir: str,
|
|
119
|
+
inputs: Inputs = Inputs.empty(),
|
|
120
|
+
input_path: str | None = None,
|
|
120
121
|
checkpoints: Checkpoints | None = None,
|
|
121
122
|
code_bundle: CodeBundle | None = None,
|
|
122
123
|
image_cache: ImageCache | None = None,
|
|
124
|
+
interactive_mode: bool = False,
|
|
123
125
|
) -> Tuple[Optional[Outputs], Optional[Error]]:
|
|
124
126
|
"""
|
|
125
127
|
This method is used to convert the inputs to native types, and run the task. It assumes you are running
|
|
@@ -130,6 +132,7 @@ async def convert_and_run(
|
|
|
130
132
|
action=action,
|
|
131
133
|
checkpoints=checkpoints,
|
|
132
134
|
code_bundle=code_bundle,
|
|
135
|
+
input_path=input_path,
|
|
133
136
|
output_path=output_path,
|
|
134
137
|
run_base_dir=run_base_dir,
|
|
135
138
|
version=version,
|
|
@@ -137,8 +140,10 @@ async def convert_and_run(
|
|
|
137
140
|
compiled_image_cache=image_cache,
|
|
138
141
|
report=flyte.report.Report(name=action.name),
|
|
139
142
|
mode="remote" if not ctx.data.task_context else ctx.data.task_context.mode,
|
|
143
|
+
interactive_mode=interactive_mode,
|
|
140
144
|
)
|
|
141
145
|
with ctx.replace_task_context(tctx):
|
|
146
|
+
inputs = await load_inputs(input_path) if input_path else inputs
|
|
142
147
|
inputs_kwargs = await convert_inputs_to_native(inputs, task.native_interface)
|
|
143
148
|
out, err = await run_task(tctx=tctx, controller=controller, task=task, inputs=inputs_kwargs)
|
|
144
149
|
if err is not None:
|
|
@@ -161,15 +166,15 @@ async def extract_download_run_upload(
|
|
|
161
166
|
code_bundle: CodeBundle | None = None,
|
|
162
167
|
input_path: str | None = None,
|
|
163
168
|
image_cache: ImageCache | None = None,
|
|
169
|
+
interactive_mode: bool = False,
|
|
164
170
|
):
|
|
165
171
|
"""
|
|
166
172
|
This method is invoked from the CLI (urun) and is used to run a task. This assumes that the context tree
|
|
167
173
|
has already been created, and the task has been loaded. It also handles the loading of the task.
|
|
168
174
|
"""
|
|
169
|
-
inputs = await load_inputs(input_path) if input_path else None
|
|
170
175
|
outputs, err = await convert_and_run(
|
|
171
176
|
task=task,
|
|
172
|
-
|
|
177
|
+
input_path=input_path,
|
|
173
178
|
action=action,
|
|
174
179
|
controller=controller,
|
|
175
180
|
raw_data_path=raw_data_path,
|
|
@@ -179,6 +184,7 @@ async def extract_download_run_upload(
|
|
|
179
184
|
checkpoints=checkpoints,
|
|
180
185
|
code_bundle=code_bundle,
|
|
181
186
|
image_cache=image_cache,
|
|
187
|
+
interactive_mode=interactive_mode,
|
|
182
188
|
)
|
|
183
189
|
if err is not None:
|
|
184
190
|
path = await upload_error(err.err, output_path)
|
flyte/_logging.py
CHANGED
|
@@ -4,7 +4,9 @@ import logging
|
|
|
4
4
|
import os
|
|
5
5
|
from typing import Optional
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
import flyte
|
|
8
|
+
|
|
9
|
+
from ._tools import ipython_check
|
|
8
10
|
|
|
9
11
|
DEFAULT_LOG_LEVEL = logging.WARNING
|
|
10
12
|
|
|
@@ -42,7 +44,8 @@ def get_rich_handler(log_level: int) -> Optional[logging.Handler]:
|
|
|
42
44
|
"""
|
|
43
45
|
Upgrades the global loggers to use Rich logging.
|
|
44
46
|
"""
|
|
45
|
-
|
|
47
|
+
ctx = flyte.ctx()
|
|
48
|
+
if ctx and ctx.is_in_cluster():
|
|
46
49
|
return None
|
|
47
50
|
if not ipython_check() and is_rich_logging_disabled():
|
|
48
51
|
return None
|