flyte 2.0.0b9__py3-none-any.whl → 2.0.0b14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of flyte might be problematic. Click here for more details.

Files changed (60) hide show
  1. flyte/__init__.py +55 -31
  2. flyte/_bin/debug.py +38 -0
  3. flyte/_bin/runtime.py +13 -0
  4. flyte/_code_bundle/_utils.py +2 -0
  5. flyte/_code_bundle/bundle.py +4 -4
  6. flyte/_context.py +1 -1
  7. flyte/_debug/__init__.py +0 -0
  8. flyte/_debug/constants.py +39 -0
  9. flyte/_debug/utils.py +17 -0
  10. flyte/_debug/vscode.py +300 -0
  11. flyte/_environment.py +5 -5
  12. flyte/_image.py +34 -19
  13. flyte/_initialize.py +15 -29
  14. flyte/_internal/controllers/remote/_action.py +2 -2
  15. flyte/_internal/controllers/remote/_controller.py +1 -1
  16. flyte/_internal/imagebuild/docker_builder.py +11 -15
  17. flyte/_internal/imagebuild/remote_builder.py +71 -22
  18. flyte/_internal/runtime/entrypoints.py +3 -0
  19. flyte/_internal/runtime/reuse.py +7 -3
  20. flyte/_internal/runtime/task_serde.py +4 -3
  21. flyte/_internal/runtime/taskrunner.py +9 -3
  22. flyte/_logging.py +5 -2
  23. flyte/_protos/common/identifier_pb2.py +25 -19
  24. flyte/_protos/common/identifier_pb2.pyi +10 -0
  25. flyte/_protos/imagebuilder/definition_pb2.py +32 -31
  26. flyte/_protos/imagebuilder/definition_pb2.pyi +25 -12
  27. flyte/_protos/workflow/queue_service_pb2.py +24 -24
  28. flyte/_protos/workflow/queue_service_pb2.pyi +6 -6
  29. flyte/_protos/workflow/run_definition_pb2.py +48 -48
  30. flyte/_protos/workflow/run_definition_pb2.pyi +20 -10
  31. flyte/_reusable_environment.py +41 -19
  32. flyte/_run.py +9 -9
  33. flyte/_secret.py +9 -5
  34. flyte/_task.py +16 -11
  35. flyte/_task_environment.py +11 -13
  36. flyte/_tools.py +0 -13
  37. flyte/_version.py +16 -3
  38. flyte/cli/_build.py +2 -3
  39. flyte/cli/_common.py +16 -5
  40. flyte/cli/_gen.py +10 -1
  41. flyte/cli/_get.py +16 -14
  42. flyte/cli/_run.py +258 -25
  43. flyte/models.py +9 -0
  44. flyte/remote/_client/auth/_authenticators/base.py +8 -2
  45. flyte/remote/_client/auth/_authenticators/device_code.py +1 -1
  46. flyte/remote/_client/auth/_authenticators/pkce.py +1 -1
  47. flyte/remote/_client/auth/_channel.py +0 -6
  48. flyte/remote/_client/auth/_client_config.py +4 -2
  49. flyte/remote/_client/controlplane.py +14 -0
  50. flyte/remote/_task.py +18 -4
  51. flyte/storage/_storage.py +83 -7
  52. flyte/types/_type_engine.py +3 -33
  53. flyte-2.0.0b14.data/scripts/debug.py +38 -0
  54. {flyte-2.0.0b9.data → flyte-2.0.0b14.data}/scripts/runtime.py +13 -0
  55. {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/METADATA +2 -2
  56. {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/RECORD +60 -54
  57. {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/WHEEL +0 -0
  58. {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/entry_points.txt +0 -0
  59. {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/licenses/LICENSE +0 -0
  60. {flyte-2.0.0b9.dist-info → flyte-2.0.0b14.dist-info}/top_level.txt +0 -0
flyte/_image.py CHANGED
@@ -252,9 +252,15 @@ class AptPackages(Layer):
252
252
  @dataclass(frozen=True, repr=True)
253
253
  class Commands(Layer):
254
254
  commands: Tuple[str, ...]
255
+ secret_mounts: Optional[Tuple[str | Secret, ...]] = None
255
256
 
256
257
  def update_hash(self, hasher: hashlib._Hash):
257
- hasher.update("".join(self.commands).encode("utf-8"))
258
+ hash_input = "".join(self.commands)
259
+
260
+ if self.secret_mounts:
261
+ for secret_mount in self.secret_mounts:
262
+ hash_input += str(secret_mount)
263
+ hasher.update(hash_input.encode("utf-8"))
258
264
 
259
265
 
260
266
  @rich.repr.auto
@@ -279,7 +285,7 @@ class DockerIgnore(Layer):
279
285
  @dataclass(frozen=True, repr=True)
280
286
  class CopyConfig(Layer):
281
287
  path_type: CopyConfigType = field(metadata={"identifier": True})
282
- src: Path = field(metadata={"identifier": True})
288
+ src: Path = field(metadata={"identifier": False})
283
289
  dst: str
284
290
  src_name: str = field(init=False)
285
291
 
@@ -451,11 +457,7 @@ class Image:
451
457
  # this default image definition may need to be updated once there is a released pypi version
452
458
  from flyte._version import __version__
453
459
 
454
- dev_mode = (
455
- (cls._is_editable_install() or (__version__ and "dev" in __version__))
456
- and not flyte_version
457
- and install_flyte
458
- )
460
+ dev_mode = (__version__ and "dev" in __version__) and not flyte_version and install_flyte
459
461
  if install_flyte is False:
460
462
  preset_tag = f"py{python_version[0]}.{python_version[1]}"
461
463
  else:
@@ -507,13 +509,6 @@ class Image:
507
509
 
508
510
  return image
509
511
 
510
- @staticmethod
511
- def _is_editable_install():
512
- """Internal hacky function to see if the current install is editable or not."""
513
- curr = Path(__file__)
514
- pyproject = curr.parent.parent.parent / "pyproject.toml"
515
- return pyproject.exists()
516
-
517
512
  @classmethod
518
513
  def from_debian_base(
519
514
  cls,
@@ -793,9 +788,24 @@ class Image:
793
788
 
794
789
  Example:
795
790
  ```python
796
- @flyte.task(image=(flyte.Image
797
- .ubuntu_python()
798
- .with_pip_packages("requests", "numpy")))
791
+ @flyte.task(image=(flyte.Image.from_debian_base().with_pip_packages("requests", "numpy")))
792
+ def my_task(x: int) -> int:
793
+ import numpy as np
794
+ return np.sum([x, 1])
795
+ ```
796
+
797
+ To mount secrets during the build process to download private packages, you can use the `secret_mounts`.
798
+ In the below example, "GITHUB_PAT" will be mounted as env var "GITHUB_PAT",
799
+ and "apt-secret" will be mounted at /etc/apt/apt-secret.
800
+ Example:
801
+ ```python
802
+ private_package = "git+https://$GITHUB_PAT@github.com/flyteorg/flytex.git@2e20a2acebfc3877d84af643fdd768edea41d533"
803
+ @flyte.task(
804
+ image=(
805
+ flyte.Image.from_debian_base()
806
+ .with_pip_packages("private_package", secret_mounts=[Secret(key="GITHUB_PAT")])
807
+ .with_apt_packages("git", secret_mounts=[Secret(key="apt-secret", mount="/etc/apt/apt-secret")])
808
+ )
799
809
  def my_task(x: int) -> int:
800
810
  import numpy as np
801
811
  return np.sum([x, 1])
@@ -920,16 +930,21 @@ class Image:
920
930
  )
921
931
  return new_image
922
932
 
923
- def with_commands(self, commands: List[str]) -> Image:
933
+ def with_commands(self, commands: List[str], secret_mounts: Optional[SecretRequest] = None) -> Image:
924
934
  """
925
935
  Use this method to create a new image with the specified commands layered on top of the current image
926
936
  Be sure not to use RUN in your command.
927
937
 
928
938
  :param commands: list of commands to run
939
+ :param secret_mounts: list of secret mounts to use for the build process.
929
940
  :return: Image
930
941
  """
931
942
  new_commands: Tuple = _ensure_tuple(commands)
932
- new_image = self.clone(addl_layer=Commands(commands=new_commands))
943
+ new_image = self.clone(
944
+ addl_layer=Commands(
945
+ commands=new_commands, secret_mounts=_ensure_tuple(secret_mounts) if secret_mounts else None
946
+ )
947
+ )
933
948
  return new_image
934
949
 
935
950
  def with_local_v2(self) -> Image:
flyte/_initialize.py CHANGED
@@ -11,7 +11,6 @@ from flyte.errors import InitializationError
11
11
  from flyte.syncify import syncify
12
12
 
13
13
  from ._logging import initialize_logger, logger
14
- from ._tools import ipython_check
15
14
 
16
15
  if TYPE_CHECKING:
17
16
  from flyte._internal.imagebuild import ImageBuildEngine
@@ -173,6 +172,7 @@ async def init(
173
172
 
174
173
  :return: None
175
174
  """
175
+ from flyte._tools import ipython_check
176
176
  from flyte._utils import get_cwd_editable_install, org_from_endpoint, sanitize_endpoint
177
177
 
178
178
  interactive_mode = ipython_check()
@@ -205,6 +205,14 @@ async def init(
205
205
  http_proxy_url=http_proxy_url,
206
206
  )
207
207
 
208
+ if not root_dir:
209
+ editable_root = get_cwd_editable_install()
210
+ if editable_root:
211
+ logger.info(f"Using editable install as root directory: {editable_root}")
212
+ root_dir = editable_root
213
+ else:
214
+ logger.info("No editable install found, using current working directory as root directory.")
215
+ root_dir = Path.cwd()
208
216
  root_dir = root_dir or get_cwd_editable_install() or Path.cwd()
209
217
  _init_config = _InitConfig(
210
218
  root_dir=root_dir,
@@ -242,17 +250,19 @@ async def init_from_config(
242
250
  cfg: config.Config
243
251
  if path_or_config is None or isinstance(path_or_config, str):
244
252
  # If a string is passed, treat it as a path to the config file
245
- if path_or_config:
253
+ if root_dir and path_or_config:
254
+ cfg = config.auto(str(root_dir / path_or_config))
255
+ elif path_or_config:
246
256
  if not Path(path_or_config).exists():
247
257
  raise InitializationError(
248
258
  "ConfigFileNotFoundError",
249
259
  "user",
250
260
  f"Configuration file '{path_or_config}' does not exist., current working directory is {Path.cwd()}",
251
261
  )
252
- if root_dir and path_or_config:
253
- cfg = config.auto(str(root_dir / path_or_config))
254
- else:
255
262
  cfg = config.auto(path_or_config)
263
+ else:
264
+ # If no path is provided, use the default config file
265
+ cfg = config.auto()
256
266
  else:
257
267
  # If a Config object is passed, use it directly
258
268
  cfg = path_or_config
@@ -374,30 +384,6 @@ def ensure_client():
374
384
  )
375
385
 
376
386
 
377
- def requires_client(func: T) -> T:
378
- """
379
- Decorator that checks if the client has been initialized before executing the function.
380
- Raises InitializationError if the client is not initialized.
381
-
382
- :param func: Function to decorate
383
- :return: Decorated function that checks for initialization
384
- """
385
-
386
- @functools.wraps(func)
387
- async def wrapper(*args, **kwargs) -> T:
388
- init_config = _get_init_config()
389
- if init_config is None or init_config.client is None:
390
- raise InitializationError(
391
- "ClientNotInitializedError",
392
- "user",
393
- f"Function '{func.__name__}' requires client to be initialized. "
394
- f"Call flyte.init() with a valid endpoint or api-key before using this function.",
395
- )
396
- return func(*args, **kwargs)
397
-
398
- return typing.cast(T, wrapper)
399
-
400
-
401
387
  def requires_storage(func: T) -> T:
402
388
  """
403
389
  Decorator that checks if the storage has been initialized before executing the function.
@@ -183,7 +183,7 @@ class Action:
183
183
  et.FromSeconds(int(end_time))
184
184
  et.nanos = int((end_time % 1) * 1e9)
185
185
 
186
- spec = (
186
+ _ = (
187
187
  task_definition_pb2.TaskSpec(task_template=tasks_pb2.TaskTemplate(interface=typed_interface))
188
188
  if typed_interface
189
189
  else None
@@ -208,6 +208,6 @@ class Action:
208
208
  output_uri=outputs_uri,
209
209
  report_uri=report_uri,
210
210
  ),
211
- spec=spec,
211
+ # spec=spec,
212
212
  ),
213
213
  )
@@ -167,7 +167,7 @@ class RemoteController(Controller):
167
167
  # It is not allowed to change the code bundle (for regular code bundles) in the middle of a run.
168
168
  code_bundle = tctx.code_bundle
169
169
 
170
- if code_bundle and code_bundle.pkl:
170
+ if tctx.interactive_mode or (code_bundle and code_bundle.pkl):
171
171
  logger.debug(f"Building new pkl bundle for task {_task.name}")
172
172
  code_bundle = await build_pkl_bundle(
173
173
  _task,
@@ -302,11 +302,9 @@ def _get_secret_commands(layers: typing.Tuple[Layer, ...]) -> typing.List[str]:
302
302
  commands = []
303
303
 
304
304
  def _get_secret_command(secret: str | Secret) -> typing.List[str]:
305
- secret_id = hash(secret)
306
305
  if isinstance(secret, str):
307
- if not os.path.exists(secret):
308
- raise FileNotFoundError(f"Secret file '{secret}' not found")
309
- return ["--secret", f"id={secret_id},src={secret}"]
306
+ secret = Secret(key=secret)
307
+ secret_id = hash(secret)
310
308
  secret_env_key = "_".join([k.upper() for k in filter(None, (secret.group, secret.key))])
311
309
  secret_env = os.getenv(secret_env_key)
312
310
  if secret_env:
@@ -329,18 +327,16 @@ def _get_secret_mounts_layer(secrets: typing.Tuple[str | Secret, ...] | None) ->
329
327
  if secrets is None:
330
328
  return ""
331
329
  secret_mounts_layer = ""
332
- for secret in secrets:
330
+ for s in secrets:
331
+ secret = Secret(key=s) if isinstance(s, str) else s
333
332
  secret_id = hash(secret)
334
- if isinstance(secret, str):
335
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},target=/run/secrets/{os.path.basename(secret)}"
336
- elif isinstance(secret, Secret):
337
- if secret.mount:
338
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},target={secret.mount}"
339
- elif secret.as_env_var:
340
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret.as_env_var}"
341
- else:
342
- secret_file_name = "_".join(list(filter(None, (secret.group, secret.key))))
343
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},src=/run/secrets/{secret_file_name}"
333
+ if secret.mount:
334
+ secret_mounts_layer += f"--mount=type=secret,id={secret_id},target={secret.mount}"
335
+ elif secret.as_env_var:
336
+ secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret.as_env_var}"
337
+ else:
338
+ secret_default_env_key = "_".join(list(filter(None, (secret.group, secret.key))))
339
+ secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret_default_env_key}"
344
340
 
345
341
  return secret_mounts_layer
346
342
 
@@ -19,21 +19,26 @@ from flyte._image import (
19
19
  CopyConfig,
20
20
  DockerIgnore,
21
21
  Env,
22
+ PipOption,
22
23
  PipPackages,
23
24
  PythonWheels,
24
25
  Requirements,
25
26
  UVProject,
26
27
  UVScript,
28
+ WorkDir,
27
29
  )
28
30
  from flyte._internal.imagebuild.image_builder import ImageBuilder, ImageChecker
29
31
  from flyte._internal.imagebuild.utils import copy_files_to_context
32
+ from flyte._internal.runtime.task_serde import get_security_context
30
33
  from flyte._logging import logger
34
+ from flyte._secret import Secret
31
35
  from flyte.remote import ActionOutputs, Run
32
36
 
33
37
  if TYPE_CHECKING:
34
38
  from flyte._protos.imagebuilder import definition_pb2 as image_definition_pb2
35
39
 
36
40
  IMAGE_TASK_NAME = "build-image"
41
+ OPTIMIZE_TASK_NAME = "optimize-task"
37
42
  IMAGE_TASK_PROJECT = "system"
38
43
  IMAGE_TASK_DOMAIN = "production"
39
44
 
@@ -95,12 +100,12 @@ class RemoteImageBuilder(ImageBuilder):
95
100
  spec, context = await _validate_configuration(image)
96
101
 
97
102
  start = datetime.now(timezone.utc)
98
- entity = remote.Task.get(
103
+ entity = await remote.Task.get(
99
104
  name=IMAGE_TASK_NAME,
100
105
  project=IMAGE_TASK_PROJECT,
101
106
  domain=IMAGE_TASK_DOMAIN,
102
107
  auto_version="latest",
103
- )
108
+ ).override.aio(secrets=_get_build_secrets_from_image(image))
104
109
  run = cast(
105
110
  Run,
106
111
  await flyte.with_runcontext(project=IMAGE_TASK_PROJECT, domain=IMAGE_TASK_DOMAIN).run.aio(
@@ -117,6 +122,19 @@ class RemoteImageBuilder(ImageBuilder):
117
122
 
118
123
  if run_details.action_details.raw_phase == run_definition_pb2.PHASE_SUCCEEDED:
119
124
  logger.warning(click.style(f"✅ Build completed in {elapsed}!", bold=True, fg="green"))
125
+ try:
126
+ entity = remote.Task.get(
127
+ name=OPTIMIZE_TASK_NAME,
128
+ project=IMAGE_TASK_PROJECT,
129
+ domain=IMAGE_TASK_DOMAIN,
130
+ auto_version="latest",
131
+ )
132
+ await flyte.with_runcontext(project=IMAGE_TASK_PROJECT, domain=IMAGE_TASK_DOMAIN).run.aio(
133
+ entity, spec=spec, context=context, target_image=image_name
134
+ )
135
+ except Exception as e:
136
+ # Ignore the error if optimize is not enabled in the backend.
137
+ logger.warning(f"Failed to run optimize task with error: {e}")
120
138
  else:
121
139
  raise flyte.errors.ImageBuildError(f"❌ Build failed in {elapsed} at {click.style(run.url, fg='cyan')}")
122
140
 
@@ -164,9 +182,27 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
164
182
 
165
183
  layers = []
166
184
  for layer in image._layers:
185
+ secret_mounts = None
186
+ pip_options = None
187
+
188
+ if isinstance(layer, PipOption):
189
+ pip_options = image_definition_pb2.PipOptions(
190
+ index_url=layer.index_url,
191
+ extra_index_urls=layer.extra_index_urls,
192
+ pre=layer.pre,
193
+ extra_args=layer.extra_args,
194
+ )
195
+
196
+ if hasattr(layer, "secret_mounts"):
197
+ sc = get_security_context(layer.secret_mounts)
198
+ secret_mounts = sc.secrets if sc else None
199
+
167
200
  if isinstance(layer, AptPackages):
168
201
  apt_layer = image_definition_pb2.Layer(
169
- apt_packages=image_definition_pb2.AptPackages(packages=layer.packages)
202
+ apt_packages=image_definition_pb2.AptPackages(
203
+ packages=layer.packages,
204
+ secret_mounts=secret_mounts,
205
+ ),
170
206
  )
171
207
  layers.append(apt_layer)
172
208
  elif isinstance(layer, PythonWheels):
@@ -174,12 +210,8 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
174
210
  wheel_layer = image_definition_pb2.Layer(
175
211
  python_wheels=image_definition_pb2.PythonWheels(
176
212
  dir=str(dst_path.relative_to(context_path)),
177
- options=image_definition_pb2.PipOptions(
178
- index_url=layer.index_url,
179
- extra_index_urls=layer.extra_index_urls,
180
- pre=layer.pre,
181
- extra_args=layer.extra_args,
182
- ),
213
+ options=pip_options,
214
+ secret_mounts=secret_mounts,
183
215
  )
184
216
  )
185
217
  layers.append(wheel_layer)
@@ -189,12 +221,8 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
189
221
  requirements_layer = image_definition_pb2.Layer(
190
222
  requirements=image_definition_pb2.Requirements(
191
223
  file=str(dst_path.relative_to(context_path)),
192
- options=image_definition_pb2.PipOptions(
193
- index_url=layer.index_url,
194
- extra_index_urls=layer.extra_index_urls,
195
- pre=layer.pre,
196
- extra_args=layer.extra_args,
197
- ),
224
+ options=pip_options,
225
+ secret_mounts=secret_mounts,
198
226
  )
199
227
  )
200
228
  layers.append(requirements_layer)
@@ -211,12 +239,8 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
211
239
  pip_layer = image_definition_pb2.Layer(
212
240
  pip_packages=image_definition_pb2.PipPackages(
213
241
  packages=packages,
214
- options=image_definition_pb2.PipOptions(
215
- index_url=layer.index_url,
216
- extra_index_urls=layer.extra_index_urls,
217
- pre=layer.pre,
218
- extra_args=layer.extra_args,
219
- ),
242
+ options=pip_options,
243
+ secret_mounts=secret_mounts,
220
244
  )
221
245
  )
222
246
  layers.append(pip_layer)
@@ -235,7 +259,10 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
235
259
  layers.append(uv_layer)
236
260
  elif isinstance(layer, Commands):
237
261
  commands_layer = image_definition_pb2.Layer(
238
- commands=image_definition_pb2.Commands(cmd=list(layer.commands))
262
+ commands=image_definition_pb2.Commands(
263
+ cmd=list(layer.commands),
264
+ secret_mounts=secret_mounts,
265
+ )
239
266
  )
240
267
  layers.append(commands_layer)
241
268
  elif isinstance(layer, DockerIgnore):
@@ -257,6 +284,11 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
257
284
  )
258
285
  )
259
286
  layers.append(env_layer)
287
+ elif isinstance(layer, WorkDir):
288
+ workdir_layer = image_definition_pb2.Layer(
289
+ workdir=image_definition_pb2.WorkDir(workdir=layer.workdir),
290
+ )
291
+ layers.append(workdir_layer)
260
292
 
261
293
  return image_definition_pb2.ImageSpec(
262
294
  base_image=image.base_image,
@@ -267,3 +299,20 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
267
299
 
268
300
  def _get_fully_qualified_image_name(outputs: ActionOutputs) -> str:
269
301
  return outputs.pb2.literals[0].value.scalar.primitive.string_value
302
+
303
+
304
+ def _get_build_secrets_from_image(image: Image) -> Optional[typing.List[Secret]]:
305
+ secrets = []
306
+ DEFAULT_SECRET_DIR = Path("etc/flyte/secrets")
307
+ for layer in image._layers:
308
+ if isinstance(layer, (PipOption, Commands, AptPackages)) and layer.secret_mounts is not None:
309
+ for secret_mount in layer.secret_mounts:
310
+ # Mount all the image secrets to a default directory that will be passed to the BuildKit server.
311
+ if isinstance(secret_mount, Secret):
312
+ secrets.append(Secret(key=secret_mount.key, group=secret_mount.group, mount=DEFAULT_SECRET_DIR))
313
+ elif isinstance(secret_mount, str):
314
+ secrets.append(Secret(key=secret_mount, mount=DEFAULT_SECRET_DIR))
315
+ else:
316
+ raise ValueError(f"Unsupported secret_mount type: {type(secret_mount)}")
317
+
318
+ return secrets
@@ -142,6 +142,7 @@ async def load_and_run_task(
142
142
  code_bundle: CodeBundle | None = None,
143
143
  input_path: str | None = None,
144
144
  image_cache: ImageCache | None = None,
145
+ interactive_mode: bool = False,
145
146
  ):
146
147
  """
147
148
  This method is invoked from the runtime/CLI and is used to run a task. This creates the context tree,
@@ -159,6 +160,7 @@ async def load_and_run_task(
159
160
  :param code_bundle: The code bundle to use for the task.
160
161
  :param input_path: The input path to use for the task.
161
162
  :param image_cache: Mappings of Image identifiers to image URIs.
163
+ :param interactive_mode: Whether to run the task in interactive mode.
162
164
  """
163
165
  task = await _download_and_load_task(code_bundle, resolver, resolver_args)
164
166
 
@@ -175,4 +177,5 @@ async def load_and_run_task(
175
177
  code_bundle=code_bundle,
176
178
  input_path=input_path,
177
179
  image_cache=image_cache,
180
+ interactive_mode=interactive_mode,
178
181
  )
@@ -54,10 +54,10 @@ def extract_unique_id_and_image(
54
54
  components += f":{reuse_policy.replicas}"
55
55
  if reuse_policy.ttl is not None:
56
56
  components += f":{reuse_policy.ttl.total_seconds()}"
57
- if reuse_policy.reuse_salt is None and code_bundle is not None:
57
+ if reuse_policy.get_scaledown_ttl() is not None:
58
+ components += f":{reuse_policy.get_scaledown_ttl()}"
59
+ if code_bundle is not None:
58
60
  components += f":{code_bundle.computed_version}"
59
- else:
60
- components += f":{reuse_policy.reuse_salt}"
61
61
  if task.security_context is not None:
62
62
  security_ctx_str = task.security_context.SerializeToString(deterministic=True)
63
63
  components += f":{security_ctx_str}"
@@ -102,6 +102,8 @@ def add_reusable(
102
102
  env_name=name, code_bundle=code_bundle, task=task, reuse_policy=reuse_policy
103
103
  )
104
104
 
105
+ scaledown_ttl = reuse_policy.get_scaledown_ttl()
106
+
105
107
  task.custom = {
106
108
  "name": name,
107
109
  "version": version[:15], # Use only the first 15 characters for the version
@@ -110,8 +112,10 @@ def add_reusable(
110
112
  "container_image": image_uri,
111
113
  "backlog_length": None,
112
114
  "parallelism": reuse_policy.concurrency,
115
+ "min_replica_count": reuse_policy.min_replicas,
113
116
  "replica_count": reuse_policy.max_replicas,
114
117
  "ttl_seconds": reuse_policy.ttl.total_seconds() if reuse_policy.ttl else None,
118
+ "scaledown_ttl_seconds": scaledown_ttl.total_seconds() if scaledown_ttl else None,
115
119
  },
116
120
  }
117
121
 
@@ -54,7 +54,7 @@ def translate_task_to_wire(
54
54
  return task_definition_pb2.TaskSpec(
55
55
  task_template=tt,
56
56
  default_inputs=default_inputs,
57
- short_name=task.friendly_name[:_MAX_TASK_SHORT_NAME_LENGTH],
57
+ short_name=task.short_name[:_MAX_TASK_SHORT_NAME_LENGTH],
58
58
  environment=env,
59
59
  )
60
60
 
@@ -145,7 +145,6 @@ def get_proto_task(task: TaskTemplate, serialize_context: SerializationContext)
145
145
  logger.debug(f"Detected pkl bundle for task {task.name}, using computed version as cache version")
146
146
  cache_version = serialize_context.code_bundle.computed_version
147
147
  else:
148
- version_parameters = None
149
148
  if isinstance(task, AsyncFunctionTaskTemplate):
150
149
  version_parameters = VersionParameters(func=task.func, image=task.image)
151
150
  else:
@@ -204,7 +203,9 @@ def _get_urun_container(
204
203
  serialize_context: SerializationContext, task_template: TaskTemplate
205
204
  ) -> Optional[tasks_pb2.Container]:
206
205
  env = (
207
- [literals_pb2.KeyValuePair(key=k, value=v) for k, v in task_template.env.items()] if task_template.env else None
206
+ [literals_pb2.KeyValuePair(key=k, value=v) for k, v in task_template.env_vars.items()]
207
+ if task_template.env_vars
208
+ else None
208
209
  )
209
210
  resources = get_proto_resources(task_template.resources)
210
211
  # pr: under what conditions should this return None?
@@ -110,16 +110,18 @@ async def run_task(
110
110
  async def convert_and_run(
111
111
  *,
112
112
  task: TaskTemplate,
113
- inputs: Inputs,
114
113
  action: ActionID,
115
114
  controller: Controller,
116
115
  raw_data_path: RawDataPath,
117
116
  version: str,
118
117
  output_path: str,
119
118
  run_base_dir: str,
119
+ inputs: Inputs = Inputs.empty(),
120
+ input_path: str | None = None,
120
121
  checkpoints: Checkpoints | None = None,
121
122
  code_bundle: CodeBundle | None = None,
122
123
  image_cache: ImageCache | None = None,
124
+ interactive_mode: bool = False,
123
125
  ) -> Tuple[Optional[Outputs], Optional[Error]]:
124
126
  """
125
127
  This method is used to convert the inputs to native types, and run the task. It assumes you are running
@@ -130,6 +132,7 @@ async def convert_and_run(
130
132
  action=action,
131
133
  checkpoints=checkpoints,
132
134
  code_bundle=code_bundle,
135
+ input_path=input_path,
133
136
  output_path=output_path,
134
137
  run_base_dir=run_base_dir,
135
138
  version=version,
@@ -137,8 +140,10 @@ async def convert_and_run(
137
140
  compiled_image_cache=image_cache,
138
141
  report=flyte.report.Report(name=action.name),
139
142
  mode="remote" if not ctx.data.task_context else ctx.data.task_context.mode,
143
+ interactive_mode=interactive_mode,
140
144
  )
141
145
  with ctx.replace_task_context(tctx):
146
+ inputs = await load_inputs(input_path) if input_path else inputs
142
147
  inputs_kwargs = await convert_inputs_to_native(inputs, task.native_interface)
143
148
  out, err = await run_task(tctx=tctx, controller=controller, task=task, inputs=inputs_kwargs)
144
149
  if err is not None:
@@ -161,15 +166,15 @@ async def extract_download_run_upload(
161
166
  code_bundle: CodeBundle | None = None,
162
167
  input_path: str | None = None,
163
168
  image_cache: ImageCache | None = None,
169
+ interactive_mode: bool = False,
164
170
  ):
165
171
  """
166
172
  This method is invoked from the CLI (urun) and is used to run a task. This assumes that the context tree
167
173
  has already been created, and the task has been loaded. It also handles the loading of the task.
168
174
  """
169
- inputs = await load_inputs(input_path) if input_path else None
170
175
  outputs, err = await convert_and_run(
171
176
  task=task,
172
- inputs=inputs or Inputs.empty(),
177
+ input_path=input_path,
173
178
  action=action,
174
179
  controller=controller,
175
180
  raw_data_path=raw_data_path,
@@ -179,6 +184,7 @@ async def extract_download_run_upload(
179
184
  checkpoints=checkpoints,
180
185
  code_bundle=code_bundle,
181
186
  image_cache=image_cache,
187
+ interactive_mode=interactive_mode,
182
188
  )
183
189
  if err is not None:
184
190
  path = await upload_error(err.err, output_path)
flyte/_logging.py CHANGED
@@ -4,7 +4,9 @@ import logging
4
4
  import os
5
5
  from typing import Optional
6
6
 
7
- from ._tools import ipython_check, is_in_cluster
7
+ import flyte
8
+
9
+ from ._tools import ipython_check
8
10
 
9
11
  DEFAULT_LOG_LEVEL = logging.WARNING
10
12
 
@@ -42,7 +44,8 @@ def get_rich_handler(log_level: int) -> Optional[logging.Handler]:
42
44
  """
43
45
  Upgrades the global loggers to use Rich logging.
44
46
  """
45
- if is_in_cluster():
47
+ ctx = flyte.ctx()
48
+ if ctx and ctx.is_in_cluster():
46
49
  return None
47
50
  if not ipython_check() and is_rich_logging_disabled():
48
51
  return None