flyte 2.0.0b21__py3-none-any.whl → 2.0.0b23__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of flyte might be problematic. Click here for more details.

Files changed (92) hide show
  1. flyte/__init__.py +5 -0
  2. flyte/_bin/runtime.py +36 -6
  3. flyte/_cache/cache.py +4 -2
  4. flyte/_cache/local_cache.py +215 -0
  5. flyte/_code_bundle/bundle.py +1 -0
  6. flyte/_debug/constants.py +0 -1
  7. flyte/_debug/vscode.py +6 -1
  8. flyte/_deploy.py +204 -55
  9. flyte/_environment.py +5 -0
  10. flyte/_excepthook.py +1 -1
  11. flyte/_image.py +101 -68
  12. flyte/_initialize.py +30 -1
  13. flyte/_interface.py +3 -1
  14. flyte/_internal/controllers/_local_controller.py +64 -24
  15. flyte/_internal/controllers/remote/_action.py +4 -1
  16. flyte/_internal/controllers/remote/_controller.py +5 -2
  17. flyte/_internal/controllers/remote/_core.py +6 -3
  18. flyte/_internal/controllers/remote/_informer.py +1 -1
  19. flyte/_internal/imagebuild/docker_builder.py +95 -28
  20. flyte/_internal/imagebuild/image_builder.py +0 -5
  21. flyte/_internal/imagebuild/remote_builder.py +6 -1
  22. flyte/_internal/runtime/io.py +13 -1
  23. flyte/_internal/runtime/rusty.py +17 -2
  24. flyte/_internal/runtime/task_serde.py +15 -11
  25. flyte/_internal/runtime/taskrunner.py +1 -1
  26. flyte/_internal/runtime/trigger_serde.py +153 -0
  27. flyte/_keyring/file.py +2 -2
  28. flyte/_logging.py +1 -1
  29. flyte/_protos/common/identifier_pb2.py +19 -1
  30. flyte/_protos/common/identifier_pb2.pyi +22 -0
  31. flyte/_protos/workflow/common_pb2.py +14 -3
  32. flyte/_protos/workflow/common_pb2.pyi +49 -0
  33. flyte/_protos/workflow/queue_service_pb2.py +41 -35
  34. flyte/_protos/workflow/queue_service_pb2.pyi +26 -12
  35. flyte/_protos/workflow/queue_service_pb2_grpc.py +34 -0
  36. flyte/_protos/workflow/run_definition_pb2.py +38 -38
  37. flyte/_protos/workflow/run_definition_pb2.pyi +4 -2
  38. flyte/_protos/workflow/run_service_pb2.py +60 -50
  39. flyte/_protos/workflow/run_service_pb2.pyi +24 -6
  40. flyte/_protos/workflow/run_service_pb2_grpc.py +34 -0
  41. flyte/_protos/workflow/task_definition_pb2.py +15 -11
  42. flyte/_protos/workflow/task_definition_pb2.pyi +19 -2
  43. flyte/_protos/workflow/task_service_pb2.py +18 -17
  44. flyte/_protos/workflow/task_service_pb2.pyi +5 -2
  45. flyte/_protos/workflow/trigger_definition_pb2.py +66 -0
  46. flyte/_protos/workflow/trigger_definition_pb2.pyi +117 -0
  47. flyte/_protos/workflow/trigger_definition_pb2_grpc.py +4 -0
  48. flyte/_protos/workflow/trigger_service_pb2.py +96 -0
  49. flyte/_protos/workflow/trigger_service_pb2.pyi +110 -0
  50. flyte/_protos/workflow/trigger_service_pb2_grpc.py +281 -0
  51. flyte/_run.py +42 -15
  52. flyte/_task.py +35 -4
  53. flyte/_task_environment.py +61 -16
  54. flyte/_trigger.py +382 -0
  55. flyte/_version.py +3 -3
  56. flyte/cli/_abort.py +3 -3
  57. flyte/cli/_build.py +1 -3
  58. flyte/cli/_common.py +17 -4
  59. flyte/cli/_create.py +74 -0
  60. flyte/cli/_delete.py +23 -1
  61. flyte/cli/_deploy.py +16 -10
  62. flyte/cli/_get.py +75 -34
  63. flyte/cli/_params.py +4 -2
  64. flyte/cli/_run.py +25 -6
  65. flyte/cli/_update.py +36 -0
  66. flyte/cli/_user.py +17 -0
  67. flyte/cli/main.py +9 -1
  68. flyte/errors.py +9 -0
  69. flyte/io/_dir.py +513 -115
  70. flyte/io/_file.py +495 -135
  71. flyte/models.py +32 -0
  72. flyte/remote/__init__.py +6 -1
  73. flyte/remote/_action.py +9 -8
  74. flyte/remote/_client/_protocols.py +36 -2
  75. flyte/remote/_client/controlplane.py +19 -3
  76. flyte/remote/_run.py +42 -2
  77. flyte/remote/_task.py +14 -1
  78. flyte/remote/_trigger.py +308 -0
  79. flyte/remote/_user.py +33 -0
  80. flyte/storage/__init__.py +6 -1
  81. flyte/storage/_storage.py +119 -101
  82. flyte/types/_pickle.py +34 -7
  83. flyte/types/_type_engine.py +6 -0
  84. {flyte-2.0.0b21.data → flyte-2.0.0b23.data}/scripts/runtime.py +36 -6
  85. {flyte-2.0.0b21.dist-info → flyte-2.0.0b23.dist-info}/METADATA +3 -1
  86. {flyte-2.0.0b21.dist-info → flyte-2.0.0b23.dist-info}/RECORD +91 -79
  87. flyte/_protos/secret/secret_pb2_grpc_grpc.py +0 -198
  88. {flyte-2.0.0b21.data → flyte-2.0.0b23.data}/scripts/debug.py +0 -0
  89. {flyte-2.0.0b21.dist-info → flyte-2.0.0b23.dist-info}/WHEEL +0 -0
  90. {flyte-2.0.0b21.dist-info → flyte-2.0.0b23.dist-info}/entry_points.txt +0 -0
  91. {flyte-2.0.0b21.dist-info → flyte-2.0.0b23.dist-info}/licenses/LICENSE +0 -0
  92. {flyte-2.0.0b21.dist-info → flyte-2.0.0b23.dist-info}/top_level.txt +0 -0
flyte/cli/_get.py CHANGED
@@ -1,10 +1,11 @@
1
1
  import asyncio
2
- from typing import Tuple, Union
2
+ from typing import Tuple, Union, get_args
3
3
 
4
4
  import rich_click as click
5
- from rich.console import Console
6
5
  from rich.pretty import pretty_repr
7
6
 
7
+ import flyte.remote as remote
8
+
8
9
  from . import _common as common
9
10
 
10
11
 
@@ -38,20 +39,24 @@ def project(cfg: common.CLIConfig, name: str | None = None):
38
39
  """
39
40
  Get a list of all projects, or details of a specific project by name.
40
41
  """
41
- from flyte.remote import Project
42
-
43
42
  cfg.init()
44
43
 
45
- console = Console()
44
+ console = common.get_console()
46
45
  if name:
47
- console.print(pretty_repr(Project.get(name)))
46
+ console.print(pretty_repr(remote.Project.get(name)))
48
47
  else:
49
- console.print(common.format("Projects", Project.listall(), cfg.output_format))
48
+ console.print(common.format("Projects", remote.Project.listall(), cfg.output_format))
50
49
 
51
50
 
52
51
  @get.command(cls=common.CommandBase)
53
52
  @click.argument("name", type=str, required=False)
54
53
  @click.option("--limit", type=int, default=100, help="Limit the number of runs to fetch when listing.")
54
+ @click.option(
55
+ "--in-phase", # multiple=True, TODO support multiple phases once values in works
56
+ type=click.Choice(get_args(remote.Phase), case_sensitive=False),
57
+ help="Filter runs by their status.",
58
+ )
59
+ @click.option("--only-mine", is_flag=True, default=False, help="Show only runs created by the current user (you).")
55
60
  @click.pass_obj
56
61
  def run(
57
62
  cfg: common.CLIConfig,
@@ -59,6 +64,8 @@ def run(
59
64
  project: str | None = None,
60
65
  domain: str | None = None,
61
66
  limit: int = 100,
67
+ in_phase: str | Tuple[str, ...] | None = None,
68
+ only_mine: bool = False,
62
69
  ):
63
70
  """
64
71
  Get a list of all runs, or details of a specific run by name.
@@ -67,16 +74,29 @@ def run(
67
74
 
68
75
  If you want to see the actions for a run, use `get action <run_name>`.
69
76
  """
70
- from flyte.remote import Run, RunDetails
71
77
 
72
78
  cfg.init(project=project, domain=domain)
73
79
 
74
- console = Console()
80
+ console = common.get_console()
75
81
  if name:
76
- details = RunDetails.get(name=name)
82
+ details = remote.RunDetails.get(name=name)
77
83
  console.print(common.format(f"Run {name}", [details], "json"))
78
84
  else:
79
- console.print(common.format("Runs", Run.listall(limit=limit), cfg.output_format))
85
+ if in_phase and isinstance(in_phase, str):
86
+ in_phase = (in_phase,)
87
+
88
+ subject = None
89
+ if only_mine:
90
+ usr = remote.User.get()
91
+ subject = usr.subject()
92
+
93
+ console.print(
94
+ common.format(
95
+ "Runs",
96
+ remote.Run.listall(limit=limit, in_phase=in_phase, created_by_subject=subject),
97
+ cfg.output_format,
98
+ )
99
+ )
80
100
 
81
101
 
82
102
  @get.command(cls=common.CommandBase)
@@ -97,22 +117,22 @@ def task(
97
117
 
98
118
  Currently, both `name` and `version` are required to get a specific task.
99
119
  """
100
- from flyte.remote import Task
101
-
102
120
  cfg.init(project=project, domain=domain)
103
121
 
104
- console = Console()
122
+ console = common.get_console()
105
123
  if name:
106
124
  if version:
107
- v = Task.get(name=name, version=version)
125
+ v = remote.Task.get(name=name, version=version)
108
126
  if v is None:
109
127
  raise click.BadParameter(f"Task {name} not found.")
110
128
  t = v.fetch()
111
129
  console.print(common.format(f"Task {name}", [t], "json"))
112
130
  else:
113
- console.print(common.format("Tasks", Task.listall(by_task_name=name, limit=limit), cfg.output_format))
131
+ console.print(
132
+ common.format("Tasks", remote.Task.listall(by_task_name=name, limit=limit), cfg.output_format)
133
+ )
114
134
  else:
115
- console.print(common.format("Tasks", Task.listall(limit=limit), cfg.output_format))
135
+ console.print(common.format("Tasks", remote.Task.listall(limit=limit), cfg.output_format))
116
136
 
117
137
 
118
138
  @get.command(cls=common.CommandBase)
@@ -129,11 +149,9 @@ def action(
129
149
  """
130
150
  Get all actions for a run or details for a specific action.
131
151
  """
132
- import flyte.remote as remote
133
-
134
152
  cfg.init(project=project, domain=domain)
135
153
 
136
- console = Console()
154
+ console = common.get_console()
137
155
  if action_name:
138
156
  console.print(
139
157
  common.format(
@@ -196,8 +214,6 @@ def logs(
196
214
  $ flyte get logs my_run my_action --pretty --lines 50
197
215
  ```
198
216
  """
199
- import flyte.remote as remote
200
-
201
217
  cfg.init(project=project, domain=domain)
202
218
 
203
219
  async def _run_log_view(_obj):
@@ -230,11 +246,9 @@ def secret(
230
246
  """
231
247
  Get a list of all secrets, or details of a specific secret by name.
232
248
  """
233
- import flyte.remote as remote
234
-
235
249
  cfg.init(project=project, domain=domain)
236
250
 
237
- console = Console()
251
+ console = common.get_console()
238
252
  if name:
239
253
  console.print(common.format("Secret", [remote.Secret.get(name)], "json"))
240
254
  else:
@@ -275,26 +289,23 @@ def io(
275
289
  if inputs_only and outputs_only:
276
290
  raise click.BadParameter("Cannot use both --inputs-only and --outputs-only")
277
291
 
278
- import flyte.remote as remote
279
- from flyte.remote import ActionDetails, ActionInputs, ActionOutputs
280
-
281
292
  cfg.init(project=project, domain=domain)
282
- console = Console()
293
+ console = common.get_console()
283
294
  if action_name:
284
- obj = ActionDetails.get(run_name=run_name, name=action_name)
295
+ obj = remote.ActionDetails.get(run_name=run_name, name=action_name)
285
296
  else:
286
297
  obj = remote.RunDetails.get(run_name)
287
298
 
288
299
  async def _get_io(
289
- details: Union[remote.RunDetails, ActionDetails],
290
- ) -> Tuple[ActionInputs | None, ActionOutputs | None | str]:
300
+ details: Union[remote.RunDetails, remote.ActionDetails],
301
+ ) -> Tuple[remote.ActionInputs | None, remote.ActionOutputs | None | str]:
291
302
  if inputs_only or outputs_only:
292
303
  if inputs_only:
293
304
  return await details.inputs(), None
294
305
  elif outputs_only:
295
306
  return None, await details.outputs()
296
307
  inputs = await details.inputs()
297
- outputs: ActionOutputs | None | str = None
308
+ outputs: remote.ActionOutputs | None | str = None
298
309
  try:
299
310
  outputs = await details.outputs()
300
311
  except Exception:
@@ -321,5 +332,35 @@ def config(cfg: common.CLIConfig):
321
332
 
322
333
  The configuration will include the endpoint, organization, and other settings that are used by the CLI.
323
334
  """
324
- console = Console()
335
+ console = common.get_console()
325
336
  console.print(cfg)
337
+
338
+
339
+ @get.command(cls=common.CommandBase)
340
+ @click.argument("task_name", type=str, required=False)
341
+ @click.argument("name", type=str, required=False)
342
+ @click.option("--limit", type=int, default=100, help="Limit the number of triggers to fetch.")
343
+ @click.pass_obj
344
+ def trigger(
345
+ cfg: common.CLIConfig,
346
+ task_name: str | None = None,
347
+ name: str | None = None,
348
+ limit: int = 100,
349
+ project: str | None = None,
350
+ domain: str | None = None,
351
+ ):
352
+ """
353
+ Get a list of all triggers, or details of a specific trigger by name.
354
+ """
355
+ if name and not task_name:
356
+ raise click.BadParameter("If you provide a trigger name, you must also provide the task name.")
357
+
358
+ from flyte.remote import Trigger
359
+
360
+ cfg.init(project=project, domain=domain)
361
+
362
+ console = common.get_console()
363
+ if name:
364
+ console.print(pretty_repr(Trigger.get(name=name, task_name=task_name)))
365
+ else:
366
+ console.print(common.format("Triggers", Trigger.listall(task_name=task_name, limit=limit), cfg.output_format))
flyte/cli/_params.py CHANGED
@@ -531,17 +531,19 @@ def to_click_option(
531
531
  if literal_var.type.metadata:
532
532
  description_extra = f": {MessageToDict(literal_var.type.metadata)}"
533
533
 
534
- # If a query has been specified, the input is never strictly required at this layer
535
534
  required = False if default_val is not None else True
536
535
  is_flag: typing.Optional[bool] = None
536
+ param_decls = [f"--{input_name}"]
537
537
  if literal_converter.is_bool():
538
538
  required = False
539
539
  is_flag = True
540
+ if default_val is True:
541
+ param_decls = [f"--{input_name}/--no-{input_name}"]
540
542
  if literal_converter.is_optional():
541
543
  required = False
542
544
 
543
545
  return click.Option(
544
- param_decls=[f"--{input_name}"],
546
+ param_decls=param_decls,
545
547
  type=literal_converter.click_type,
546
548
  is_flag=is_flag,
547
549
  default=default_val,
flyte/cli/_run.py CHANGED
@@ -9,7 +9,6 @@ from types import ModuleType
9
9
  from typing import Any, Dict, List, cast
10
10
 
11
11
  import rich_click as click
12
- from rich.console import Console
13
12
  from typing_extensions import get_args
14
13
 
15
14
  from .._code_bundle._utils import CopyFiles
@@ -23,14 +22,14 @@ RUN_REMOTE_CMD = "deployed-task"
23
22
 
24
23
 
25
24
  @lru_cache()
26
- def _initialize_config(ctx: click.Context, project: str, domain: str):
25
+ def _initialize_config(ctx: click.Context, project: str, domain: str, root_dir: str | None = None):
27
26
  obj: CLIConfig | None = ctx.obj
28
27
  if obj is None:
29
28
  import flyte.config
30
29
 
31
30
  obj = CLIConfig(flyte.config.auto(), ctx)
32
31
 
33
- obj.init(project, domain)
32
+ obj.init(project, domain, root_dir)
34
33
  return obj
35
34
 
36
35
 
@@ -77,6 +76,16 @@ class RunArguments:
77
76
  )
78
77
  },
79
78
  )
79
+ root_dir: str | None = field(
80
+ default=None,
81
+ metadata={
82
+ "click.option": click.Option(
83
+ ["--root-dir"],
84
+ type=str,
85
+ help="Override the root source directory, helpful when working with monorepos.",
86
+ )
87
+ },
88
+ )
80
89
  name: str | None = field(
81
90
  default=None,
82
91
  metadata={
@@ -121,18 +130,28 @@ class RunTaskCommand(click.RichCommand):
121
130
  super().__init__(obj_name, *args, **kwargs)
122
131
 
123
132
  def invoke(self, ctx: click.Context):
124
- obj: CLIConfig = _initialize_config(ctx, self.run_args.project, self.run_args.domain)
133
+ obj: CLIConfig = _initialize_config(ctx, self.run_args.project, self.run_args.domain, self.run_args.root_dir)
125
134
 
126
135
  async def _run():
127
136
  import flyte
128
137
 
138
+ console = common.get_console()
129
139
  r = await flyte.with_runcontext(
130
140
  copy_style=self.run_args.copy_style,
131
141
  mode="local" if self.run_args.local else "remote",
132
142
  name=self.run_args.name,
133
143
  ).run.aio(self.obj, **ctx.params)
144
+ if self.run_args.local:
145
+ console.print(
146
+ common.get_panel(
147
+ "Local Run",
148
+ f"[green]Completed Local Run, data stored in path: {r.url} [/green] \n"
149
+ f"➡️ Outputs: {r.outputs()}",
150
+ obj.output_format,
151
+ )
152
+ )
153
+ return
134
154
  if isinstance(r, Run) and r.action is not None:
135
- console = Console()
136
155
  console.print(
137
156
  common.get_panel(
138
157
  "Run",
@@ -212,6 +231,7 @@ class RunReferenceTaskCommand(click.RichCommand):
212
231
  import flyte.remote
213
232
 
214
233
  task = flyte.remote.Task.get(self.task_name, version=self.version, auto_version="latest")
234
+ console = common.get_console()
215
235
 
216
236
  r = await flyte.with_runcontext(
217
237
  copy_style=self.run_args.copy_style,
@@ -219,7 +239,6 @@ class RunReferenceTaskCommand(click.RichCommand):
219
239
  name=self.run_args.name,
220
240
  ).run.aio(task, **ctx.params)
221
241
  if isinstance(r, Run) and r.action is not None:
222
- console = Console()
223
242
  console.print(
224
243
  common.get_panel(
225
244
  "Run",
flyte/cli/_update.py ADDED
@@ -0,0 +1,36 @@
1
+ import rich_click as click
2
+
3
+ import flyte.remote as remote
4
+
5
+ from . import _common as common
6
+
7
+
8
+ @click.group(name="update")
9
+ def update():
10
+ """
11
+ Update various flyte entities.
12
+ """
13
+
14
+
15
+ @update.command("trigger", cls=common.CommandBase)
16
+ @click.argument("name", type=str)
17
+ @click.argument("task_name", type=str)
18
+ @click.option("--activate/--deactivate", default=None, required=True, help="Activate or deactivate the trigger.")
19
+ @click.pass_obj
20
+ def trigger(cfg: common.CLIConfig, name: str, task_name: str, activate: bool, project: str | None, domain: str | None):
21
+ """
22
+ Update a trigger.
23
+
24
+ \b
25
+ Example usage:
26
+
27
+ ```bash
28
+ flyte update trigger <trigger_name> <task_name> --activate | --deactivate
29
+ [--project <project_name> --domain <domain_name>]
30
+ ```
31
+ """
32
+ cfg.init(project, domain)
33
+ console = common.get_console()
34
+ with console.status(f"Updating trigger {name} for task {task_name} to {'active' if activate else 'deactive'}..."):
35
+ remote.Trigger.update(name, task_name, activate)
36
+ console.print("Trigger updated.")
flyte/cli/_user.py ADDED
@@ -0,0 +1,17 @@
1
+ import rich_click as click
2
+
3
+ import flyte.remote as remote
4
+
5
+ from . import _common as common
6
+
7
+
8
+ @click.command()
9
+ @click.pass_obj
10
+ def whoami(
11
+ cfg: common.CLIConfig,
12
+ ):
13
+ """Display the current user information."""
14
+ cfg.init()
15
+ console = common.get_console()
16
+ user_info = remote.User.get()
17
+ console.print(user_info.to_json())
flyte/cli/main.py CHANGED
@@ -13,6 +13,8 @@ from ._deploy import deploy
13
13
  from ._gen import gen
14
14
  from ._get import get
15
15
  from ._run import run
16
+ from ._update import update
17
+ from ._user import whoami
16
18
 
17
19
  help_config = click.RichHelpConfiguration(
18
20
  use_markdown=True,
@@ -25,7 +27,7 @@ help_config = click.RichHelpConfiguration(
25
27
  },
26
28
  {
27
29
  "name": "Management of various objects.",
28
- "commands": ["create", "get", "delete"],
30
+ "commands": ["create", "get", "delete", "update"],
29
31
  },
30
32
  {
31
33
  "name": "Build and deploy environments, tasks and images.",
@@ -35,6 +37,10 @@ help_config = click.RichHelpConfiguration(
35
37
  "name": "Documentation generation",
36
38
  "commands": ["gen"],
37
39
  },
40
+ {
41
+ "name": "User information",
42
+ "commands": ["whoami"],
43
+ },
38
44
  ]
39
45
  },
40
46
  )
@@ -193,3 +199,5 @@ main.add_command(abort) # type: ignore
193
199
  main.add_command(gen) # type: ignore
194
200
  main.add_command(delete) # type: ignore
195
201
  main.add_command(build)
202
+ main.add_command(whoami) # type: ignore
203
+ main.add_command(update) # type: ignore
flyte/errors.py CHANGED
@@ -232,3 +232,12 @@ class SlowDownError(RuntimeUserError):
232
232
 
233
233
  def __init__(self, message: str):
234
234
  super().__init__("SlowDownError", message, "user")
235
+
236
+
237
+ class OnlyAsyncIOSupportedError(RuntimeUserError):
238
+ """
239
+ This error is raised when the user tries to use sync IO in an async task.
240
+ """
241
+
242
+ def __init__(self, message: str):
243
+ super().__init__("OnlyAsyncIOSupportedError", message, "user")