flyte 0.0.1b3__py3-none-any.whl → 0.2.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of flyte might be problematic. Click here for more details.

Files changed (319) hide show
  1. flyte/__init__.py +20 -4
  2. flyte/_bin/runtime.py +33 -7
  3. flyte/_build.py +3 -2
  4. flyte/_cache/cache.py +1 -2
  5. flyte/_code_bundle/_packaging.py +1 -1
  6. flyte/_code_bundle/_utils.py +0 -16
  7. flyte/_code_bundle/bundle.py +43 -12
  8. flyte/_context.py +8 -2
  9. flyte/_deploy.py +56 -15
  10. flyte/_environment.py +45 -4
  11. flyte/_excepthook.py +37 -0
  12. flyte/_group.py +2 -1
  13. flyte/_image.py +8 -4
  14. flyte/_initialize.py +112 -254
  15. flyte/_interface.py +3 -3
  16. flyte/_internal/controllers/__init__.py +19 -6
  17. flyte/_internal/controllers/_local_controller.py +83 -8
  18. flyte/_internal/controllers/_trace.py +2 -1
  19. flyte/_internal/controllers/remote/__init__.py +27 -7
  20. flyte/_internal/controllers/remote/_action.py +7 -2
  21. flyte/_internal/controllers/remote/_client.py +5 -1
  22. flyte/_internal/controllers/remote/_controller.py +159 -26
  23. flyte/_internal/controllers/remote/_core.py +13 -5
  24. flyte/_internal/controllers/remote/_informer.py +4 -4
  25. flyte/_internal/controllers/remote/_service_protocol.py +6 -6
  26. flyte/_internal/imagebuild/docker_builder.py +12 -1
  27. flyte/_internal/imagebuild/image_builder.py +16 -11
  28. flyte/_internal/runtime/convert.py +164 -21
  29. flyte/_internal/runtime/entrypoints.py +1 -1
  30. flyte/_internal/runtime/io.py +3 -3
  31. flyte/_internal/runtime/task_serde.py +140 -20
  32. flyte/_internal/runtime/taskrunner.py +4 -3
  33. flyte/_internal/runtime/types_serde.py +1 -1
  34. flyte/_logging.py +12 -1
  35. flyte/_map.py +215 -0
  36. flyte/_pod.py +19 -0
  37. flyte/_protos/common/list_pb2.py +3 -3
  38. flyte/_protos/common/list_pb2.pyi +2 -0
  39. flyte/_protos/logs/dataplane/payload_pb2.py +28 -24
  40. flyte/_protos/logs/dataplane/payload_pb2.pyi +11 -2
  41. flyte/_protos/workflow/common_pb2.py +27 -0
  42. flyte/_protos/workflow/common_pb2.pyi +14 -0
  43. flyte/_protos/workflow/environment_pb2.py +29 -0
  44. flyte/_protos/workflow/environment_pb2.pyi +12 -0
  45. flyte/_protos/workflow/queue_service_pb2.py +40 -41
  46. flyte/_protos/workflow/queue_service_pb2.pyi +35 -30
  47. flyte/_protos/workflow/queue_service_pb2_grpc.py +15 -15
  48. flyte/_protos/workflow/run_definition_pb2.py +61 -61
  49. flyte/_protos/workflow/run_definition_pb2.pyi +8 -4
  50. flyte/_protos/workflow/run_service_pb2.py +20 -24
  51. flyte/_protos/workflow/run_service_pb2.pyi +2 -6
  52. flyte/_protos/workflow/state_service_pb2.py +36 -28
  53. flyte/_protos/workflow/state_service_pb2.pyi +19 -15
  54. flyte/_protos/workflow/state_service_pb2_grpc.py +28 -28
  55. flyte/_protos/workflow/task_definition_pb2.py +29 -22
  56. flyte/_protos/workflow/task_definition_pb2.pyi +21 -5
  57. flyte/_protos/workflow/task_service_pb2.py +27 -11
  58. flyte/_protos/workflow/task_service_pb2.pyi +29 -1
  59. flyte/_protos/workflow/task_service_pb2_grpc.py +34 -0
  60. flyte/_run.py +166 -95
  61. flyte/_task.py +110 -28
  62. flyte/_task_environment.py +55 -72
  63. flyte/_trace.py +6 -14
  64. flyte/_utils/__init__.py +6 -0
  65. flyte/_utils/async_cache.py +139 -0
  66. flyte/_utils/coro_management.py +0 -2
  67. flyte/_utils/helpers.py +45 -19
  68. flyte/_utils/org_discovery.py +57 -0
  69. flyte/_version.py +2 -2
  70. flyte/cli/__init__.py +3 -0
  71. flyte/cli/_abort.py +28 -0
  72. flyte/{_cli → cli}/_common.py +73 -23
  73. flyte/cli/_create.py +145 -0
  74. flyte/{_cli → cli}/_delete.py +4 -4
  75. flyte/{_cli → cli}/_deploy.py +26 -14
  76. flyte/cli/_gen.py +163 -0
  77. flyte/{_cli → cli}/_get.py +98 -23
  78. {union/_cli → flyte/cli}/_params.py +106 -147
  79. flyte/{_cli → cli}/_run.py +99 -20
  80. flyte/cli/main.py +166 -0
  81. flyte/config/__init__.py +3 -0
  82. flyte/config/_config.py +216 -0
  83. flyte/config/_internal.py +64 -0
  84. flyte/config/_reader.py +207 -0
  85. flyte/errors.py +29 -0
  86. flyte/extras/_container.py +33 -43
  87. flyte/io/__init__.py +17 -1
  88. flyte/io/_dir.py +2 -2
  89. flyte/io/_file.py +3 -4
  90. flyte/io/{structured_dataset → _structured_dataset}/basic_dfs.py +1 -1
  91. flyte/io/{structured_dataset → _structured_dataset}/structured_dataset.py +1 -1
  92. flyte/{_datastructures.py → models.py} +56 -7
  93. flyte/remote/__init__.py +2 -1
  94. flyte/remote/_client/_protocols.py +2 -0
  95. flyte/remote/_client/auth/_auth_utils.py +14 -0
  96. flyte/remote/_client/auth/_channel.py +34 -3
  97. flyte/remote/_client/auth/_token_client.py +3 -3
  98. flyte/remote/_client/controlplane.py +13 -13
  99. flyte/remote/_console.py +1 -1
  100. flyte/remote/_data.py +10 -6
  101. flyte/remote/_logs.py +89 -29
  102. flyte/remote/_project.py +8 -9
  103. flyte/remote/_run.py +228 -131
  104. flyte/remote/_secret.py +12 -12
  105. flyte/remote/_task.py +179 -15
  106. flyte/report/_report.py +4 -4
  107. flyte/storage/__init__.py +5 -0
  108. flyte/storage/_config.py +233 -0
  109. flyte/storage/_storage.py +23 -3
  110. flyte/syncify/__init__.py +56 -0
  111. flyte/syncify/_api.py +371 -0
  112. flyte/types/__init__.py +23 -0
  113. flyte/types/_interface.py +22 -7
  114. flyte/{io/pickle/transformer.py → types/_pickle.py} +2 -1
  115. flyte/types/_type_engine.py +95 -18
  116. flyte-0.2.0a0.dist-info/METADATA +249 -0
  117. flyte-0.2.0a0.dist-info/RECORD +218 -0
  118. {flyte-0.0.1b3.dist-info → flyte-0.2.0a0.dist-info}/entry_points.txt +1 -1
  119. flyte/_api_commons.py +0 -3
  120. flyte/_cli/__init__.py +0 -0
  121. flyte/_cli/_create.py +0 -42
  122. flyte/_cli/main.py +0 -72
  123. flyte/_internal/controllers/pbhash.py +0 -39
  124. flyte/io/_dataframe.py +0 -0
  125. flyte/io/pickle/__init__.py +0 -0
  126. flyte-0.0.1b3.dist-info/METADATA +0 -179
  127. flyte-0.0.1b3.dist-info/RECORD +0 -390
  128. union/__init__.py +0 -54
  129. union/_api_commons.py +0 -3
  130. union/_bin/__init__.py +0 -0
  131. union/_bin/runtime.py +0 -113
  132. union/_build.py +0 -25
  133. union/_cache/__init__.py +0 -12
  134. union/_cache/cache.py +0 -141
  135. union/_cache/defaults.py +0 -9
  136. union/_cache/policy_function_body.py +0 -42
  137. union/_cli/__init__.py +0 -0
  138. union/_cli/_common.py +0 -263
  139. union/_cli/_create.py +0 -40
  140. union/_cli/_delete.py +0 -23
  141. union/_cli/_deploy.py +0 -120
  142. union/_cli/_get.py +0 -162
  143. union/_cli/_run.py +0 -150
  144. union/_cli/main.py +0 -72
  145. union/_code_bundle/__init__.py +0 -8
  146. union/_code_bundle/_ignore.py +0 -113
  147. union/_code_bundle/_packaging.py +0 -187
  148. union/_code_bundle/_utils.py +0 -342
  149. union/_code_bundle/bundle.py +0 -176
  150. union/_context.py +0 -146
  151. union/_datastructures.py +0 -295
  152. union/_deploy.py +0 -185
  153. union/_doc.py +0 -29
  154. union/_docstring.py +0 -26
  155. union/_environment.py +0 -43
  156. union/_group.py +0 -31
  157. union/_hash.py +0 -23
  158. union/_image.py +0 -760
  159. union/_initialize.py +0 -585
  160. union/_interface.py +0 -84
  161. union/_internal/__init__.py +0 -3
  162. union/_internal/controllers/__init__.py +0 -77
  163. union/_internal/controllers/_local_controller.py +0 -77
  164. union/_internal/controllers/pbhash.py +0 -39
  165. union/_internal/controllers/remote/__init__.py +0 -40
  166. union/_internal/controllers/remote/_action.py +0 -131
  167. union/_internal/controllers/remote/_client.py +0 -43
  168. union/_internal/controllers/remote/_controller.py +0 -169
  169. union/_internal/controllers/remote/_core.py +0 -341
  170. union/_internal/controllers/remote/_informer.py +0 -260
  171. union/_internal/controllers/remote/_service_protocol.py +0 -44
  172. union/_internal/imagebuild/__init__.py +0 -11
  173. union/_internal/imagebuild/docker_builder.py +0 -416
  174. union/_internal/imagebuild/image_builder.py +0 -243
  175. union/_internal/imagebuild/remote_builder.py +0 -0
  176. union/_internal/resolvers/__init__.py +0 -0
  177. union/_internal/resolvers/_task_module.py +0 -31
  178. union/_internal/resolvers/common.py +0 -24
  179. union/_internal/resolvers/default.py +0 -27
  180. union/_internal/runtime/__init__.py +0 -0
  181. union/_internal/runtime/convert.py +0 -163
  182. union/_internal/runtime/entrypoints.py +0 -121
  183. union/_internal/runtime/io.py +0 -136
  184. union/_internal/runtime/resources_serde.py +0 -134
  185. union/_internal/runtime/task_serde.py +0 -202
  186. union/_internal/runtime/taskrunner.py +0 -179
  187. union/_internal/runtime/types_serde.py +0 -53
  188. union/_logging.py +0 -124
  189. union/_protos/__init__.py +0 -0
  190. union/_protos/common/authorization_pb2.py +0 -66
  191. union/_protos/common/authorization_pb2.pyi +0 -106
  192. union/_protos/common/identifier_pb2.py +0 -71
  193. union/_protos/common/identifier_pb2.pyi +0 -82
  194. union/_protos/common/identity_pb2.py +0 -48
  195. union/_protos/common/identity_pb2.pyi +0 -72
  196. union/_protos/common/identity_pb2_grpc.py +0 -4
  197. union/_protos/common/list_pb2.py +0 -36
  198. union/_protos/common/list_pb2.pyi +0 -69
  199. union/_protos/common/list_pb2_grpc.py +0 -4
  200. union/_protos/common/policy_pb2.py +0 -37
  201. union/_protos/common/policy_pb2.pyi +0 -27
  202. union/_protos/common/policy_pb2_grpc.py +0 -4
  203. union/_protos/common/role_pb2.py +0 -37
  204. union/_protos/common/role_pb2.pyi +0 -51
  205. union/_protos/common/role_pb2_grpc.py +0 -4
  206. union/_protos/common/runtime_version_pb2.py +0 -28
  207. union/_protos/common/runtime_version_pb2.pyi +0 -24
  208. union/_protos/common/runtime_version_pb2_grpc.py +0 -4
  209. union/_protos/logs/dataplane/payload_pb2.py +0 -96
  210. union/_protos/logs/dataplane/payload_pb2.pyi +0 -168
  211. union/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
  212. union/_protos/secret/definition_pb2.py +0 -49
  213. union/_protos/secret/definition_pb2.pyi +0 -93
  214. union/_protos/secret/definition_pb2_grpc.py +0 -4
  215. union/_protos/secret/payload_pb2.py +0 -62
  216. union/_protos/secret/payload_pb2.pyi +0 -94
  217. union/_protos/secret/payload_pb2_grpc.py +0 -4
  218. union/_protos/secret/secret_pb2.py +0 -38
  219. union/_protos/secret/secret_pb2.pyi +0 -6
  220. union/_protos/secret/secret_pb2_grpc.py +0 -198
  221. union/_protos/validate/validate/validate_pb2.py +0 -76
  222. union/_protos/workflow/node_execution_service_pb2.py +0 -26
  223. union/_protos/workflow/node_execution_service_pb2.pyi +0 -4
  224. union/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
  225. union/_protos/workflow/queue_service_pb2.py +0 -75
  226. union/_protos/workflow/queue_service_pb2.pyi +0 -103
  227. union/_protos/workflow/queue_service_pb2_grpc.py +0 -172
  228. union/_protos/workflow/run_definition_pb2.py +0 -100
  229. union/_protos/workflow/run_definition_pb2.pyi +0 -256
  230. union/_protos/workflow/run_definition_pb2_grpc.py +0 -4
  231. union/_protos/workflow/run_logs_service_pb2.py +0 -41
  232. union/_protos/workflow/run_logs_service_pb2.pyi +0 -28
  233. union/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
  234. union/_protos/workflow/run_service_pb2.py +0 -133
  235. union/_protos/workflow/run_service_pb2.pyi +0 -173
  236. union/_protos/workflow/run_service_pb2_grpc.py +0 -412
  237. union/_protos/workflow/state_service_pb2.py +0 -58
  238. union/_protos/workflow/state_service_pb2.pyi +0 -69
  239. union/_protos/workflow/state_service_pb2_grpc.py +0 -138
  240. union/_protos/workflow/task_definition_pb2.py +0 -72
  241. union/_protos/workflow/task_definition_pb2.pyi +0 -65
  242. union/_protos/workflow/task_definition_pb2_grpc.py +0 -4
  243. union/_protos/workflow/task_service_pb2.py +0 -44
  244. union/_protos/workflow/task_service_pb2.pyi +0 -31
  245. union/_protos/workflow/task_service_pb2_grpc.py +0 -104
  246. union/_resources.py +0 -226
  247. union/_retry.py +0 -32
  248. union/_reusable_environment.py +0 -25
  249. union/_run.py +0 -374
  250. union/_secret.py +0 -61
  251. union/_task.py +0 -354
  252. union/_task_environment.py +0 -186
  253. union/_timeout.py +0 -47
  254. union/_tools.py +0 -27
  255. union/_utils/__init__.py +0 -11
  256. union/_utils/asyn.py +0 -119
  257. union/_utils/file_handling.py +0 -71
  258. union/_utils/helpers.py +0 -46
  259. union/_utils/lazy_module.py +0 -54
  260. union/_utils/uv_script_parser.py +0 -49
  261. union/_version.py +0 -21
  262. union/connectors/__init__.py +0 -0
  263. union/errors.py +0 -128
  264. union/extras/__init__.py +0 -5
  265. union/extras/_container.py +0 -263
  266. union/io/__init__.py +0 -11
  267. union/io/_dataframe.py +0 -0
  268. union/io/_dir.py +0 -425
  269. union/io/_file.py +0 -418
  270. union/io/pickle/__init__.py +0 -0
  271. union/io/pickle/transformer.py +0 -117
  272. union/io/structured_dataset/__init__.py +0 -122
  273. union/io/structured_dataset/basic_dfs.py +0 -219
  274. union/io/structured_dataset/structured_dataset.py +0 -1057
  275. union/py.typed +0 -0
  276. union/remote/__init__.py +0 -23
  277. union/remote/_client/__init__.py +0 -0
  278. union/remote/_client/_protocols.py +0 -129
  279. union/remote/_client/auth/__init__.py +0 -12
  280. union/remote/_client/auth/_authenticators/__init__.py +0 -0
  281. union/remote/_client/auth/_authenticators/base.py +0 -391
  282. union/remote/_client/auth/_authenticators/client_credentials.py +0 -73
  283. union/remote/_client/auth/_authenticators/device_code.py +0 -120
  284. union/remote/_client/auth/_authenticators/external_command.py +0 -77
  285. union/remote/_client/auth/_authenticators/factory.py +0 -200
  286. union/remote/_client/auth/_authenticators/pkce.py +0 -515
  287. union/remote/_client/auth/_channel.py +0 -184
  288. union/remote/_client/auth/_client_config.py +0 -83
  289. union/remote/_client/auth/_default_html.py +0 -32
  290. union/remote/_client/auth/_grpc_utils/__init__.py +0 -0
  291. union/remote/_client/auth/_grpc_utils/auth_interceptor.py +0 -204
  292. union/remote/_client/auth/_grpc_utils/default_metadata_interceptor.py +0 -144
  293. union/remote/_client/auth/_keyring.py +0 -154
  294. union/remote/_client/auth/_token_client.py +0 -258
  295. union/remote/_client/auth/errors.py +0 -16
  296. union/remote/_client/controlplane.py +0 -86
  297. union/remote/_data.py +0 -149
  298. union/remote/_logs.py +0 -74
  299. union/remote/_project.py +0 -86
  300. union/remote/_run.py +0 -820
  301. union/remote/_secret.py +0 -132
  302. union/remote/_task.py +0 -193
  303. union/report/__init__.py +0 -3
  304. union/report/_report.py +0 -178
  305. union/report/_template.html +0 -124
  306. union/storage/__init__.py +0 -24
  307. union/storage/_remote_fs.py +0 -34
  308. union/storage/_storage.py +0 -247
  309. union/storage/_utils.py +0 -5
  310. union/types/__init__.py +0 -11
  311. union/types/_renderer.py +0 -162
  312. union/types/_string_literals.py +0 -120
  313. union/types/_type_engine.py +0 -2131
  314. union/types/_utils.py +0 -80
  315. /union/_protos/common/authorization_pb2_grpc.py → /flyte/_protos/workflow/common_pb2_grpc.py +0 -0
  316. /union/_protos/common/identifier_pb2_grpc.py → /flyte/_protos/workflow/environment_pb2_grpc.py +0 -0
  317. /flyte/io/{structured_dataset → _structured_dataset}/__init__.py +0 -0
  318. {flyte-0.0.1b3.dist-info → flyte-0.2.0a0.dist-info}/WHEEL +0 -0
  319. {flyte-0.0.1b3.dist-info → flyte-0.2.0a0.dist-info}/top_level.txt +0 -0
flyte/__init__.py CHANGED
@@ -17,16 +17,15 @@ async def my_task():
17
17
  """
18
18
 
19
19
  __all__ = [
20
- "ABFS",
21
- "GCS",
22
20
  "GPU",
23
- "S3",
24
21
  "TPU",
25
22
  "Cache",
26
23
  "CachePolicy",
27
24
  "CacheRequest",
28
25
  "Device",
26
+ "Environment",
29
27
  "Image",
28
+ "PodTemplate",
30
29
  "Resources",
31
30
  "RetryStrategy",
32
31
  "ReusePolicy",
@@ -40,17 +39,25 @@ __all__ = [
40
39
  "deploy",
41
40
  "group",
42
41
  "init",
42
+ "init_from_config",
43
+ "map",
43
44
  "run",
44
45
  "trace",
45
46
  "with_runcontext",
46
47
  ]
47
48
 
49
+ import sys
50
+
48
51
  from ._cache import Cache, CachePolicy, CacheRequest
49
52
  from ._context import ctx
50
53
  from ._deploy import deploy
54
+ from ._environment import Environment
55
+ from ._excepthook import custom_excepthook
51
56
  from ._group import group
52
57
  from ._image import Image
53
- from ._initialize import ABFS, GCS, S3, init
58
+ from ._initialize import init, init_from_config
59
+ from ._map import map
60
+ from ._pod import PodTemplate
54
61
  from ._resources import GPU, TPU, Device, Resources
55
62
  from ._retry import RetryStrategy
56
63
  from ._reusable_environment import ReusePolicy
@@ -60,3 +67,12 @@ from ._task_environment import TaskEnvironment
60
67
  from ._timeout import Timeout, TimeoutType
61
68
  from ._trace import trace
62
69
  from ._version import __version__
70
+
71
+ sys.excepthook = custom_excepthook
72
+
73
+
74
+ def version() -> str:
75
+ """
76
+ Returns the version of the Flyte SDK.
77
+ """
78
+ return __version__
flyte/_bin/runtime.py CHANGED
@@ -8,7 +8,7 @@ Refrain from importing any modules here. If you need to import any modules, do i
8
8
  import asyncio
9
9
  import os
10
10
  import sys
11
- from typing import List
11
+ from typing import Any, List
12
12
 
13
13
  import click
14
14
 
@@ -27,6 +27,9 @@ ORG_NAME = "_U_ORG_NAME"
27
27
  ENDPOINT_OVERRIDE = "_U_EP_OVERRIDE"
28
28
  RUN_OUTPUT_BASE_DIR = "_U_RUN_BASE"
29
29
 
30
+ # TODO: Remove this after proper auth is implemented
31
+ _UNION_EAGER_API_KEY_ENV_VAR = "_UNION_EAGER_API_KEY"
32
+
30
33
 
31
34
  @click.command("a0")
32
35
  @click.option("--inputs", "-i", required=True)
@@ -73,12 +76,16 @@ def main(
73
76
  ):
74
77
  sys.path.insert(0, ".")
75
78
 
79
+ import flyte
76
80
  import flyte._utils as utils
77
- from flyte._datastructures import ActionID, Checkpoints, CodeBundle, RawDataPath
78
- from flyte._initialize import S3, initialize_in_cluster
81
+ from flyte._initialize import init
79
82
  from flyte._internal.controllers import create_controller
80
83
  from flyte._internal.imagebuild.image_builder import ImageCache
81
84
  from flyte._internal.runtime.entrypoints import load_and_run_task
85
+ from flyte._logging import logger
86
+ from flyte.models import ActionID, Checkpoints, CodeBundle, RawDataPath
87
+
88
+ logger.info(f"Initializing flyte runtime - version {flyte.__version__}")
82
89
 
83
90
  assert org, "Org is required for now"
84
91
  assert project, "Project is required"
@@ -91,12 +98,31 @@ def main(
91
98
  if name.startswith("{{"):
92
99
  name = os.getenv("ACTION_NAME", "")
93
100
 
94
- ep = os.environ.get(ENDPOINT_OVERRIDE, "host.docker.internal:8090")
101
+ # Figure out how to connect
102
+ # This detection of api key is a hack for now.
103
+ controller_kwargs: dict[str, Any] = {"insecure": False}
104
+ if api_key := os.getenv(_UNION_EAGER_API_KEY_ENV_VAR):
105
+ logger.info("Using api key from environment")
106
+ controller_kwargs["api_key"] = api_key
107
+ else:
108
+ ep = os.environ.get(ENDPOINT_OVERRIDE, "host.docker.internal:8090")
109
+ controller_kwargs["endpoint"] = ep
110
+ if "localhost" in ep or "docker" in ep:
111
+ controller_kwargs["insecure"] = True
112
+ logger.debug(f"Using controller endpoint: {ep} with kwargs: {controller_kwargs}")
95
113
 
96
114
  bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
97
- # TODO configure storage correctly for cluster
98
- initialize_in_cluster(storage=S3.auto())
99
- controller = create_controller(ct="remote", endpoint=ep, insecure=True)
115
+ # We init regular client here so that reference tasks can work
116
+ # Current reference tasks will not work with remote controller, because we create 2 different
117
+ # channels on different threads and this is not supported by grpcio or the auth system. It ends up leading
118
+ # File "src/python/grpcio/grpc/_cython/_cygrpc/aio/completion_queue.pyx.pxi", line 147,
119
+ # in grpc._cython.cygrpc.PollerCompletionQueue._handle_events
120
+ # BlockingIOError: [Errno 11] Resource temporarily unavailable
121
+ # init(org=org, project=project, domain=domain, **controller_kwargs)
122
+ # TODO solution is to use a single channel for both controller and reference tasks, but this requires a refactor
123
+ init()
124
+ # Controller is created with the same kwargs as init, so that it can be used to run tasks
125
+ controller = create_controller(ct="remote", **controller_kwargs)
100
126
 
101
127
  ic = ImageCache.from_transport(image_cache) if image_cache else None
102
128
 
flyte/_build.py CHANGED
@@ -1,10 +1,11 @@
1
1
  from __future__ import annotations
2
2
 
3
- from ._api_commons import syncer
3
+ from flyte.syncify import syncify
4
+
4
5
  from ._image import Image
5
6
 
6
7
 
7
- @syncer.wrap
8
+ @syncify
8
9
  async def build(image: Image) -> str:
9
10
  """
10
11
  Build an image. The existing async context will be used.
flyte/_cache/cache.py CHANGED
@@ -14,10 +14,9 @@ from typing import (
14
14
  import rich.repr
15
15
  from typing_extensions import Literal, ParamSpec, TypeVar, get_args
16
16
 
17
- from flyte._datastructures import CodeBundle
18
-
19
17
  # if TYPE_CHECKING:
20
18
  from flyte._image import Image
19
+ from flyte.models import CodeBundle
21
20
 
22
21
  P = ParamSpec("P")
23
22
  FuncOut = TypeVar("FuncOut")
@@ -91,7 +91,7 @@ def list_files_to_bundle(
91
91
  ignore = IgnoreGroup(source, *ignores)
92
92
 
93
93
  ls, ls_digest = ls_files(source, copy_style, deref_symlinks, ignore)
94
- logger.debug(f"Hash digest: {ls_digest}")
94
+ logger.debug(f"Hash of files to be included in the code bundle: {ls_digest}")
95
95
  return ls, ls_digest
96
96
 
97
97
 
@@ -14,7 +14,6 @@ import tempfile
14
14
  import typing
15
15
  from datetime import datetime, timezone
16
16
  from functools import lru_cache
17
- from pathlib import Path
18
17
  from types import ModuleType
19
18
  from typing import List, Literal, Optional, Tuple, Union
20
19
 
@@ -322,18 +321,3 @@ def hash_file(file_path: typing.Union[os.PathLike, str]) -> Tuple[bytes, str, in
322
321
  size += len(chunk)
323
322
 
324
323
  return h.digest(), h.hexdigest(), size
325
-
326
-
327
- def _find_project_root(source_path) -> str:
328
- """
329
- Find the root of the project.
330
- The root of the project is considered to be the first ancestor from source_path that does
331
- not contain a __init__.py file.
332
-
333
- N.B.: This assumption only holds for regular packages (as opposed to namespace packages)
334
- """
335
- # Start from the directory right above source_path
336
- path = Path(source_path).parent.resolve()
337
- while os.path.exists(os.path.join(path, "__init__.py")):
338
- path = path.parent
339
- return str(path)
@@ -5,13 +5,13 @@ import os
5
5
  import pathlib
6
6
  import tempfile
7
7
  from pathlib import Path
8
- from typing import Type
8
+ from typing import ClassVar, Type
9
9
 
10
10
  from flyteidl.core.tasks_pb2 import TaskTemplate
11
11
 
12
- import flyte.storage as storage
13
- from flyte._datastructures import CodeBundle
14
12
  from flyte._logging import log, logger
13
+ from flyte._utils import AsyncLRUCache
14
+ from flyte.models import CodeBundle
15
15
 
16
16
  from ._ignore import GitIgnore, Ignore, StandardIgnore
17
17
  from ._packaging import create_bundle, list_files_to_bundle, print_ls_tree
@@ -21,10 +21,34 @@ _pickled_file_extension = ".pkl.gz"
21
21
  _tar_file_extension = ".tar.gz"
22
22
 
23
23
 
24
+ class _PklCache:
25
+ _pkl_cache: ClassVar[AsyncLRUCache[str, str]] = AsyncLRUCache[str, str](maxsize=100)
26
+
27
+ @classmethod
28
+ async def put(cls, digest: str, upload_to_path: str, from_path: pathlib.Path) -> str:
29
+ """
30
+ Get the pickled code bundle from the cache or build it if not present.
31
+
32
+ :param digest: The hash digest of the task template.
33
+ :param upload_to_path: The path to upload the pickled file to.
34
+ :param from_path: The path to read the pickled file from.
35
+ :return: CodeBundle object containing the pickled file path and the computed version.
36
+ """
37
+ import flyte.storage as storage
38
+
39
+ async def put_data() -> str:
40
+ return await storage.put(str(from_path), to_path=str(upload_to_path))
41
+
42
+ return await cls._pkl_cache.get(
43
+ key=digest,
44
+ value_func=put_data,
45
+ )
46
+
47
+
24
48
  async def build_pkl_bundle(
25
49
  o: TaskTemplate,
26
50
  upload_to_controlplane: bool = True,
27
- upload_from_dataplane_path: str | None = None,
51
+ upload_from_dataplane_base_path: str | None = None,
28
52
  copy_bundle_to: pathlib.Path | None = None,
29
53
  ) -> CodeBundle:
30
54
  """
@@ -36,16 +60,14 @@ async def build_pkl_bundle(
36
60
 
37
61
  :param o: Object to be pickled. This is the task template.
38
62
  :param upload_to_controlplane: Whether to upload the pickled file to the control plane or not
39
- :param upload_from_dataplane_path: If we are on the dataplane, this is the path where the
63
+ :param upload_from_dataplane_base_path: If we are on the dataplane, this is the path where the
40
64
  pickled file should be uploaded to. upload_to_controlplane has to be False in this case.
41
65
  :param copy_bundle_to: If set, the bundle will be copied to this path. This is used for testing purposes.
42
66
  :return: CodeBundle object containing the pickled file path and the computed version.
43
67
  """
44
68
  import cloudpickle
45
69
 
46
- import flyte.storage as storage
47
-
48
- if upload_to_controlplane and upload_from_dataplane_path:
70
+ if upload_to_controlplane and upload_from_dataplane_base_path:
49
71
  raise ValueError("Cannot upload to control plane and upload from dataplane path at the same time.")
50
72
 
51
73
  logger.debug("Building pickled code bundle.")
@@ -61,10 +83,17 @@ async def build_pkl_bundle(
61
83
  hash_digest, remote_path = await upload_file(dest)
62
84
  return CodeBundle(pkl=remote_path, computed_version=hash_digest)
63
85
 
64
- elif upload_from_dataplane_path:
65
- logger.debug(f"Uploading pickled code bundle to dataplane path {upload_from_dataplane_path}.")
86
+ elif upload_from_dataplane_base_path:
87
+ from flyte._internal.runtime import io
88
+
66
89
  _, str_digest, _ = hash_file(file_path=dest)
67
- final_path = await storage.put(str(dest), upload_from_dataplane_path)
90
+ upload_path = io.pkl_path(upload_from_dataplane_base_path, str_digest)
91
+ logger.debug(f"Uploading pickled code bundle to dataplane path {upload_path}.")
92
+ final_path = await _PklCache.put(
93
+ digest=str_digest,
94
+ upload_to_path=upload_path,
95
+ from_path=dest,
96
+ )
68
97
  return CodeBundle(pkl=final_path, computed_version=str_digest)
69
98
 
70
99
  else:
@@ -117,7 +146,7 @@ async def build_code_bundle(
117
146
  logger.info(f"Code bundle created at {bundle_path}, size: {tar_size} MB, archive size: {archive_size} MB")
118
147
  if not dryrun:
119
148
  hash_digest, remote_path = await upload_file(bundle_path)
120
- logger.info(f"Code bundle uploaded to {remote_path}")
149
+ logger.debug(f"Code bundle uploaded to {remote_path}")
121
150
  else:
122
151
  remote_path = "na"
123
152
  if copy_bundle_to:
@@ -138,6 +167,8 @@ async def download_bundle(bundle: CodeBundle) -> pathlib.Path:
138
167
 
139
168
  :return: The path to the downloaded code bundle.
140
169
  """
170
+ import flyte.storage as storage
171
+
141
172
  dest = pathlib.Path(bundle.destination)
142
173
  if not dest.is_dir():
143
174
  raise ValueError(f"Destination path should be a directory, found {dest}, {dest.stat()}")
flyte/_context.py CHANGED
@@ -4,7 +4,8 @@ import contextvars
4
4
  from dataclasses import dataclass, replace
5
5
  from typing import TYPE_CHECKING, Awaitable, Callable, Optional, ParamSpec, TypeVar
6
6
 
7
- from flyte._datastructures import GroupData, RawDataPath, TaskContext
7
+ from flyte._logging import logger
8
+ from flyte.models import GroupData, RawDataPath, TaskContext
8
9
 
9
10
  if TYPE_CHECKING:
10
11
  from flyte.report import Report
@@ -49,6 +50,7 @@ class Context:
49
50
  raise ValueError("Cannot create a new context without contextdata.")
50
51
  self._data = data
51
52
  self._id = id(self) # Immutable unique identifier
53
+ self._token = None # Context variable token to restore the previous context
52
54
 
53
55
  @property
54
56
  def data(self) -> ContextData:
@@ -106,7 +108,11 @@ class Context:
106
108
 
107
109
  def __exit__(self, exc_type, exc_val, exc_tb):
108
110
  """Exit the context, restoring the previous context."""
109
- root_context_var.reset(self._token)
111
+ try:
112
+ root_context_var.reset(self._token)
113
+ except Exception as e:
114
+ logger.warn(f"Failed to reset context: {e}")
115
+ raise e
110
116
 
111
117
  async def __aenter__(self):
112
118
  """Async version of context entry."""
flyte/_deploy.py CHANGED
@@ -6,11 +6,13 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
6
6
 
7
7
  import rich.repr
8
8
 
9
- from ._api_commons import syncer
10
- from ._datastructures import SerializationContext
9
+ import flyte.errors
10
+ from flyte.models import SerializationContext
11
+ from flyte.syncify import syncify
12
+
11
13
  from ._environment import Environment
12
14
  from ._image import Image
13
- from ._initialize import get_client, get_common_config, requires_client, requires_initialization
15
+ from ._initialize import ensure_client, get_client, get_common_config, requires_initialization
14
16
  from ._logging import logger
15
17
  from ._task import TaskTemplate
16
18
  from ._task_environment import TaskEnvironment
@@ -45,22 +47,55 @@ class Deployment:
45
47
  )
46
48
  return f"Deployment(envs=[{env_names}], tasks=[{task_names_versions}])"
47
49
 
50
+ def task_repr(self) -> List[List[Tuple[str, str]]]:
51
+ """
52
+ Returns a detailed representation of the deployed tasks.
53
+ """
54
+ tuples = []
55
+ if self.deployed_tasks:
56
+ for task in self.deployed_tasks:
57
+ tuples.append(
58
+ [
59
+ ("name", task.task_template.id.name),
60
+ ("version", task.task_template.id.version),
61
+ ]
62
+ )
63
+ return tuples
64
+
65
+ def env_repr(self) -> List[List[Tuple[str, str]]]:
66
+ """
67
+ Returns a detailed representation of the deployed environments.
68
+ """
69
+ tuples = []
70
+ for env_name, env in self.envs.items():
71
+ tuples.append(
72
+ [
73
+ ("environment", env_name),
74
+ ("image", env.image.uri if isinstance(env.image, Image) else env.image or ""),
75
+ ]
76
+ )
77
+ return tuples
78
+
48
79
 
49
- @requires_client
50
80
  async def _deploy_task(
51
81
  task: TaskTemplate, serialization_context: SerializationContext, dryrun: bool = False
52
82
  ) -> task_definition_pb2.TaskSpec:
53
83
  """
54
84
  Deploy the given task.
55
85
  """
86
+ ensure_client()
87
+ from ._internal.runtime.convert import convert_upload_default_inputs
56
88
  from ._internal.runtime.task_serde import translate_task_to_wire
57
89
  from ._protos.workflow import task_definition_pb2, task_service_pb2
58
90
 
59
91
  image_uri = task.image.uri if isinstance(task.image, Image) else task.image
60
92
 
61
- spec = translate_task_to_wire(task, serialization_context)
62
93
  if dryrun:
63
- return spec
94
+ return translate_task_to_wire(task, serialization_context)
95
+
96
+ default_inputs = await convert_upload_default_inputs(task.interface)
97
+ spec = translate_task_to_wire(task, serialization_context, default_inputs=default_inputs)
98
+
64
99
  msg = f"Deploying task {task.name}, with image {image_uri} version {serialization_context.version}"
65
100
  if spec.task_template.HasField("container") and spec.task_template.container.args:
66
101
  msg += f" from {spec.task_template.container.args[-3]}.{spec.task_template.container.args[-1]}"
@@ -98,7 +133,7 @@ async def build_images(deployment: DeploymentPlan) -> ImageCache:
98
133
  image_identifier_map = {}
99
134
  for env_name, env in deployment.envs.items():
100
135
  if not isinstance(env.image, str):
101
- logger.info(f"Building Image for environment {env_name}, image: {env.image}")
136
+ logger.warning(f"Building Image for environment {env_name}, image: {env.image}")
102
137
  images.append(_build_image_bg(env_name, env.image))
103
138
 
104
139
  elif env.image == "auto" and "auto" not in image_identifier_map:
@@ -107,7 +142,7 @@ async def build_images(deployment: DeploymentPlan) -> ImageCache:
107
142
  final_images = await asyncio.gather(*images)
108
143
 
109
144
  for env_name, image_uri in final_images:
110
- logger.info(f"Built Image for environment {env_name}, image: {image_uri}")
145
+ logger.warning(f"Built Image for environment {env_name}, image: {image_uri}")
111
146
  env = deployment.envs[env_name]
112
147
  if isinstance(env.image, Image):
113
148
  image_identifier_map[env.image.identifier] = env.image.uri
@@ -121,19 +156,24 @@ async def apply(deployment: DeploymentPlan, copy_style: CopyFiles, dryrun: bool
121
156
 
122
157
  cfg = get_common_config()
123
158
  image_cache = await build_images(deployment)
124
- if copy_style == "none":
125
- code_bundle = None
126
- assert deployment.version is not None, "Version must be set when copy_style is none"
159
+
160
+ version = deployment.version
161
+ code_bundle = None
162
+ if copy_style == "none" and not version:
163
+ raise flyte.errors.DeploymentError("Version must be set when copy_style is none")
127
164
  else:
128
165
  code_bundle = await build_code_bundle(from_dir=cfg.root_dir, dryrun=dryrun, copy_style=copy_style)
129
- deployment.version = code_bundle.computed_version
166
+ version = version or code_bundle.computed_version
167
+ # TODO we should update the version to include the image cache digest and code bundle digest. This is
168
+ # to ensure that changes in image dependencies, cause an update to the deployment version.
169
+ # TODO Also hash the environment and tasks to ensure that changes in the environment or tasks
130
170
 
131
171
  sc = SerializationContext(
132
172
  project=cfg.project,
133
173
  domain=cfg.domain,
134
174
  org=cfg.org,
135
175
  code_bundle=code_bundle,
136
- version=deployment.version,
176
+ version=version,
137
177
  image_cache=image_cache,
138
178
  root_dir=cfg.root_dir,
139
179
  )
@@ -141,6 +181,7 @@ async def apply(deployment: DeploymentPlan, copy_style: CopyFiles, dryrun: bool
141
181
  tasks = []
142
182
  for env_name, env in deployment.envs.items():
143
183
  logger.info(f"Deploying environment {env_name}")
184
+ # TODO Make this pluggable based on the environment type
144
185
  if isinstance(env, TaskEnvironment):
145
186
  for task in env.tasks.values():
146
187
  tasks.append(_deploy_task(task, dryrun=dryrun, serialization_context=sc))
@@ -161,7 +202,7 @@ def _recursive_discover(
161
202
  if env.name in planned_envs:
162
203
  continue
163
204
  # Recursively discover dependent environments
164
- for dependent_env in env.env_dep_hints:
205
+ for dependent_env in env.depends_on:
165
206
  _recursive_discover(planned_envs, dependent_env)
166
207
  # Add the environment to the existing envs
167
208
  planned_envs[env.name] = env
@@ -175,7 +216,7 @@ def plan_deploy(*envs: Environment, version: Optional[str] = None) -> Deployment
175
216
  return DeploymentPlan(planned_envs, version=version)
176
217
 
177
218
 
178
- @syncer.wrap
219
+ @syncify
179
220
  async def deploy(
180
221
  *envs: Environment,
181
222
  dryrun: bool = False,
flyte/_environment.py CHANGED
@@ -1,7 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import re
3
4
  from dataclasses import dataclass, field
4
- from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union
5
+ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
5
6
 
6
7
  import rich.repr
7
8
 
@@ -14,6 +15,10 @@ if TYPE_CHECKING:
14
15
  from kubernetes.client import V1PodTemplate
15
16
 
16
17
 
18
+ def is_snake_or_kebab_with_numbers(s: str) -> bool:
19
+ return re.fullmatch(r"^[a-z0-9]+([_-][a-z0-9]+)*$", s) is not None
20
+
21
+
17
22
  @rich.repr.auto
18
23
  @dataclass(init=True, repr=True)
19
24
  class Environment:
@@ -23,12 +28,12 @@ class Environment:
23
28
  :param resources: Resources to allocate for the environment.
24
29
  :param env: Environment variables to set for the environment.
25
30
  :param secrets: Secrets to inject into the environment.
26
- :param env_dep_hints: Environment dependencies to hint, so when you deploy the environment, the dependencies are
31
+ :param depends_on: Environment dependencies to hint, so when you deploy the environment, the dependencies are
27
32
  also deployed. This is useful when you have a set of environments that depend on each other.
28
33
  """
29
34
 
30
35
  name: str
31
- env_dep_hints: List[Environment] = field(default_factory=list)
36
+ depends_on: List[Environment] = field(default_factory=list)
32
37
  pod_template: Optional[Union[str, "V1PodTemplate"]] = None
33
38
  description: Optional[str] = None
34
39
  secrets: Optional[SecretRequest] = None
@@ -36,8 +41,44 @@ class Environment:
36
41
  resources: Optional[Resources] = None
37
42
  image: Union[str, Image, Literal["auto"]] = "auto"
38
43
 
44
+ def __post_init__(self):
45
+ if not is_snake_or_kebab_with_numbers(self.name):
46
+ raise ValueError(f"Environment name '{self.name}' must be in snake_case or kebab-case format.")
47
+
39
48
  def add_dependency(self, *env: Environment):
40
49
  """
41
50
  Add a dependency to the environment.
42
51
  """
43
- self.env_dep_hints.extend(env)
52
+ self.depends_on.extend(env)
53
+
54
+ def clone_with(
55
+ self,
56
+ name: str,
57
+ image: Optional[Union[str, Image, Literal["auto"]]] = None,
58
+ resources: Optional[Resources] = None,
59
+ env: Optional[Dict[str, str]] = None,
60
+ secrets: Optional[SecretRequest] = None,
61
+ depends_on: Optional[List[Environment]] = None,
62
+ **kwargs: Any,
63
+ ) -> Environment:
64
+ raise NotImplementedError
65
+
66
+ def _get_kwargs(self) -> Dict[str, Any]:
67
+ """
68
+ Get the keyword arguments for the environment.
69
+ """
70
+ kwargs: Dict[str, Any] = {
71
+ "depends_on": self.depends_on,
72
+ "image": self.image,
73
+ }
74
+ if self.resources is not None:
75
+ kwargs["resources"] = self.resources
76
+ if self.secrets is not None:
77
+ kwargs["secrets"] = self.secrets
78
+ if self.env is not None:
79
+ kwargs["env"] = self.env
80
+ if self.pod_template is not None:
81
+ kwargs["pod_template"] = self.pod_template
82
+ if self.description is not None:
83
+ kwargs["description"] = self.description
84
+ return kwargs
flyte/_excepthook.py ADDED
@@ -0,0 +1,37 @@
1
+ import logging
2
+ import sys
3
+ import traceback
4
+
5
+ from flyte._logging import logger
6
+
7
+ # Save the original excepthook so we can call it later
8
+ original_excepthook = sys.excepthook
9
+
10
+ # Filters: exclude frames where filename or function name contains these substrings
11
+ EXCLUDED_MODULE_SUBSTRINGS = ["_internal", "syncify"]
12
+ EXCLUDED_FILE_SUBSTRINGS = ["syncify", "_code_bundle"]
13
+
14
+
15
+ def should_include_frame(frame: traceback.FrameSummary) -> bool:
16
+ return not (
17
+ any(sub in frame.name for sub in EXCLUDED_MODULE_SUBSTRINGS)
18
+ or any(sub in frame.filename for sub in EXCLUDED_FILE_SUBSTRINGS)
19
+ )
20
+
21
+
22
+ def custom_excepthook(exc_type, exc_value, exc_tb):
23
+ """
24
+ Custom exception hook to filter and format tracebacks.
25
+ If the logger's level is DEBUG, it uses the original excepthook.
26
+ """
27
+
28
+ if logger.getEffectiveLevel() <= logging.DEBUG:
29
+ original_excepthook(exc_type, exc_value, exc_tb)
30
+ else:
31
+ # Extract and filter traceback
32
+ tb_list = traceback.extract_tb(exc_tb)
33
+ filtered_tb = [frame for frame in tb_list if should_include_frame(frame)]
34
+ # Print the filtered version (custom format)
35
+ print("Filtered traceback (most recent call last):")
36
+ print("".join(traceback.format_list(filtered_tb)))
37
+ print(f"{exc_type.__name__}: {exc_value}\n")
flyte/_group.py CHANGED
@@ -1,7 +1,7 @@
1
1
  from contextlib import contextmanager
2
2
 
3
3
  from ._context import internal_ctx
4
- from ._datastructures import GroupData
4
+ from .models import GroupData
5
5
 
6
6
 
7
7
  @contextmanager
@@ -29,3 +29,4 @@ def group(name: str):
29
29
  new_tctx = tctx.replace(group_data=GroupData(name))
30
30
  with ctx.replace_task_context(new_tctx):
31
31
  yield
32
+ # Exit the context and restore the previous context
flyte/_image.py CHANGED
@@ -342,8 +342,7 @@ class Image:
342
342
  """Internal hacky function to see if the current install is editable or not."""
343
343
  curr = Path(__file__)
344
344
  pyproject = curr.parent.parent.parent / "pyproject.toml"
345
- dist_folder = curr.parent.parent.parent / "dist"
346
- return pyproject.exists() and dist_folder
345
+ return pyproject.exists()
347
346
 
348
347
  @classmethod
349
348
  def from_uv_debian(
@@ -445,8 +444,7 @@ class Image:
445
444
  ```
446
445
 
447
446
  For more information on the uv script format, see the documentation:
448
- <href="https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies">
449
- UV: Declaring script dependencies</href>
447
+ [UV: Declaring script dependencies](https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies)
450
448
 
451
449
  :param name: name of the image
452
450
  :param registry: registry to use for the image
@@ -468,9 +466,15 @@ class Image:
468
466
  header = parse_uv_script_file(script)
469
467
  if registry is None:
470
468
  raise ValueError("registry must be specified")
469
+
471
470
  img = cls.from_uv_debian(registry=registry, name=name, arch=arch, python_version=python_version)
471
+
472
+ # add ca-certificates to the image by default
473
+ img = img.with_apt_packages(["ca-certificates"])
474
+
472
475
  if header.dependencies:
473
476
  return img.with_pip_packages(header.dependencies)
477
+
474
478
  # todo: override the _identifier_override to be the script name or a hash of the script contents
475
479
  # This is needed because inside the image, the identifier will be computed to be something different.
476
480
  return img