flyte 2.0.0b13__py3-none-any.whl → 2.0.0b30__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. flyte/__init__.py +18 -2
  2. flyte/_bin/debug.py +38 -0
  3. flyte/_bin/runtime.py +62 -8
  4. flyte/_cache/cache.py +4 -2
  5. flyte/_cache/local_cache.py +216 -0
  6. flyte/_code_bundle/_ignore.py +12 -4
  7. flyte/_code_bundle/_packaging.py +13 -9
  8. flyte/_code_bundle/_utils.py +18 -10
  9. flyte/_code_bundle/bundle.py +17 -9
  10. flyte/_constants.py +1 -0
  11. flyte/_context.py +4 -1
  12. flyte/_custom_context.py +73 -0
  13. flyte/_debug/constants.py +38 -0
  14. flyte/_debug/utils.py +17 -0
  15. flyte/_debug/vscode.py +307 -0
  16. flyte/_deploy.py +235 -61
  17. flyte/_environment.py +20 -6
  18. flyte/_excepthook.py +1 -1
  19. flyte/_hash.py +1 -16
  20. flyte/_image.py +178 -81
  21. flyte/_initialize.py +132 -51
  22. flyte/_interface.py +39 -2
  23. flyte/_internal/controllers/__init__.py +4 -5
  24. flyte/_internal/controllers/_local_controller.py +70 -29
  25. flyte/_internal/controllers/_trace.py +1 -1
  26. flyte/_internal/controllers/remote/__init__.py +0 -2
  27. flyte/_internal/controllers/remote/_action.py +14 -16
  28. flyte/_internal/controllers/remote/_client.py +1 -1
  29. flyte/_internal/controllers/remote/_controller.py +68 -70
  30. flyte/_internal/controllers/remote/_core.py +127 -99
  31. flyte/_internal/controllers/remote/_informer.py +19 -10
  32. flyte/_internal/controllers/remote/_service_protocol.py +7 -7
  33. flyte/_internal/imagebuild/docker_builder.py +181 -69
  34. flyte/_internal/imagebuild/image_builder.py +0 -5
  35. flyte/_internal/imagebuild/remote_builder.py +155 -64
  36. flyte/_internal/imagebuild/utils.py +51 -2
  37. flyte/_internal/resolvers/_task_module.py +5 -38
  38. flyte/_internal/resolvers/default.py +2 -2
  39. flyte/_internal/runtime/convert.py +110 -21
  40. flyte/_internal/runtime/entrypoints.py +27 -1
  41. flyte/_internal/runtime/io.py +21 -8
  42. flyte/_internal/runtime/resources_serde.py +20 -6
  43. flyte/_internal/runtime/reuse.py +1 -1
  44. flyte/_internal/runtime/rusty.py +20 -5
  45. flyte/_internal/runtime/task_serde.py +34 -19
  46. flyte/_internal/runtime/taskrunner.py +22 -4
  47. flyte/_internal/runtime/trigger_serde.py +160 -0
  48. flyte/_internal/runtime/types_serde.py +1 -1
  49. flyte/_keyring/__init__.py +0 -0
  50. flyte/_keyring/file.py +115 -0
  51. flyte/_logging.py +201 -39
  52. flyte/_map.py +111 -14
  53. flyte/_module.py +70 -0
  54. flyte/_pod.py +4 -3
  55. flyte/_resources.py +213 -31
  56. flyte/_run.py +110 -39
  57. flyte/_task.py +75 -16
  58. flyte/_task_environment.py +105 -29
  59. flyte/_task_plugins.py +4 -2
  60. flyte/_trace.py +5 -0
  61. flyte/_trigger.py +1000 -0
  62. flyte/_utils/__init__.py +2 -1
  63. flyte/_utils/asyn.py +3 -1
  64. flyte/_utils/coro_management.py +2 -1
  65. flyte/_utils/docker_credentials.py +173 -0
  66. flyte/_utils/module_loader.py +17 -2
  67. flyte/_version.py +3 -3
  68. flyte/cli/_abort.py +3 -3
  69. flyte/cli/_build.py +3 -6
  70. flyte/cli/_common.py +78 -7
  71. flyte/cli/_create.py +182 -4
  72. flyte/cli/_delete.py +23 -1
  73. flyte/cli/_deploy.py +63 -16
  74. flyte/cli/_get.py +79 -34
  75. flyte/cli/_params.py +26 -10
  76. flyte/cli/_plugins.py +209 -0
  77. flyte/cli/_run.py +151 -26
  78. flyte/cli/_serve.py +64 -0
  79. flyte/cli/_update.py +37 -0
  80. flyte/cli/_user.py +17 -0
  81. flyte/cli/main.py +30 -4
  82. flyte/config/_config.py +10 -6
  83. flyte/config/_internal.py +1 -0
  84. flyte/config/_reader.py +29 -8
  85. flyte/connectors/__init__.py +11 -0
  86. flyte/connectors/_connector.py +270 -0
  87. flyte/connectors/_server.py +197 -0
  88. flyte/connectors/utils.py +135 -0
  89. flyte/errors.py +22 -2
  90. flyte/extend.py +8 -1
  91. flyte/extras/_container.py +6 -1
  92. flyte/git/__init__.py +3 -0
  93. flyte/git/_config.py +21 -0
  94. flyte/io/__init__.py +2 -0
  95. flyte/io/_dataframe/__init__.py +2 -0
  96. flyte/io/_dataframe/basic_dfs.py +17 -8
  97. flyte/io/_dataframe/dataframe.py +98 -132
  98. flyte/io/_dir.py +575 -113
  99. flyte/io/_file.py +582 -139
  100. flyte/io/_hashing_io.py +342 -0
  101. flyte/models.py +74 -15
  102. flyte/remote/__init__.py +6 -1
  103. flyte/remote/_action.py +34 -26
  104. flyte/remote/_client/_protocols.py +39 -4
  105. flyte/remote/_client/auth/_authenticators/device_code.py +4 -5
  106. flyte/remote/_client/auth/_authenticators/pkce.py +1 -1
  107. flyte/remote/_client/auth/_channel.py +10 -6
  108. flyte/remote/_client/controlplane.py +17 -5
  109. flyte/remote/_console.py +3 -2
  110. flyte/remote/_data.py +6 -6
  111. flyte/remote/_logs.py +3 -3
  112. flyte/remote/_run.py +64 -8
  113. flyte/remote/_secret.py +26 -17
  114. flyte/remote/_task.py +75 -33
  115. flyte/remote/_trigger.py +306 -0
  116. flyte/remote/_user.py +33 -0
  117. flyte/report/_report.py +1 -1
  118. flyte/storage/__init__.py +6 -1
  119. flyte/storage/_config.py +5 -1
  120. flyte/storage/_parallel_reader.py +274 -0
  121. flyte/storage/_storage.py +200 -103
  122. flyte/types/__init__.py +16 -0
  123. flyte/types/_interface.py +2 -2
  124. flyte/types/_pickle.py +35 -8
  125. flyte/types/_string_literals.py +8 -9
  126. flyte/types/_type_engine.py +40 -70
  127. flyte/types/_utils.py +1 -1
  128. flyte-2.0.0b30.data/scripts/debug.py +38 -0
  129. {flyte-2.0.0b13.data → flyte-2.0.0b30.data}/scripts/runtime.py +62 -8
  130. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/METADATA +11 -3
  131. flyte-2.0.0b30.dist-info/RECORD +192 -0
  132. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/entry_points.txt +3 -0
  133. flyte/_protos/common/authorization_pb2.py +0 -66
  134. flyte/_protos/common/authorization_pb2.pyi +0 -108
  135. flyte/_protos/common/authorization_pb2_grpc.py +0 -4
  136. flyte/_protos/common/identifier_pb2.py +0 -93
  137. flyte/_protos/common/identifier_pb2.pyi +0 -110
  138. flyte/_protos/common/identifier_pb2_grpc.py +0 -4
  139. flyte/_protos/common/identity_pb2.py +0 -48
  140. flyte/_protos/common/identity_pb2.pyi +0 -72
  141. flyte/_protos/common/identity_pb2_grpc.py +0 -4
  142. flyte/_protos/common/list_pb2.py +0 -36
  143. flyte/_protos/common/list_pb2.pyi +0 -71
  144. flyte/_protos/common/list_pb2_grpc.py +0 -4
  145. flyte/_protos/common/policy_pb2.py +0 -37
  146. flyte/_protos/common/policy_pb2.pyi +0 -27
  147. flyte/_protos/common/policy_pb2_grpc.py +0 -4
  148. flyte/_protos/common/role_pb2.py +0 -37
  149. flyte/_protos/common/role_pb2.pyi +0 -53
  150. flyte/_protos/common/role_pb2_grpc.py +0 -4
  151. flyte/_protos/common/runtime_version_pb2.py +0 -28
  152. flyte/_protos/common/runtime_version_pb2.pyi +0 -24
  153. flyte/_protos/common/runtime_version_pb2_grpc.py +0 -4
  154. flyte/_protos/imagebuilder/definition_pb2.py +0 -59
  155. flyte/_protos/imagebuilder/definition_pb2.pyi +0 -140
  156. flyte/_protos/imagebuilder/definition_pb2_grpc.py +0 -4
  157. flyte/_protos/imagebuilder/payload_pb2.py +0 -32
  158. flyte/_protos/imagebuilder/payload_pb2.pyi +0 -21
  159. flyte/_protos/imagebuilder/payload_pb2_grpc.py +0 -4
  160. flyte/_protos/imagebuilder/service_pb2.py +0 -29
  161. flyte/_protos/imagebuilder/service_pb2.pyi +0 -5
  162. flyte/_protos/imagebuilder/service_pb2_grpc.py +0 -66
  163. flyte/_protos/logs/dataplane/payload_pb2.py +0 -100
  164. flyte/_protos/logs/dataplane/payload_pb2.pyi +0 -177
  165. flyte/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
  166. flyte/_protos/secret/definition_pb2.py +0 -49
  167. flyte/_protos/secret/definition_pb2.pyi +0 -93
  168. flyte/_protos/secret/definition_pb2_grpc.py +0 -4
  169. flyte/_protos/secret/payload_pb2.py +0 -62
  170. flyte/_protos/secret/payload_pb2.pyi +0 -94
  171. flyte/_protos/secret/payload_pb2_grpc.py +0 -4
  172. flyte/_protos/secret/secret_pb2.py +0 -38
  173. flyte/_protos/secret/secret_pb2.pyi +0 -6
  174. flyte/_protos/secret/secret_pb2_grpc.py +0 -198
  175. flyte/_protos/secret/secret_pb2_grpc_grpc.py +0 -198
  176. flyte/_protos/validate/validate/validate_pb2.py +0 -76
  177. flyte/_protos/workflow/common_pb2.py +0 -27
  178. flyte/_protos/workflow/common_pb2.pyi +0 -14
  179. flyte/_protos/workflow/common_pb2_grpc.py +0 -4
  180. flyte/_protos/workflow/environment_pb2.py +0 -29
  181. flyte/_protos/workflow/environment_pb2.pyi +0 -12
  182. flyte/_protos/workflow/environment_pb2_grpc.py +0 -4
  183. flyte/_protos/workflow/node_execution_service_pb2.py +0 -26
  184. flyte/_protos/workflow/node_execution_service_pb2.pyi +0 -4
  185. flyte/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
  186. flyte/_protos/workflow/queue_service_pb2.py +0 -109
  187. flyte/_protos/workflow/queue_service_pb2.pyi +0 -166
  188. flyte/_protos/workflow/queue_service_pb2_grpc.py +0 -172
  189. flyte/_protos/workflow/run_definition_pb2.py +0 -121
  190. flyte/_protos/workflow/run_definition_pb2.pyi +0 -327
  191. flyte/_protos/workflow/run_definition_pb2_grpc.py +0 -4
  192. flyte/_protos/workflow/run_logs_service_pb2.py +0 -41
  193. flyte/_protos/workflow/run_logs_service_pb2.pyi +0 -28
  194. flyte/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
  195. flyte/_protos/workflow/run_service_pb2.py +0 -137
  196. flyte/_protos/workflow/run_service_pb2.pyi +0 -185
  197. flyte/_protos/workflow/run_service_pb2_grpc.py +0 -446
  198. flyte/_protos/workflow/state_service_pb2.py +0 -67
  199. flyte/_protos/workflow/state_service_pb2.pyi +0 -76
  200. flyte/_protos/workflow/state_service_pb2_grpc.py +0 -138
  201. flyte/_protos/workflow/task_definition_pb2.py +0 -79
  202. flyte/_protos/workflow/task_definition_pb2.pyi +0 -81
  203. flyte/_protos/workflow/task_definition_pb2_grpc.py +0 -4
  204. flyte/_protos/workflow/task_service_pb2.py +0 -60
  205. flyte/_protos/workflow/task_service_pb2.pyi +0 -59
  206. flyte/_protos/workflow/task_service_pb2_grpc.py +0 -138
  207. flyte-2.0.0b13.dist-info/RECORD +0 -239
  208. /flyte/{_protos → _debug}/__init__.py +0 -0
  209. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/WHEEL +0 -0
  210. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/licenses/LICENSE +0 -0
  211. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/top_level.txt +0 -0
flyte/types/__init__.py CHANGED
@@ -19,6 +19,10 @@ It is always possible to bypass the type system and use the `FlytePickle` type t
19
19
  written in python. The Pickled objects cannot be represented in the UI, and may be in-efficient for large datasets.
20
20
  """
21
21
 
22
+ from importlib.metadata import entry_points
23
+
24
+ from flyte._logging import logger
25
+
22
26
  from ._interface import guess_interface
23
27
  from ._pickle import FlytePickle
24
28
  from ._renderer import Renderable
@@ -34,3 +38,15 @@ __all__ = [
34
38
  "guess_interface",
35
39
  "literal_string_repr",
36
40
  ]
41
+
42
+
43
+ def _load_custom_type_transformers():
44
+ plugins = entry_points(group="flyte.plugins.types")
45
+ for ep in plugins:
46
+ try:
47
+ logger.info(f"Loading type transformer: {ep.name}")
48
+ loaded = ep.load()
49
+ if callable(loaded):
50
+ loaded()
51
+ except Exception as e:
52
+ logger.warning(f"Failed to load type transformer {ep.name} with error: {e}")
flyte/types/_interface.py CHANGED
@@ -1,9 +1,9 @@
1
1
  import inspect
2
2
  from typing import Any, Dict, Iterable, Tuple, Type, cast
3
3
 
4
- from flyteidl.core import interface_pb2, literals_pb2
4
+ from flyteidl2.core import interface_pb2, literals_pb2
5
+ from flyteidl2.task import common_pb2
5
6
 
6
- from flyte._protos.workflow import common_pb2
7
7
  from flyte.models import NativeInterface
8
8
 
9
9
 
flyte/types/_pickle.py CHANGED
@@ -1,17 +1,19 @@
1
1
  import hashlib
2
2
  import os
3
+ import sys
3
4
  import typing
4
5
  from typing import Type
5
6
 
6
7
  import aiofiles
7
8
  import cloudpickle
8
- from flyteidl.core import literals_pb2, types_pb2
9
+ from flyteidl2.core import literals_pb2, types_pb2
9
10
 
10
11
  import flyte.storage as storage
11
12
 
12
13
  from ._type_engine import TypeEngine, TypeTransformer
13
14
 
14
15
  T = typing.TypeVar("T")
16
+ DEFAULT_PICKLE_BYTES_LIMIT = 2**10 * 10 # 10KB
15
17
 
16
18
 
17
19
  class FlytePickle(typing.Generic[T]):
@@ -76,8 +78,13 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
76
78
  ...
77
79
 
78
80
  async def to_python_value(self, lv: literals_pb2.Literal, expected_python_type: Type[T]) -> T:
79
- uri = lv.scalar.blob.uri
80
- return await FlytePickle.from_pickle(uri)
81
+ if lv.scalar.blob and lv.scalar.blob.uri:
82
+ uri = lv.scalar.blob.uri
83
+ return await FlytePickle.from_pickle(uri)
84
+ elif lv.scalar.binary and lv.scalar.binary.value:
85
+ return cloudpickle.loads(lv.scalar.binary.value)
86
+ else:
87
+ raise ValueError(f"Cannot convert {lv} to {expected_python_type}")
81
88
 
82
89
  async def to_literal(
83
90
  self,
@@ -92,8 +99,15 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
92
99
  format=self.PYTHON_PICKLE_FORMAT, dimensionality=types_pb2.BlobType.BlobDimensionality.SINGLE
93
100
  )
94
101
  )
95
- remote_path = await FlytePickle.to_pickle(python_val)
96
- return literals_pb2.Literal(scalar=literals_pb2.Scalar(blob=literals_pb2.Blob(metadata=meta, uri=remote_path)))
102
+ if sys.getsizeof(python_val) > DEFAULT_PICKLE_BYTES_LIMIT:
103
+ remote_path = await FlytePickle.to_pickle(python_val)
104
+ return literals_pb2.Literal(
105
+ scalar=literals_pb2.Scalar(blob=literals_pb2.Blob(metadata=meta, uri=remote_path))
106
+ )
107
+ else:
108
+ return literals_pb2.Literal(
109
+ scalar=literals_pb2.Scalar(binary=literals_pb2.Binary(value=cloudpickle.dumps(python_val)))
110
+ )
97
111
 
98
112
  def guess_python_type(self, literal_type: types_pb2.LiteralType) -> typing.Type[FlytePickle[typing.Any]]:
99
113
  if (
@@ -102,14 +116,27 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
102
116
  and literal_type.blob.format == FlytePickleTransformer.PYTHON_PICKLE_FORMAT
103
117
  ):
104
118
  return FlytePickle
119
+ if literal_type.simple == types_pb2.SimpleType.BINARY:
120
+ return FlytePickle
105
121
 
106
122
  raise ValueError(f"Transformer {self} cannot reverse {literal_type}")
107
123
 
108
124
  def get_literal_type(self, t: Type[T]) -> types_pb2.LiteralType:
109
125
  lt = types_pb2.LiteralType(
110
- blob=types_pb2.BlobType(
111
- format=self.PYTHON_PICKLE_FORMAT, dimensionality=types_pb2.BlobType.BlobDimensionality.SINGLE
112
- )
126
+ union_type=types_pb2.UnionType(
127
+ variants=[
128
+ types_pb2.LiteralType(
129
+ blob=types_pb2.BlobType(
130
+ format=self.PYTHON_PICKLE_FORMAT,
131
+ dimensionality=types_pb2.BlobType.BlobDimensionality.SINGLE,
132
+ ),
133
+ structure=types_pb2.TypeStructure(tag=self.name),
134
+ ),
135
+ types_pb2.LiteralType(
136
+ simple=types_pb2.SimpleType.BINARY, structure=types_pb2.TypeStructure(tag=self.name)
137
+ ),
138
+ ]
139
+ ),
113
140
  )
114
141
  lt.metadata = {"python_class_name": str(t)}
115
142
  return lt
@@ -3,11 +3,10 @@ import json
3
3
  from typing import Any, Dict, Union
4
4
 
5
5
  import msgpack
6
- from flyteidl.core import literals_pb2
6
+ from flyteidl2.core import literals_pb2
7
+ from flyteidl2.task import common_pb2
7
8
  from google.protobuf.json_format import MessageToDict
8
9
 
9
- from flyte._protos.workflow import run_definition_pb2
10
-
11
10
 
12
11
  def _primitive_to_string(primitive: literals_pb2.Primitive) -> Any:
13
12
  """
@@ -88,9 +87,9 @@ def _dict_literal_repr(lmd: Dict[str, literals_pb2.Literal]) -> Dict[str, Any]:
88
87
  def literal_string_repr(
89
88
  lm: Union[
90
89
  literals_pb2.Literal,
91
- run_definition_pb2.NamedLiteral,
92
- run_definition_pb2.Inputs,
93
- run_definition_pb2.Outputs,
90
+ common_pb2.NamedLiteral,
91
+ common_pb2.Inputs,
92
+ common_pb2.Outputs,
94
93
  literals_pb2.LiteralMap,
95
94
  Dict[str, literals_pb2.Literal],
96
95
  ],
@@ -105,13 +104,13 @@ def literal_string_repr(
105
104
  return _literal_string_repr(lm)
106
105
  case literals_pb2.LiteralMap():
107
106
  return _dict_literal_repr(lm.literals)
108
- case run_definition_pb2.NamedLiteral():
107
+ case common_pb2.NamedLiteral():
109
108
  lmd = {lm.name: lm.value}
110
109
  return _dict_literal_repr(lmd)
111
- case run_definition_pb2.Inputs():
110
+ case common_pb2.Inputs():
112
111
  lmd = {n.name: n.value for n in lm.literals}
113
112
  return _dict_literal_repr(lmd)
114
- case run_definition_pb2.Outputs():
113
+ case common_pb2.Outputs():
115
114
  lmd = {n.name: n.value for n in lm.literals}
116
115
  return _dict_literal_repr(lmd)
117
116
  case dict():
@@ -20,9 +20,9 @@ from types import GenericAlias, NoneType
20
20
  from typing import Any, Dict, NamedTuple, Optional, Type, cast
21
21
 
22
22
  import msgpack
23
- from flyteidl.core import interface_pb2, literals_pb2, types_pb2
24
- from flyteidl.core.literals_pb2 import Binary, Literal, LiteralCollection, LiteralMap, Primitive, Scalar, Union, Void
25
- from flyteidl.core.types_pb2 import LiteralType, SimpleType, TypeAnnotation, TypeStructure, UnionType
23
+ from flyteidl2.core import interface_pb2, literals_pb2, types_pb2
24
+ from flyteidl2.core.literals_pb2 import Binary, Literal, LiteralCollection, LiteralMap, Primitive, Scalar, Union, Void
25
+ from flyteidl2.core.types_pb2 import LiteralType, SimpleType, TypeAnnotation, TypeStructure, UnionType
26
26
  from fsspec.asyn import _run_coros_in_chunks # pylint: disable=W0212
27
27
  from google.protobuf import json_format as _json_format
28
28
  from google.protobuf import struct_pb2
@@ -39,7 +39,6 @@ from pydantic import BaseModel
39
39
  from typing_extensions import Annotated, get_args, get_origin
40
40
 
41
41
  import flyte.storage as storage
42
- from flyte._hash import HashMethod
43
42
  from flyte._logging import logger
44
43
  from flyte._utils.helpers import load_proto_from_file
45
44
  from flyte.models import NativeInterface
@@ -456,35 +455,6 @@ class DataclassTransformer(TypeTransformer[object]):
456
455
  2. Deserialization: The dataclass transformer converts the MessagePack Bytes back to a dataclass.
457
456
  (1) Convert MessagePack Bytes to a dataclass using mashumaro.
458
457
  (2) Handle dataclass attributes to ensure they are of the correct types.
459
-
460
- TODO: Update the example using mashumaro instead of the older library
461
-
462
- Example
463
-
464
- .. code-block:: python
465
-
466
- @dataclass
467
- class Test:
468
- a: int
469
- b: str
470
-
471
- t = Test(a=10,b="e")
472
- JSONSchema().dump(t.schema())
473
-
474
- Output will look like
475
-
476
- .. code-block:: json
477
-
478
- {'$schema': 'http://json-schema.org/draft-07/schema#',
479
- 'definitions': {'TestSchema': {'properties': {'a': {'title': 'a',
480
- 'type': 'number',
481
- 'format': 'integer'},
482
- 'b': {'title': 'b', 'type': 'string'}},
483
- 'type': 'object',
484
- 'additionalProperties': False}},
485
- '$ref': '#/definitions/TestSchema'}
486
-
487
-
488
458
  """
489
459
 
490
460
  def __init__(self) -> None:
@@ -616,7 +586,7 @@ class DataclassTransformer(TypeTransformer[object]):
616
586
  }
617
587
  )
618
588
 
619
- # The type engine used to publish a type structure for attribute access. As of v2, this is no longer needed.
589
+ # The type engine used to publish the type `structure` for attribute access. As of v2, this is no longer needed.
620
590
  return types_pb2.LiteralType(
621
591
  simple=types_pb2.SimpleType.STRUCT,
622
592
  metadata=schema,
@@ -816,6 +786,13 @@ class EnumTransformer(TypeTransformer[enum.Enum]):
816
786
  return LiteralType(enum_type=types_pb2.EnumType(values=values))
817
787
 
818
788
  async def to_literal(self, python_val: enum.Enum, python_type: Type[T], expected: LiteralType) -> Literal:
789
+ if isinstance(python_val, str):
790
+ # this is the case when python Literals are used as enums
791
+ if python_val not in expected.enum_type.values:
792
+ raise TypeTransformerFailedError(
793
+ f"Value {python_val} is not valid value, expected - {expected.enum_type.values}"
794
+ )
795
+ return Literal(scalar=Scalar(primitive=Primitive(string_value=python_val))) # type: ignore
819
796
  if type(python_val).__class__ != enum.EnumMeta:
820
797
  raise TypeTransformerFailedError("Expected an enum")
821
798
  if type(python_val.value) is not str:
@@ -826,6 +803,12 @@ class EnumTransformer(TypeTransformer[enum.Enum]):
826
803
  async def to_python_value(self, lv: Literal, expected_python_type: Type[T]) -> T:
827
804
  if lv.HasField("scalar") and lv.scalar.HasField("binary"):
828
805
  return self.from_binary_idl(lv.scalar.binary, expected_python_type) # type: ignore
806
+ from flyte._interface import LITERAL_ENUM
807
+
808
+ if expected_python_type.__name__ is LITERAL_ENUM:
809
+ # This is the case when python Literal types are used as enums. The class name is always LiteralEnum an
810
+ # hardcoded in flyte.models
811
+ return lv.scalar.primitive.string_value
829
812
  return expected_python_type(lv.scalar.primitive.string_value) # type: ignore
830
813
 
831
814
  def guess_python_type(self, literal_type: LiteralType) -> Type[enum.Enum]:
@@ -1096,23 +1079,6 @@ class TypeEngine(typing.Generic[T]):
1096
1079
  ):
1097
1080
  raise TypeTransformerFailedError(f"Python value cannot be None, expected {python_type}/{expected}")
1098
1081
 
1099
- @classmethod
1100
- def calculate_hash(cls, python_val: typing.Any, python_type: Type[T]) -> Optional[str]:
1101
- # In case the value is an annotated type we inspect the annotations and look for hash-related annotations.
1102
- hsh = None
1103
- if is_annotated(python_type):
1104
- # We are now dealing with one of two cases:
1105
- # 1. The annotated type is a `HashMethod`, which indicates that we should produce the hash using
1106
- # the method indicated in the annotation.
1107
- # 2. The annotated type is being used for a different purpose other than calculating hash values,
1108
- # in which case we should just continue.
1109
- for annotation in get_args(python_type)[1:]:
1110
- if not isinstance(annotation, HashMethod):
1111
- continue
1112
- hsh = annotation.calculate(python_val)
1113
- break
1114
- return hsh
1115
-
1116
1082
  @classmethod
1117
1083
  async def to_literal(
1118
1084
  cls, python_val: typing.Any, python_type: Type[T], expected: types_pb2.LiteralType
@@ -1125,8 +1091,6 @@ class TypeEngine(typing.Generic[T]):
1125
1091
  lv = await transformer.to_literal(python_val, python_type, expected)
1126
1092
 
1127
1093
  modify_literal_uris(lv)
1128
- calculated_hash = cls.calculate_hash(python_val, python_type) or ""
1129
- lv.hash = calculated_hash
1130
1094
  return lv
1131
1095
 
1132
1096
  @classmethod
@@ -1204,20 +1168,26 @@ class TypeEngine(typing.Generic[T]):
1204
1168
  f"Received more input values {len(lm.literals)}"
1205
1169
  f" than allowed by the input spec {len(python_interface_inputs)}"
1206
1170
  )
1171
+ # Create tasks for converting each kwarg
1172
+ tasks = {}
1173
+ for k in lm.literals:
1174
+ tasks[k] = asyncio.create_task(TypeEngine.to_python_value(lm.literals[k], python_interface_inputs[k]))
1175
+
1176
+ # Gather all tasks, returning exceptions instead of raising them
1177
+ results = await asyncio.gather(*tasks.values(), return_exceptions=True)
1178
+
1179
+ # Check for exceptions and raise with specific kwarg name
1207
1180
  kwargs = {}
1208
- try:
1209
- for i, k in enumerate(lm.literals):
1210
- kwargs[k] = asyncio.create_task(TypeEngine.to_python_value(lm.literals[k], python_interface_inputs[k]))
1211
- await asyncio.gather(*kwargs.values())
1212
- except Exception as e:
1213
- raise TypeTransformerFailedError(
1214
- f"Error converting input:\n"
1215
- f"Literal value: {lm.literals[k]}\n"
1216
- f"Expected Python type: {python_interface_inputs[k]}\n"
1217
- f"Exception: {e}"
1218
- )
1181
+ for (key, task), result in zip(tasks.items(), results):
1182
+ if isinstance(result, Exception):
1183
+ raise TypeTransformerFailedError(
1184
+ f"Error converting input '{key}':\n"
1185
+ f"Literal value: {lm.literals[key]}\n"
1186
+ f"Expected Python type: {python_interface_inputs[key]}\n"
1187
+ f"Exception: {result}"
1188
+ ) from result
1189
+ kwargs[key] = result
1219
1190
 
1220
- kwargs = {k: v.result() for k, v in kwargs.items() if v is not None}
1221
1191
  return kwargs
1222
1192
 
1223
1193
  @classmethod
@@ -1252,7 +1222,7 @@ class TypeEngine(typing.Generic[T]):
1252
1222
  e: BaseException = literal_map[k].exception() # type: ignore
1253
1223
  if isinstance(e, TypeError):
1254
1224
  raise TypeError(
1255
- f"Error converting: Var:{k}, type:{type(v)}, into:{python_type}, received_value {v}"
1225
+ f"Error converting: Var:{k}, type:{type(d[k])}, into:{python_type}, received_value {d[k]}"
1256
1226
  )
1257
1227
  else:
1258
1228
  raise e
@@ -1795,7 +1765,6 @@ class DictTransformer(TypeTransformer[dict]):
1795
1765
  for k, v in python_val.items():
1796
1766
  if type(k) is not str:
1797
1767
  raise ValueError("Flyte MapType expects all keys to be strings")
1798
- # TODO: log a warning for Annotated objects that contain HashMethod
1799
1768
 
1800
1769
  _, v_type = self.extract_types(python_type)
1801
1770
  lit_map[k] = TypeEngine.to_literal(v, cast(type, v_type), expected.map_value_type)
@@ -1942,7 +1911,6 @@ def _get_element_type(element_property: typing.Dict[str, str]) -> Type:
1942
1911
  return str
1943
1912
 
1944
1913
 
1945
- # pr: han-ru is this still needed?
1946
1914
  def dataclass_from_dict(cls: type, src: typing.Dict[str, typing.Any]) -> typing.Any:
1947
1915
  """
1948
1916
  Utility function to construct a dataclass object from dict
@@ -2022,7 +1990,7 @@ def _handle_flyte_console_float_input_to_int(lv: Literal) -> int:
2022
1990
 
2023
1991
  def _check_and_convert_void(lv: Literal) -> None:
2024
1992
  if not lv.scalar.HasField("none_type"):
2025
- raise TypeTransformerFailedError(f"Cannot convert literal {lv} to None")
1993
+ raise TypeTransformerFailedError(f"Cannot convert literal '{lv}' to None")
2026
1994
  return None
2027
1995
 
2028
1996
 
@@ -2083,7 +2051,9 @@ DateTransformer = SimpleTransformer(
2083
2051
  lambda x: Literal(
2084
2052
  scalar=Scalar(primitive=Primitive(datetime=datetime.datetime.combine(x, datetime.time.min)))
2085
2053
  ), # convert datetime to date
2086
- lambda x: x.scalar.primitive.datetime.date() if x.scalar.primitive.HasField("datetime") else None,
2054
+ lambda x: x.scalar.primitive.datetime.ToDatetime().replace(tzinfo=datetime.timezone.utc).date()
2055
+ if x.scalar.primitive.HasField("datetime")
2056
+ else None,
2087
2057
  )
2088
2058
 
2089
2059
  NoneTransformer = SimpleTransformer(
flyte/types/_utils.py CHANGED
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  import importlib
4
4
  import typing
5
5
 
6
- from flyteidl.core.types_pb2 import EnumType, LiteralType, UnionType
6
+ from flyteidl2.core.types_pb2 import EnumType, LiteralType, UnionType
7
7
 
8
8
  T = typing.TypeVar("T")
9
9
 
@@ -0,0 +1,38 @@
1
+ import click
2
+
3
+
4
+ @click.group()
5
+ def _debug():
6
+ """Debug commands for Flyte."""
7
+
8
+
9
+ @_debug.command("resume")
10
+ @click.option("--pid", "-m", type=int, required=True, help="PID of the vscode server.")
11
+ def resume(pid):
12
+ """
13
+ Resume a Flyte task for debugging purposes.
14
+
15
+ Args:
16
+ pid (int): PID of the vscode server.
17
+ """
18
+ import os
19
+ import signal
20
+
21
+ print("Terminating server and resuming task.")
22
+ answer = (
23
+ input(
24
+ "This operation will kill the server. All unsaved data will be lost,"
25
+ " and you will no longer be able to connect to it. Do you really want to terminate? (Y/N): "
26
+ )
27
+ .strip()
28
+ .upper()
29
+ )
30
+ if answer == "Y":
31
+ os.kill(pid, signal.SIGTERM)
32
+ print("The server has been terminated and the task has been resumed.")
33
+ else:
34
+ print("Operation canceled.")
35
+
36
+
37
+ if __name__ == "__main__":
38
+ _debug()
@@ -12,6 +12,9 @@ from typing import Any, List
12
12
 
13
13
  import click
14
14
 
15
+ from flyte._utils.helpers import str2bool
16
+ from flyte.models import PathRewrite
17
+
15
18
  # Todo: work with pvditt to make these the names
16
19
  # ACTION_NAME = "_U_ACTION_NAME"
17
20
  # RUN_NAME = "_U_RUN_NAME"
@@ -21,14 +24,16 @@ import click
21
24
 
22
25
  ACTION_NAME = "ACTION_NAME"
23
26
  RUN_NAME = "RUN_NAME"
24
- PROJECT_NAME = "FLYTE_INTERNAL_TASK_PROJECT"
25
- DOMAIN_NAME = "FLYTE_INTERNAL_TASK_DOMAIN"
27
+ PROJECT_NAME = "FLYTE_INTERNAL_EXECUTION_PROJECT"
28
+ DOMAIN_NAME = "FLYTE_INTERNAL_EXECUTION_DOMAIN"
26
29
  ORG_NAME = "_U_ORG_NAME"
27
30
  ENDPOINT_OVERRIDE = "_U_EP_OVERRIDE"
31
+ INSECURE_SKIP_VERIFY_OVERRIDE = "_U_INSECURE_SKIP_VERIFY"
28
32
  RUN_OUTPUT_BASE_DIR = "_U_RUN_BASE"
33
+ FLYTE_ENABLE_VSCODE_KEY = "_F_E_VS"
29
34
 
30
- # TODO: Remove this after proper auth is implemented
31
35
  _UNION_EAGER_API_KEY_ENV_VAR = "_UNION_EAGER_API_KEY"
36
+ _F_PATH_REWRITE = "_F_PATH_REWRITE"
32
37
 
33
38
 
34
39
  @click.group()
@@ -49,6 +54,8 @@ def _pass_through():
49
54
  @click.option("--project", envvar=PROJECT_NAME, required=False)
50
55
  @click.option("--domain", envvar=DOMAIN_NAME, required=False)
51
56
  @click.option("--org", envvar=ORG_NAME, required=False)
57
+ @click.option("--debug", envvar=FLYTE_ENABLE_VSCODE_KEY, type=click.BOOL, required=False)
58
+ @click.option("--interactive-mode", type=click.BOOL, required=False)
52
59
  @click.option("--image-cache", required=False)
53
60
  @click.option("--tgz", required=False)
54
61
  @click.option("--pkl", required=False)
@@ -59,12 +66,16 @@ def _pass_through():
59
66
  type=click.UNPROCESSED,
60
67
  nargs=-1,
61
68
  )
69
+ @click.pass_context
62
70
  def main(
71
+ ctx: click.Context,
63
72
  run_name: str,
64
73
  name: str,
65
74
  project: str,
66
75
  domain: str,
67
76
  org: str,
77
+ debug: bool,
78
+ interactive_mode: bool,
68
79
  image_cache: str,
69
80
  version: str,
70
81
  inputs: str,
@@ -87,6 +98,7 @@ def main(
87
98
  import flyte
88
99
  import flyte._utils as utils
89
100
  import flyte.errors
101
+ import flyte.storage as storage
90
102
  from flyte._initialize import init
91
103
  from flyte._internal.controllers import create_controller
92
104
  from flyte._internal.imagebuild.image_builder import ImageCache
@@ -109,6 +121,13 @@ def main(
109
121
  if name.startswith("{{"):
110
122
  name = os.getenv("ACTION_NAME", "")
111
123
 
124
+ logger.warning(f"Flyte runtime started for action {name} with run name {run_name}")
125
+
126
+ if debug and name == "a0":
127
+ from flyte._debug.vscode import _start_vscode_server
128
+
129
+ asyncio.run(_start_vscode_server(ctx))
130
+
112
131
  # Figure out how to connect
113
132
  # This detection of api key is a hack for now.
114
133
  controller_kwargs: dict[str, Any] = {"insecure": False}
@@ -122,19 +141,40 @@ def main(
122
141
  controller_kwargs["insecure"] = True
123
142
  logger.debug(f"Using controller endpoint: {ep} with kwargs: {controller_kwargs}")
124
143
 
125
- bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
126
- init(org=org, project=project, domain=domain, **controller_kwargs)
144
+ # Check for insecure_skip_verify override (e.g. for self-signed certs)
145
+ insecure_skip_verify_str = os.getenv(INSECURE_SKIP_VERIFY_OVERRIDE, "")
146
+ if str2bool(insecure_skip_verify_str):
147
+ controller_kwargs["insecure_skip_verify"] = True
148
+ logger.info("SSL certificate verification disabled (insecure_skip_verify=True)")
149
+
150
+ bundle = None
151
+ if tgz or pkl:
152
+ bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
153
+ init(org=org, project=project, domain=domain, image_builder="remote", **controller_kwargs)
127
154
  # Controller is created with the same kwargs as init, so that it can be used to run tasks
128
155
  controller = create_controller(ct="remote", **controller_kwargs)
129
156
 
130
157
  ic = ImageCache.from_transport(image_cache) if image_cache else None
131
158
 
159
+ path_rewrite_cfg = os.getenv(_F_PATH_REWRITE, None)
160
+ path_rewrite = None
161
+ if path_rewrite_cfg:
162
+ potential_path_rewrite = PathRewrite.from_str(path_rewrite_cfg)
163
+ if storage.exists_sync(potential_path_rewrite.new_prefix):
164
+ path_rewrite = potential_path_rewrite
165
+ logger.info(f"Path rewrite configured for {path_rewrite.new_prefix}")
166
+ else:
167
+ logger.error(
168
+ f"Path rewrite failed for path {potential_path_rewrite.new_prefix}, "
169
+ f"not found, reverting to original path {potential_path_rewrite.old_prefix}"
170
+ )
171
+
132
172
  # Create a coroutine to load the task and run it
133
173
  task_coroutine = load_and_run_task(
134
174
  resolver=resolver,
135
175
  resolver_args=resolver_args,
136
176
  action=ActionID(name=name, run_name=run_name, project=project, domain=domain, org=org),
137
- raw_data_path=RawDataPath(path=raw_data_path),
177
+ raw_data_path=RawDataPath(path=raw_data_path, path_rewrite=path_rewrite),
138
178
  checkpoints=Checkpoints(checkpoint_path, prev_checkpoint),
139
179
  code_bundle=bundle,
140
180
  input_path=inputs,
@@ -143,6 +183,7 @@ def main(
143
183
  version=version,
144
184
  controller=controller,
145
185
  image_cache=ic,
186
+ interactive_mode=interactive_mode or debug,
146
187
  )
147
188
  # Create a coroutine to watch for errors
148
189
  controller_failure = controller.watch_for_errors()
@@ -151,10 +192,23 @@ def main(
151
192
  async def _run_and_stop():
152
193
  loop = asyncio.get_event_loop()
153
194
  loop.set_exception_handler(flyte.errors.silence_grpc_polling_error)
154
- await utils.run_coros(controller_failure, task_coroutine)
155
- await controller.stop()
195
+ try:
196
+ await utils.run_coros(controller_failure, task_coroutine)
197
+ await controller.stop()
198
+ except flyte.errors.RuntimeSystemError as e:
199
+ logger.error(f"Runtime system error: {e}")
200
+ from flyte._internal.runtime.convert import convert_from_native_to_error
201
+ from flyte._internal.runtime.io import upload_error
202
+
203
+ logger.error(f"Flyte runtime failed for action {name} with run name {run_name}, error: {e}")
204
+ err = convert_from_native_to_error(e)
205
+ path = await upload_error(err.err, outputs_path)
206
+ logger.error(f"Run {run_name} Action {name} failed with error: {err}. Uploaded error to {path}")
207
+ await controller.stop()
208
+ raise
156
209
 
157
210
  asyncio.run(_run_and_stop())
211
+ logger.warning(f"Flyte runtime completed for action {name} with run name {run_name}")
158
212
 
159
213
 
160
214
  if __name__ == "__main__":
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flyte
3
- Version: 2.0.0b13
3
+ Version: 2.0.0b30
4
4
  Summary: Add your description here
5
5
  Author-email: Ketan Umare <kumare3@users.noreply.github.com>
6
6
  Requires-Python: >=3.10
@@ -8,7 +8,7 @@ Description-Content-Type: text/markdown
8
8
  License-File: LICENSE
9
9
  Requires-Dist: aiofiles>=24.1.0
10
10
  Requires-Dist: click>=8.2.1
11
- Requires-Dist: flyteidl>=1.15.4b0
11
+ Requires-Dist: flyteidl<2.0.0,>=1.15.4b0
12
12
  Requires-Dist: cloudpickle>=3.1.1
13
13
  Requires-Dist: fsspec>=2025.3.0
14
14
  Requires-Dist: grpcio>=1.71.0
@@ -16,7 +16,7 @@ Requires-Dist: obstore>=0.7.3
16
16
  Requires-Dist: protobuf>=6.30.1
17
17
  Requires-Dist: pydantic>=2.10.6
18
18
  Requires-Dist: pyyaml>=6.0.2
19
- Requires-Dist: rich-click>=1.8.9
19
+ Requires-Dist: rich-click==1.8.9
20
20
  Requires-Dist: httpx<1.0.0,>=0.28.1
21
21
  Requires-Dist: keyring>=25.6.0
22
22
  Requires-Dist: msgpack>=1.1.0
@@ -24,6 +24,14 @@ Requires-Dist: toml>=0.10.2
24
24
  Requires-Dist: async-lru>=2.0.5
25
25
  Requires-Dist: mashumaro
26
26
  Requires-Dist: dataclasses_json
27
+ Requires-Dist: aiolimiter>=1.2.1
28
+ Requires-Dist: flyteidl2==2.0.0a14
29
+ Provides-Extra: aiosqlite
30
+ Requires-Dist: aiosqlite>=0.21.0; extra == "aiosqlite"
31
+ Provides-Extra: connector
32
+ Requires-Dist: grpcio-health-checking; extra == "connector"
33
+ Requires-Dist: httpx; extra == "connector"
34
+ Requires-Dist: prometheus-client; extra == "connector"
27
35
  Dynamic: license-file
28
36
 
29
37
  # Flyte 2 SDK 🚀