flyte 0.0.1b0__py3-none-any.whl → 2.0.0b46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flyte/__init__.py +83 -30
- flyte/_bin/connect.py +61 -0
- flyte/_bin/debug.py +38 -0
- flyte/_bin/runtime.py +87 -19
- flyte/_bin/serve.py +351 -0
- flyte/_build.py +3 -2
- flyte/_cache/cache.py +6 -5
- flyte/_cache/local_cache.py +216 -0
- flyte/_code_bundle/_ignore.py +31 -5
- flyte/_code_bundle/_packaging.py +42 -11
- flyte/_code_bundle/_utils.py +57 -34
- flyte/_code_bundle/bundle.py +130 -27
- flyte/_constants.py +1 -0
- flyte/_context.py +21 -5
- flyte/_custom_context.py +73 -0
- flyte/_debug/constants.py +37 -0
- flyte/_debug/utils.py +17 -0
- flyte/_debug/vscode.py +315 -0
- flyte/_deploy.py +396 -75
- flyte/_deployer.py +109 -0
- flyte/_environment.py +94 -11
- flyte/_excepthook.py +37 -0
- flyte/_group.py +2 -1
- flyte/_hash.py +1 -16
- flyte/_image.py +544 -234
- flyte/_initialize.py +443 -294
- flyte/_interface.py +40 -5
- flyte/_internal/controllers/__init__.py +22 -8
- flyte/_internal/controllers/_local_controller.py +159 -35
- flyte/_internal/controllers/_trace.py +18 -10
- flyte/_internal/controllers/remote/__init__.py +38 -9
- flyte/_internal/controllers/remote/_action.py +82 -12
- flyte/_internal/controllers/remote/_client.py +6 -2
- flyte/_internal/controllers/remote/_controller.py +290 -64
- flyte/_internal/controllers/remote/_core.py +155 -95
- flyte/_internal/controllers/remote/_informer.py +40 -20
- flyte/_internal/controllers/remote/_service_protocol.py +2 -2
- flyte/_internal/imagebuild/__init__.py +2 -10
- flyte/_internal/imagebuild/docker_builder.py +391 -84
- flyte/_internal/imagebuild/image_builder.py +111 -55
- flyte/_internal/imagebuild/remote_builder.py +409 -0
- flyte/_internal/imagebuild/utils.py +79 -0
- flyte/_internal/resolvers/_app_env_module.py +92 -0
- flyte/_internal/resolvers/_task_module.py +5 -38
- flyte/_internal/resolvers/app_env.py +26 -0
- flyte/_internal/resolvers/common.py +8 -1
- flyte/_internal/resolvers/default.py +2 -2
- flyte/_internal/runtime/convert.py +322 -33
- flyte/_internal/runtime/entrypoints.py +106 -18
- flyte/_internal/runtime/io.py +71 -23
- flyte/_internal/runtime/resources_serde.py +21 -7
- flyte/_internal/runtime/reuse.py +125 -0
- flyte/_internal/runtime/rusty.py +196 -0
- flyte/_internal/runtime/task_serde.py +239 -66
- flyte/_internal/runtime/taskrunner.py +48 -8
- flyte/_internal/runtime/trigger_serde.py +162 -0
- flyte/_internal/runtime/types_serde.py +7 -16
- flyte/_keyring/file.py +115 -0
- flyte/_link.py +30 -0
- flyte/_logging.py +241 -42
- flyte/_map.py +312 -0
- flyte/_metrics.py +59 -0
- flyte/_module.py +74 -0
- flyte/_pod.py +30 -0
- flyte/_resources.py +296 -33
- flyte/_retry.py +1 -7
- flyte/_reusable_environment.py +72 -7
- flyte/_run.py +461 -132
- flyte/_secret.py +47 -11
- flyte/_serve.py +333 -0
- flyte/_task.py +245 -56
- flyte/_task_environment.py +219 -97
- flyte/_task_plugins.py +47 -0
- flyte/_tools.py +8 -8
- flyte/_trace.py +15 -24
- flyte/_trigger.py +1027 -0
- flyte/_utils/__init__.py +12 -1
- flyte/_utils/asyn.py +3 -1
- flyte/_utils/async_cache.py +139 -0
- flyte/_utils/coro_management.py +5 -4
- flyte/_utils/description_parser.py +19 -0
- flyte/_utils/docker_credentials.py +173 -0
- flyte/_utils/helpers.py +45 -19
- flyte/_utils/module_loader.py +123 -0
- flyte/_utils/org_discovery.py +57 -0
- flyte/_utils/uv_script_parser.py +8 -1
- flyte/_version.py +16 -3
- flyte/app/__init__.py +27 -0
- flyte/app/_app_environment.py +362 -0
- flyte/app/_connector_environment.py +40 -0
- flyte/app/_deploy.py +130 -0
- flyte/app/_parameter.py +343 -0
- flyte/app/_runtime/__init__.py +3 -0
- flyte/app/_runtime/app_serde.py +383 -0
- flyte/app/_types.py +113 -0
- flyte/app/extras/__init__.py +9 -0
- flyte/app/extras/_auth_middleware.py +217 -0
- flyte/app/extras/_fastapi.py +93 -0
- flyte/app/extras/_model_loader/__init__.py +3 -0
- flyte/app/extras/_model_loader/config.py +7 -0
- flyte/app/extras/_model_loader/loader.py +288 -0
- flyte/cli/__init__.py +12 -0
- flyte/cli/_abort.py +28 -0
- flyte/cli/_build.py +114 -0
- flyte/cli/_common.py +493 -0
- flyte/cli/_create.py +371 -0
- flyte/cli/_delete.py +45 -0
- flyte/cli/_deploy.py +401 -0
- flyte/cli/_gen.py +316 -0
- flyte/cli/_get.py +446 -0
- flyte/cli/_option.py +33 -0
- {union/_cli → flyte/cli}/_params.py +152 -153
- flyte/cli/_plugins.py +209 -0
- flyte/cli/_prefetch.py +292 -0
- flyte/cli/_run.py +690 -0
- flyte/cli/_serve.py +338 -0
- flyte/cli/_update.py +86 -0
- flyte/cli/_user.py +20 -0
- flyte/cli/main.py +246 -0
- flyte/config/__init__.py +3 -0
- flyte/config/_config.py +248 -0
- flyte/config/_internal.py +73 -0
- flyte/config/_reader.py +225 -0
- flyte/connectors/__init__.py +11 -0
- flyte/connectors/_connector.py +330 -0
- flyte/connectors/_server.py +194 -0
- flyte/connectors/utils.py +159 -0
- flyte/errors.py +134 -2
- flyte/extend.py +24 -0
- flyte/extras/_container.py +69 -56
- flyte/git/__init__.py +3 -0
- flyte/git/_config.py +279 -0
- flyte/io/__init__.py +8 -1
- flyte/io/{structured_dataset → _dataframe}/__init__.py +32 -30
- flyte/io/{structured_dataset → _dataframe}/basic_dfs.py +75 -68
- flyte/io/{structured_dataset/structured_dataset.py → _dataframe/dataframe.py} +207 -242
- flyte/io/_dir.py +575 -113
- flyte/io/_file.py +587 -141
- flyte/io/_hashing_io.py +342 -0
- flyte/io/extend.py +7 -0
- flyte/models.py +635 -0
- flyte/prefetch/__init__.py +22 -0
- flyte/prefetch/_hf_model.py +563 -0
- flyte/remote/__init__.py +14 -3
- flyte/remote/_action.py +879 -0
- flyte/remote/_app.py +346 -0
- flyte/remote/_auth_metadata.py +42 -0
- flyte/remote/_client/_protocols.py +62 -4
- flyte/remote/_client/auth/_auth_utils.py +19 -0
- flyte/remote/_client/auth/_authenticators/base.py +8 -2
- flyte/remote/_client/auth/_authenticators/device_code.py +4 -5
- flyte/remote/_client/auth/_authenticators/factory.py +4 -0
- flyte/remote/_client/auth/_authenticators/passthrough.py +79 -0
- flyte/remote/_client/auth/_authenticators/pkce.py +17 -18
- flyte/remote/_client/auth/_channel.py +47 -18
- flyte/remote/_client/auth/_client_config.py +5 -3
- flyte/remote/_client/auth/_keyring.py +15 -2
- flyte/remote/_client/auth/_token_client.py +3 -3
- flyte/remote/_client/controlplane.py +206 -18
- flyte/remote/_common.py +66 -0
- flyte/remote/_data.py +107 -22
- flyte/remote/_logs.py +116 -33
- flyte/remote/_project.py +21 -19
- flyte/remote/_run.py +164 -631
- flyte/remote/_secret.py +72 -29
- flyte/remote/_task.py +387 -46
- flyte/remote/_trigger.py +368 -0
- flyte/remote/_user.py +43 -0
- flyte/report/_report.py +10 -6
- flyte/storage/__init__.py +13 -1
- flyte/storage/_config.py +237 -0
- flyte/storage/_parallel_reader.py +289 -0
- flyte/storage/_storage.py +268 -59
- flyte/syncify/__init__.py +56 -0
- flyte/syncify/_api.py +414 -0
- flyte/types/__init__.py +39 -0
- flyte/types/_interface.py +22 -7
- flyte/{io/pickle/transformer.py → types/_pickle.py} +37 -9
- flyte/types/_string_literals.py +8 -9
- flyte/types/_type_engine.py +230 -129
- flyte/types/_utils.py +1 -1
- flyte-2.0.0b46.data/scripts/debug.py +38 -0
- flyte-2.0.0b46.data/scripts/runtime.py +194 -0
- flyte-2.0.0b46.dist-info/METADATA +352 -0
- flyte-2.0.0b46.dist-info/RECORD +221 -0
- flyte-2.0.0b46.dist-info/entry_points.txt +8 -0
- flyte-2.0.0b46.dist-info/licenses/LICENSE +201 -0
- flyte/_api_commons.py +0 -3
- flyte/_cli/_common.py +0 -287
- flyte/_cli/_create.py +0 -42
- flyte/_cli/_delete.py +0 -23
- flyte/_cli/_deploy.py +0 -140
- flyte/_cli/_get.py +0 -235
- flyte/_cli/_run.py +0 -152
- flyte/_cli/main.py +0 -72
- flyte/_datastructures.py +0 -342
- flyte/_internal/controllers/pbhash.py +0 -39
- flyte/_protos/common/authorization_pb2.py +0 -66
- flyte/_protos/common/authorization_pb2.pyi +0 -108
- flyte/_protos/common/authorization_pb2_grpc.py +0 -4
- flyte/_protos/common/identifier_pb2.py +0 -71
- flyte/_protos/common/identifier_pb2.pyi +0 -82
- flyte/_protos/common/identifier_pb2_grpc.py +0 -4
- flyte/_protos/common/identity_pb2.py +0 -48
- flyte/_protos/common/identity_pb2.pyi +0 -72
- flyte/_protos/common/identity_pb2_grpc.py +0 -4
- flyte/_protos/common/list_pb2.py +0 -36
- flyte/_protos/common/list_pb2.pyi +0 -69
- flyte/_protos/common/list_pb2_grpc.py +0 -4
- flyte/_protos/common/policy_pb2.py +0 -37
- flyte/_protos/common/policy_pb2.pyi +0 -27
- flyte/_protos/common/policy_pb2_grpc.py +0 -4
- flyte/_protos/common/role_pb2.py +0 -37
- flyte/_protos/common/role_pb2.pyi +0 -53
- flyte/_protos/common/role_pb2_grpc.py +0 -4
- flyte/_protos/common/runtime_version_pb2.py +0 -28
- flyte/_protos/common/runtime_version_pb2.pyi +0 -24
- flyte/_protos/common/runtime_version_pb2_grpc.py +0 -4
- flyte/_protos/logs/dataplane/payload_pb2.py +0 -96
- flyte/_protos/logs/dataplane/payload_pb2.pyi +0 -168
- flyte/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
- flyte/_protos/secret/definition_pb2.py +0 -49
- flyte/_protos/secret/definition_pb2.pyi +0 -93
- flyte/_protos/secret/definition_pb2_grpc.py +0 -4
- flyte/_protos/secret/payload_pb2.py +0 -62
- flyte/_protos/secret/payload_pb2.pyi +0 -94
- flyte/_protos/secret/payload_pb2_grpc.py +0 -4
- flyte/_protos/secret/secret_pb2.py +0 -38
- flyte/_protos/secret/secret_pb2.pyi +0 -6
- flyte/_protos/secret/secret_pb2_grpc.py +0 -198
- flyte/_protos/secret/secret_pb2_grpc_grpc.py +0 -198
- flyte/_protos/validate/validate/validate_pb2.py +0 -76
- flyte/_protos/workflow/node_execution_service_pb2.py +0 -26
- flyte/_protos/workflow/node_execution_service_pb2.pyi +0 -4
- flyte/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
- flyte/_protos/workflow/queue_service_pb2.py +0 -106
- flyte/_protos/workflow/queue_service_pb2.pyi +0 -141
- flyte/_protos/workflow/queue_service_pb2_grpc.py +0 -172
- flyte/_protos/workflow/run_definition_pb2.py +0 -128
- flyte/_protos/workflow/run_definition_pb2.pyi +0 -310
- flyte/_protos/workflow/run_definition_pb2_grpc.py +0 -4
- flyte/_protos/workflow/run_logs_service_pb2.py +0 -41
- flyte/_protos/workflow/run_logs_service_pb2.pyi +0 -28
- flyte/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
- flyte/_protos/workflow/run_service_pb2.py +0 -133
- flyte/_protos/workflow/run_service_pb2.pyi +0 -175
- flyte/_protos/workflow/run_service_pb2_grpc.py +0 -412
- flyte/_protos/workflow/state_service_pb2.py +0 -58
- flyte/_protos/workflow/state_service_pb2.pyi +0 -71
- flyte/_protos/workflow/state_service_pb2_grpc.py +0 -138
- flyte/_protos/workflow/task_definition_pb2.py +0 -72
- flyte/_protos/workflow/task_definition_pb2.pyi +0 -65
- flyte/_protos/workflow/task_definition_pb2_grpc.py +0 -4
- flyte/_protos/workflow/task_service_pb2.py +0 -44
- flyte/_protos/workflow/task_service_pb2.pyi +0 -31
- flyte/_protos/workflow/task_service_pb2_grpc.py +0 -104
- flyte/io/_dataframe.py +0 -0
- flyte/io/pickle/__init__.py +0 -0
- flyte/remote/_console.py +0 -18
- flyte-0.0.1b0.dist-info/METADATA +0 -179
- flyte-0.0.1b0.dist-info/RECORD +0 -390
- flyte-0.0.1b0.dist-info/entry_points.txt +0 -3
- union/__init__.py +0 -54
- union/_api_commons.py +0 -3
- union/_bin/__init__.py +0 -0
- union/_bin/runtime.py +0 -113
- union/_build.py +0 -25
- union/_cache/__init__.py +0 -12
- union/_cache/cache.py +0 -141
- union/_cache/defaults.py +0 -9
- union/_cache/policy_function_body.py +0 -42
- union/_cli/__init__.py +0 -0
- union/_cli/_common.py +0 -263
- union/_cli/_create.py +0 -40
- union/_cli/_delete.py +0 -23
- union/_cli/_deploy.py +0 -120
- union/_cli/_get.py +0 -162
- union/_cli/_run.py +0 -150
- union/_cli/main.py +0 -72
- union/_code_bundle/__init__.py +0 -8
- union/_code_bundle/_ignore.py +0 -113
- union/_code_bundle/_packaging.py +0 -187
- union/_code_bundle/_utils.py +0 -342
- union/_code_bundle/bundle.py +0 -176
- union/_context.py +0 -146
- union/_datastructures.py +0 -295
- union/_deploy.py +0 -185
- union/_doc.py +0 -29
- union/_docstring.py +0 -26
- union/_environment.py +0 -43
- union/_group.py +0 -31
- union/_hash.py +0 -23
- union/_image.py +0 -760
- union/_initialize.py +0 -585
- union/_interface.py +0 -84
- union/_internal/__init__.py +0 -3
- union/_internal/controllers/__init__.py +0 -77
- union/_internal/controllers/_local_controller.py +0 -77
- union/_internal/controllers/pbhash.py +0 -39
- union/_internal/controllers/remote/__init__.py +0 -40
- union/_internal/controllers/remote/_action.py +0 -131
- union/_internal/controllers/remote/_client.py +0 -43
- union/_internal/controllers/remote/_controller.py +0 -169
- union/_internal/controllers/remote/_core.py +0 -341
- union/_internal/controllers/remote/_informer.py +0 -260
- union/_internal/controllers/remote/_service_protocol.py +0 -44
- union/_internal/imagebuild/__init__.py +0 -11
- union/_internal/imagebuild/docker_builder.py +0 -416
- union/_internal/imagebuild/image_builder.py +0 -243
- union/_internal/imagebuild/remote_builder.py +0 -0
- union/_internal/resolvers/__init__.py +0 -0
- union/_internal/resolvers/_task_module.py +0 -31
- union/_internal/resolvers/common.py +0 -24
- union/_internal/resolvers/default.py +0 -27
- union/_internal/runtime/__init__.py +0 -0
- union/_internal/runtime/convert.py +0 -163
- union/_internal/runtime/entrypoints.py +0 -121
- union/_internal/runtime/io.py +0 -136
- union/_internal/runtime/resources_serde.py +0 -134
- union/_internal/runtime/task_serde.py +0 -202
- union/_internal/runtime/taskrunner.py +0 -179
- union/_internal/runtime/types_serde.py +0 -53
- union/_logging.py +0 -124
- union/_protos/__init__.py +0 -0
- union/_protos/common/authorization_pb2.py +0 -66
- union/_protos/common/authorization_pb2.pyi +0 -106
- union/_protos/common/authorization_pb2_grpc.py +0 -4
- union/_protos/common/identifier_pb2.py +0 -71
- union/_protos/common/identifier_pb2.pyi +0 -82
- union/_protos/common/identifier_pb2_grpc.py +0 -4
- union/_protos/common/identity_pb2.py +0 -48
- union/_protos/common/identity_pb2.pyi +0 -72
- union/_protos/common/identity_pb2_grpc.py +0 -4
- union/_protos/common/list_pb2.py +0 -36
- union/_protos/common/list_pb2.pyi +0 -69
- union/_protos/common/list_pb2_grpc.py +0 -4
- union/_protos/common/policy_pb2.py +0 -37
- union/_protos/common/policy_pb2.pyi +0 -27
- union/_protos/common/policy_pb2_grpc.py +0 -4
- union/_protos/common/role_pb2.py +0 -37
- union/_protos/common/role_pb2.pyi +0 -51
- union/_protos/common/role_pb2_grpc.py +0 -4
- union/_protos/common/runtime_version_pb2.py +0 -28
- union/_protos/common/runtime_version_pb2.pyi +0 -24
- union/_protos/common/runtime_version_pb2_grpc.py +0 -4
- union/_protos/logs/dataplane/payload_pb2.py +0 -96
- union/_protos/logs/dataplane/payload_pb2.pyi +0 -168
- union/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
- union/_protos/secret/definition_pb2.py +0 -49
- union/_protos/secret/definition_pb2.pyi +0 -93
- union/_protos/secret/definition_pb2_grpc.py +0 -4
- union/_protos/secret/payload_pb2.py +0 -62
- union/_protos/secret/payload_pb2.pyi +0 -94
- union/_protos/secret/payload_pb2_grpc.py +0 -4
- union/_protos/secret/secret_pb2.py +0 -38
- union/_protos/secret/secret_pb2.pyi +0 -6
- union/_protos/secret/secret_pb2_grpc.py +0 -198
- union/_protos/validate/validate/validate_pb2.py +0 -76
- union/_protos/workflow/node_execution_service_pb2.py +0 -26
- union/_protos/workflow/node_execution_service_pb2.pyi +0 -4
- union/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
- union/_protos/workflow/queue_service_pb2.py +0 -75
- union/_protos/workflow/queue_service_pb2.pyi +0 -103
- union/_protos/workflow/queue_service_pb2_grpc.py +0 -172
- union/_protos/workflow/run_definition_pb2.py +0 -100
- union/_protos/workflow/run_definition_pb2.pyi +0 -256
- union/_protos/workflow/run_definition_pb2_grpc.py +0 -4
- union/_protos/workflow/run_logs_service_pb2.py +0 -41
- union/_protos/workflow/run_logs_service_pb2.pyi +0 -28
- union/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
- union/_protos/workflow/run_service_pb2.py +0 -133
- union/_protos/workflow/run_service_pb2.pyi +0 -173
- union/_protos/workflow/run_service_pb2_grpc.py +0 -412
- union/_protos/workflow/state_service_pb2.py +0 -58
- union/_protos/workflow/state_service_pb2.pyi +0 -69
- union/_protos/workflow/state_service_pb2_grpc.py +0 -138
- union/_protos/workflow/task_definition_pb2.py +0 -72
- union/_protos/workflow/task_definition_pb2.pyi +0 -65
- union/_protos/workflow/task_definition_pb2_grpc.py +0 -4
- union/_protos/workflow/task_service_pb2.py +0 -44
- union/_protos/workflow/task_service_pb2.pyi +0 -31
- union/_protos/workflow/task_service_pb2_grpc.py +0 -104
- union/_resources.py +0 -226
- union/_retry.py +0 -32
- union/_reusable_environment.py +0 -25
- union/_run.py +0 -374
- union/_secret.py +0 -61
- union/_task.py +0 -354
- union/_task_environment.py +0 -186
- union/_timeout.py +0 -47
- union/_tools.py +0 -27
- union/_utils/__init__.py +0 -11
- union/_utils/asyn.py +0 -119
- union/_utils/file_handling.py +0 -71
- union/_utils/helpers.py +0 -46
- union/_utils/lazy_module.py +0 -54
- union/_utils/uv_script_parser.py +0 -49
- union/_version.py +0 -21
- union/connectors/__init__.py +0 -0
- union/errors.py +0 -128
- union/extras/__init__.py +0 -5
- union/extras/_container.py +0 -263
- union/io/__init__.py +0 -11
- union/io/_dataframe.py +0 -0
- union/io/_dir.py +0 -425
- union/io/_file.py +0 -418
- union/io/pickle/__init__.py +0 -0
- union/io/pickle/transformer.py +0 -117
- union/io/structured_dataset/__init__.py +0 -122
- union/io/structured_dataset/basic_dfs.py +0 -219
- union/io/structured_dataset/structured_dataset.py +0 -1057
- union/py.typed +0 -0
- union/remote/__init__.py +0 -23
- union/remote/_client/__init__.py +0 -0
- union/remote/_client/_protocols.py +0 -129
- union/remote/_client/auth/__init__.py +0 -12
- union/remote/_client/auth/_authenticators/__init__.py +0 -0
- union/remote/_client/auth/_authenticators/base.py +0 -391
- union/remote/_client/auth/_authenticators/client_credentials.py +0 -73
- union/remote/_client/auth/_authenticators/device_code.py +0 -120
- union/remote/_client/auth/_authenticators/external_command.py +0 -77
- union/remote/_client/auth/_authenticators/factory.py +0 -200
- union/remote/_client/auth/_authenticators/pkce.py +0 -515
- union/remote/_client/auth/_channel.py +0 -184
- union/remote/_client/auth/_client_config.py +0 -83
- union/remote/_client/auth/_default_html.py +0 -32
- union/remote/_client/auth/_grpc_utils/__init__.py +0 -0
- union/remote/_client/auth/_grpc_utils/auth_interceptor.py +0 -204
- union/remote/_client/auth/_grpc_utils/default_metadata_interceptor.py +0 -144
- union/remote/_client/auth/_keyring.py +0 -154
- union/remote/_client/auth/_token_client.py +0 -258
- union/remote/_client/auth/errors.py +0 -16
- union/remote/_client/controlplane.py +0 -86
- union/remote/_data.py +0 -149
- union/remote/_logs.py +0 -74
- union/remote/_project.py +0 -86
- union/remote/_run.py +0 -820
- union/remote/_secret.py +0 -132
- union/remote/_task.py +0 -193
- union/report/__init__.py +0 -3
- union/report/_report.py +0 -178
- union/report/_template.html +0 -124
- union/storage/__init__.py +0 -24
- union/storage/_remote_fs.py +0 -34
- union/storage/_storage.py +0 -247
- union/storage/_utils.py +0 -5
- union/types/__init__.py +0 -11
- union/types/_renderer.py +0 -162
- union/types/_string_literals.py +0 -120
- union/types/_type_engine.py +0 -2131
- union/types/_utils.py +0 -80
- /flyte/{_cli → _debug}/__init__.py +0 -0
- /flyte/{_protos → _keyring}/__init__.py +0 -0
- {flyte-0.0.1b0.dist-info → flyte-2.0.0b46.dist-info}/WHEEL +0 -0
- {flyte-0.0.1b0.dist-info → flyte-2.0.0b46.dist-info}/top_level.txt +0 -0
flyte/syncify/_api.py
ADDED
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import atexit
|
|
5
|
+
import concurrent.futures
|
|
6
|
+
import functools
|
|
7
|
+
import inspect
|
|
8
|
+
import logging
|
|
9
|
+
import threading
|
|
10
|
+
from typing import (
|
|
11
|
+
Any,
|
|
12
|
+
AsyncIterator,
|
|
13
|
+
Awaitable,
|
|
14
|
+
Callable,
|
|
15
|
+
Coroutine,
|
|
16
|
+
Iterator,
|
|
17
|
+
ParamSpec,
|
|
18
|
+
Protocol,
|
|
19
|
+
TypeVar,
|
|
20
|
+
Union,
|
|
21
|
+
cast,
|
|
22
|
+
overload,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
from flyte._logging import logger
|
|
26
|
+
|
|
27
|
+
P = ParamSpec("P")
|
|
28
|
+
|
|
29
|
+
# Track loops that have already had the gRPC error handler installed
|
|
30
|
+
_configured_loops: set[int] = set()
|
|
31
|
+
_configured_loops_lock = threading.Lock()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _ensure_grpc_error_handler_installed() -> None:
|
|
35
|
+
"""
|
|
36
|
+
Install the gRPC error handler on the current event loop if not already done.
|
|
37
|
+
Uses a thread-safe set to track which loops have been configured.
|
|
38
|
+
"""
|
|
39
|
+
try:
|
|
40
|
+
loop = asyncio.get_running_loop()
|
|
41
|
+
except RuntimeError:
|
|
42
|
+
return # No running loop, nothing to do
|
|
43
|
+
|
|
44
|
+
loop_id = id(loop)
|
|
45
|
+
with _configured_loops_lock:
|
|
46
|
+
if loop_id in _configured_loops:
|
|
47
|
+
return # Already configured
|
|
48
|
+
_configured_loops.add(loop_id)
|
|
49
|
+
|
|
50
|
+
import flyte.errors
|
|
51
|
+
|
|
52
|
+
loop.set_exception_handler(flyte.errors.silence_grpc_polling_error)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
R_co = TypeVar("R_co", covariant=True)
|
|
56
|
+
T = TypeVar("T")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class SyncFunction(Protocol[P, R_co]):
|
|
60
|
+
"""
|
|
61
|
+
A protocol that defines the interface for synchronous functions or methods that can be converted from asynchronous
|
|
62
|
+
ones.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __call__(self, *args: Any, **kwargs: Any) -> R_co: ...
|
|
66
|
+
|
|
67
|
+
def aio(self, *args: Any, **kwargs: Any) -> Awaitable[R_co]: ...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class SyncGenFunction(Protocol[P, R_co]):
|
|
71
|
+
"""
|
|
72
|
+
A protocol that defines the interface for synchronous functions or methods that can be converted from asynchronous
|
|
73
|
+
ones.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
def __call__(self, *args: Any, **kwargs: Any) -> Iterator[R_co]: ...
|
|
77
|
+
|
|
78
|
+
def aio(self, *args: Any, **kwargs: Any) -> AsyncIterator[R_co]: ...
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class _BackgroundLoop:
|
|
82
|
+
"""
|
|
83
|
+
A background event loop that runs in a separate thread and used the `Syncify` decorator to run asynchronous
|
|
84
|
+
functions or methods synchronously.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
def __init__(self, name: str):
|
|
88
|
+
self.loop = asyncio.new_event_loop()
|
|
89
|
+
self.thread = threading.Thread(name=name, target=self._run, daemon=True)
|
|
90
|
+
self.thread.start()
|
|
91
|
+
atexit.register(self.stop)
|
|
92
|
+
|
|
93
|
+
def _run(self):
|
|
94
|
+
# Delay import to avoid deadlock during module initialization
|
|
95
|
+
def exception_handler(loop, context):
|
|
96
|
+
import flyte.errors
|
|
97
|
+
|
|
98
|
+
flyte.errors.silence_grpc_polling_error(loop, context)
|
|
99
|
+
|
|
100
|
+
# Set the exception handler to silence specific gRPC polling errors
|
|
101
|
+
self.loop.set_exception_handler(exception_handler)
|
|
102
|
+
asyncio.set_event_loop(self.loop)
|
|
103
|
+
self.loop.run_forever()
|
|
104
|
+
|
|
105
|
+
def stop(self):
|
|
106
|
+
# stop the loop and wait briefly for thread to exit
|
|
107
|
+
self.loop.call_soon_threadsafe(self.loop.stop)
|
|
108
|
+
self.thread.join(timeout=1)
|
|
109
|
+
|
|
110
|
+
def is_in_loop(self) -> bool:
|
|
111
|
+
"""
|
|
112
|
+
Check if the current thread is the background loop thread.
|
|
113
|
+
"""
|
|
114
|
+
# If the current thread is not the background loop thread, return False
|
|
115
|
+
if threading.current_thread() != self.thread:
|
|
116
|
+
return False
|
|
117
|
+
|
|
118
|
+
if not self.thread.is_alive():
|
|
119
|
+
# If the thread is not alive, we cannot be in the loop
|
|
120
|
+
return False
|
|
121
|
+
|
|
122
|
+
# Lets get the current event loop and check if it matches the background loop
|
|
123
|
+
loop = None
|
|
124
|
+
try:
|
|
125
|
+
loop = asyncio.get_running_loop()
|
|
126
|
+
except RuntimeError:
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
return loop == self.loop
|
|
130
|
+
|
|
131
|
+
def iterate_in_loop_sync(self, async_gen: AsyncIterator[R_co]) -> Iterator[R_co]:
|
|
132
|
+
# Create an iterator that pulls items from the async generator
|
|
133
|
+
assert self.thread.name != threading.current_thread().name, (
|
|
134
|
+
f"Cannot run coroutine in the same thread {self.thread.name}"
|
|
135
|
+
)
|
|
136
|
+
while True:
|
|
137
|
+
try:
|
|
138
|
+
# use __anext__() and cast to Coroutine so mypy is happy
|
|
139
|
+
future: concurrent.futures.Future[R_co] = asyncio.run_coroutine_threadsafe(
|
|
140
|
+
cast(Coroutine[Any, Any, R_co], async_gen.__anext__()),
|
|
141
|
+
self.loop,
|
|
142
|
+
)
|
|
143
|
+
yield future.result()
|
|
144
|
+
except (StopAsyncIteration, StopIteration):
|
|
145
|
+
break
|
|
146
|
+
except Exception as e:
|
|
147
|
+
if logger.getEffectiveLevel() > logging.DEBUG:
|
|
148
|
+
# If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the
|
|
149
|
+
# user
|
|
150
|
+
# This is because the stack trace will include the Syncify wrapper and the background loop thread
|
|
151
|
+
tb = e.__traceback__
|
|
152
|
+
while tb and tb.tb_next:
|
|
153
|
+
if tb.tb_frame.f_code.co_name == "":
|
|
154
|
+
break
|
|
155
|
+
tb = tb.tb_next
|
|
156
|
+
raise e.with_traceback(tb)
|
|
157
|
+
# If the log level is DEBUG, we will keep the extra stack frames to help with debugging
|
|
158
|
+
raise e
|
|
159
|
+
|
|
160
|
+
def call_in_loop_sync(self, coro: Coroutine[Any, Any, R_co]) -> R_co | Iterator[R_co]:
|
|
161
|
+
"""
|
|
162
|
+
Run the given coroutine in the background loop and return its result.
|
|
163
|
+
"""
|
|
164
|
+
future: concurrent.futures.Future[R_co | AsyncIterator[R_co]] = asyncio.run_coroutine_threadsafe(
|
|
165
|
+
coro, self.loop
|
|
166
|
+
)
|
|
167
|
+
result = future.result()
|
|
168
|
+
if result is not None and hasattr(result, "__aiter__"):
|
|
169
|
+
# If the result is an async iterator, we need to convert it to a sync iterator
|
|
170
|
+
return cast(Iterator[R_co], self.iterate_in_loop_sync(cast(AsyncIterator[R_co], result)))
|
|
171
|
+
# Otherwise, just return the result
|
|
172
|
+
return result
|
|
173
|
+
|
|
174
|
+
async def iterate_in_loop(self, async_gen: AsyncIterator[R_co]) -> AsyncIterator[R_co]:
|
|
175
|
+
"""
|
|
176
|
+
Run the given async iterator in the background loop and yield its results.
|
|
177
|
+
"""
|
|
178
|
+
if self.is_in_loop():
|
|
179
|
+
# If we are already in the background loop, just return the async iterator
|
|
180
|
+
async for r in async_gen:
|
|
181
|
+
yield r
|
|
182
|
+
return
|
|
183
|
+
|
|
184
|
+
# Install the gRPC error handler on the caller's event loop as well.
|
|
185
|
+
# This is needed because gRPC's async polling events may be delivered to
|
|
186
|
+
# the caller's loop (e.g., FastAPI's event loop) when using .aio().
|
|
187
|
+
_ensure_grpc_error_handler_installed()
|
|
188
|
+
|
|
189
|
+
while True:
|
|
190
|
+
try:
|
|
191
|
+
# same replacement here for the async path
|
|
192
|
+
future: concurrent.futures.Future[R_co] = asyncio.run_coroutine_threadsafe(
|
|
193
|
+
cast(Coroutine[Any, Any, R_co], async_gen.__anext__()),
|
|
194
|
+
self.loop,
|
|
195
|
+
)
|
|
196
|
+
# Wrap the future in an asyncio Future to yield it in an async context
|
|
197
|
+
aio_future: asyncio.Future[R_co] = asyncio.wrap_future(future)
|
|
198
|
+
# await for the future to complete and yield its result
|
|
199
|
+
v = await aio_future
|
|
200
|
+
yield v
|
|
201
|
+
except StopAsyncIteration:
|
|
202
|
+
break
|
|
203
|
+
except Exception as e:
|
|
204
|
+
if logger.getEffectiveLevel() > logging.DEBUG:
|
|
205
|
+
# If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the
|
|
206
|
+
# user.
|
|
207
|
+
# This is because the stack trace will include the Syncify wrapper and the background loop thread
|
|
208
|
+
tb = e.__traceback__
|
|
209
|
+
while tb and tb.tb_next:
|
|
210
|
+
if tb.tb_frame.f_code.co_name == "":
|
|
211
|
+
break
|
|
212
|
+
tb = tb.tb_next
|
|
213
|
+
raise e.with_traceback(tb)
|
|
214
|
+
# If the log level is DEBUG, we will keep the extra stack frames to help with debugging
|
|
215
|
+
raise e
|
|
216
|
+
|
|
217
|
+
async def aio(self, coro: Coroutine[Any, Any, R_co]) -> R_co:
|
|
218
|
+
"""
|
|
219
|
+
Run the given coroutine in the background loop and return its result.
|
|
220
|
+
"""
|
|
221
|
+
if self.is_in_loop():
|
|
222
|
+
# If we are already in the background loop, just run the coroutine
|
|
223
|
+
return await coro
|
|
224
|
+
try:
|
|
225
|
+
# Install the gRPC error handler on the caller's event loop as well.
|
|
226
|
+
# This is needed because gRPC's async polling events may be delivered to
|
|
227
|
+
# the caller's loop (e.g., FastAPI's event loop) when using .aio().
|
|
228
|
+
_ensure_grpc_error_handler_installed()
|
|
229
|
+
|
|
230
|
+
# Otherwise, run it in the background loop and wait for the result
|
|
231
|
+
future: concurrent.futures.Future[R_co] = asyncio.run_coroutine_threadsafe(coro, self.loop)
|
|
232
|
+
# Wrap the future in an asyncio Future to await it in an async context
|
|
233
|
+
aio_future: asyncio.Future[R_co] = asyncio.wrap_future(future)
|
|
234
|
+
# await for the future to complete and return its result
|
|
235
|
+
return await aio_future
|
|
236
|
+
except Exception as e:
|
|
237
|
+
if logger.getEffectiveLevel() > logging.DEBUG:
|
|
238
|
+
# If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the user
|
|
239
|
+
# This is because the stack trace will include the Syncify wrapper and the background loop thread
|
|
240
|
+
tb = e.__traceback__
|
|
241
|
+
while tb and tb.tb_next:
|
|
242
|
+
if tb.tb_frame.f_code.co_name == "":
|
|
243
|
+
break
|
|
244
|
+
tb = tb.tb_next
|
|
245
|
+
raise e.with_traceback(tb)
|
|
246
|
+
# If the log level is DEBUG, we will keep the extra stack frames to help with debugging
|
|
247
|
+
raise e
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
class _SyncWrapper:
|
|
251
|
+
"""
|
|
252
|
+
A wrapper class that the Syncify decorator uses to convert asynchronous functions or methods into synchronous ones.
|
|
253
|
+
"""
|
|
254
|
+
|
|
255
|
+
def __init__(
|
|
256
|
+
self,
|
|
257
|
+
fn: Any,
|
|
258
|
+
bg_loop: _BackgroundLoop,
|
|
259
|
+
underlying_obj: Any = None,
|
|
260
|
+
):
|
|
261
|
+
self.fn = fn
|
|
262
|
+
self._bg_loop = bg_loop
|
|
263
|
+
self._underlying_obj = underlying_obj
|
|
264
|
+
|
|
265
|
+
def __get__(self, instance: Any, owner: Any) -> Any:
|
|
266
|
+
"""
|
|
267
|
+
This method is called when the wrapper is accessed as a method of a class instance.
|
|
268
|
+
:param instance:
|
|
269
|
+
:param owner:
|
|
270
|
+
:return:
|
|
271
|
+
"""
|
|
272
|
+
fn: Any = self.fn
|
|
273
|
+
if instance is not None:
|
|
274
|
+
# If we have an instance, we need to bind the method to the instance (for instance methods)
|
|
275
|
+
fn = self.fn.__get__(instance, owner)
|
|
276
|
+
|
|
277
|
+
if instance is None and owner is not None and self._underlying_obj is not None:
|
|
278
|
+
# If we have an owner, we need to bind the method to the owner (for classmethods or staticmethods)
|
|
279
|
+
fn = self._underlying_obj.__get__(None, owner)
|
|
280
|
+
|
|
281
|
+
wrapper = _SyncWrapper(fn, bg_loop=self._bg_loop, underlying_obj=self._underlying_obj)
|
|
282
|
+
functools.update_wrapper(wrapper, self.fn)
|
|
283
|
+
return wrapper
|
|
284
|
+
|
|
285
|
+
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
|
286
|
+
if threading.current_thread().name == self._bg_loop.thread.name:
|
|
287
|
+
# If we are already in the background loop thread, we can call the function directly
|
|
288
|
+
raise AssertionError(
|
|
289
|
+
f"Deadlock detected: blocking call used in syncify thread {self._bg_loop.thread.name} "
|
|
290
|
+
f"when calling function {self.fn}, use .aio() if in an async call."
|
|
291
|
+
)
|
|
292
|
+
try:
|
|
293
|
+
# bind method if needed
|
|
294
|
+
coro_fn = self.fn
|
|
295
|
+
|
|
296
|
+
if inspect.isasyncgenfunction(coro_fn):
|
|
297
|
+
# Handle async iterator by converting to sync iterator
|
|
298
|
+
async_gen = coro_fn(*args, **kwargs)
|
|
299
|
+
return self._bg_loop.iterate_in_loop_sync(async_gen)
|
|
300
|
+
else:
|
|
301
|
+
return self._bg_loop.call_in_loop_sync(coro_fn(*args, **kwargs))
|
|
302
|
+
except Exception as e:
|
|
303
|
+
if logger.getEffectiveLevel() > logging.DEBUG:
|
|
304
|
+
# If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the user
|
|
305
|
+
# This is because the stack trace will include the Syncify wrapper and the background loop thread
|
|
306
|
+
tb = e.__traceback__
|
|
307
|
+
while tb and tb.tb_next:
|
|
308
|
+
if tb.tb_frame.f_code.co_name == self.fn.__name__:
|
|
309
|
+
break
|
|
310
|
+
tb = tb.tb_next
|
|
311
|
+
raise e.with_traceback(tb)
|
|
312
|
+
# If the log level is DEBUG, we will keep the extra stack frames to help with debugging
|
|
313
|
+
raise e
|
|
314
|
+
|
|
315
|
+
def aio(self, *args: Any, **kwargs: Any) -> Any:
|
|
316
|
+
fn = self.fn
|
|
317
|
+
|
|
318
|
+
try:
|
|
319
|
+
if inspect.isasyncgenfunction(fn):
|
|
320
|
+
# If the function is an async generator, we need to handle it differently
|
|
321
|
+
async_iter = fn(*args, **kwargs)
|
|
322
|
+
return self._bg_loop.iterate_in_loop(async_iter)
|
|
323
|
+
else:
|
|
324
|
+
# If we are already in the background loop, just return the coroutine
|
|
325
|
+
coro = fn(*args, **kwargs)
|
|
326
|
+
if hasattr(coro, "__aiter__"):
|
|
327
|
+
# If the coroutine is an async iterator, we need to handle it differently
|
|
328
|
+
return self._bg_loop.iterate_in_loop(coro)
|
|
329
|
+
return self._bg_loop.aio(coro)
|
|
330
|
+
except Exception as e:
|
|
331
|
+
if logger.getEffectiveLevel() > logging.DEBUG:
|
|
332
|
+
# If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the user
|
|
333
|
+
# This is because the stack trace will include the Syncify wrapper and the background loop thread
|
|
334
|
+
tb = e.__traceback__
|
|
335
|
+
while tb and tb.tb_next:
|
|
336
|
+
if tb.tb_frame.f_code.co_name == self.fn.__name__:
|
|
337
|
+
break
|
|
338
|
+
tb = tb.tb_next
|
|
339
|
+
raise e.with_traceback(tb)
|
|
340
|
+
# If the log level is DEBUG, we will keep the extra stack frames to help with debugging
|
|
341
|
+
raise e
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class Syncify:
|
|
345
|
+
"""
|
|
346
|
+
A decorator to convert asynchronous functions or methods into synchronous ones.
|
|
347
|
+
|
|
348
|
+
This is useful for integrating async code into synchronous contexts.
|
|
349
|
+
|
|
350
|
+
Example::
|
|
351
|
+
|
|
352
|
+
```python
|
|
353
|
+
syncer = Syncify()
|
|
354
|
+
|
|
355
|
+
@syncer
|
|
356
|
+
async def async_function(x: str) -> str:
|
|
357
|
+
return f"Hello, Async World {x}!"
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# now you can call it synchronously
|
|
361
|
+
result = async_function("Async World")
|
|
362
|
+
print(result)
|
|
363
|
+
# Output: Hello, Async World Async World!
|
|
364
|
+
|
|
365
|
+
# or call it asynchronously
|
|
366
|
+
async def main():
|
|
367
|
+
result = await async_function.aio("World")
|
|
368
|
+
print(result)
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
"""
|
|
372
|
+
|
|
373
|
+
def __init__(self, name: str = "flyte_syncify"):
|
|
374
|
+
self._bg_loop = _BackgroundLoop(name=name)
|
|
375
|
+
|
|
376
|
+
@overload
|
|
377
|
+
def __call__(self, func: Callable[P, Awaitable[R_co]]) -> SyncFunction[P, R_co]: ...
|
|
378
|
+
|
|
379
|
+
@overload
|
|
380
|
+
def __call__(self, func: Callable[P, Iterator[R_co] | AsyncIterator[R_co]]) -> SyncGenFunction[P, R_co]: ...
|
|
381
|
+
|
|
382
|
+
# def __call__(self, func: Callable[[Type[T], *P.args, *P.kwargs], Awaitable[R_co]])
|
|
383
|
+
# -> SyncFunction[[Type[T], *P.args, *P.kwargs], R_co]: ...
|
|
384
|
+
@overload
|
|
385
|
+
def __call__(self, func: classmethod) -> Union[SyncFunction[P, R_co], SyncGenFunction[P, R_co]]: ...
|
|
386
|
+
|
|
387
|
+
@overload
|
|
388
|
+
def __call__(self, func: staticmethod) -> staticmethod: ...
|
|
389
|
+
|
|
390
|
+
def __call__(self, obj):
|
|
391
|
+
if isinstance(obj, classmethod):
|
|
392
|
+
wrapper = _SyncWrapper(obj.__func__, bg_loop=self._bg_loop, underlying_obj=obj)
|
|
393
|
+
functools.update_wrapper(wrapper, obj.__func__)
|
|
394
|
+
return wrapper
|
|
395
|
+
|
|
396
|
+
if isinstance(obj, staticmethod):
|
|
397
|
+
fn = obj.__func__
|
|
398
|
+
wrapper = _SyncWrapper(fn, bg_loop=self._bg_loop)
|
|
399
|
+
functools.update_wrapper(wrapper, fn)
|
|
400
|
+
return staticmethod(wrapper)
|
|
401
|
+
|
|
402
|
+
if inspect.isasyncgenfunction(obj):
|
|
403
|
+
wrapper = _SyncWrapper(obj, bg_loop=self._bg_loop)
|
|
404
|
+
functools.update_wrapper(wrapper, obj)
|
|
405
|
+
return cast(Callable[P, Iterator[R_co]], wrapper)
|
|
406
|
+
|
|
407
|
+
if inspect.iscoroutinefunction(obj):
|
|
408
|
+
wrapper = _SyncWrapper(obj, bg_loop=self._bg_loop)
|
|
409
|
+
functools.update_wrapper(wrapper, obj)
|
|
410
|
+
return wrapper
|
|
411
|
+
|
|
412
|
+
raise TypeError(
|
|
413
|
+
"Syncify can only be applied to async functions, async generators, async classmethods or staticmethods."
|
|
414
|
+
)
|
flyte/types/__init__.py
CHANGED
|
@@ -1,9 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
# Flyte Type System
|
|
3
|
+
|
|
4
|
+
The Flyte type system provides a way to define, transform, and manipulate types in Flyte workflows.
|
|
5
|
+
Since the data flowing through Flyte has to often cross process, container and langauge boundaries, the type system
|
|
6
|
+
is designed to be serializable to a universal format that can be understood across different environments. This
|
|
7
|
+
universal format is based on Protocol Buffers. The types are called LiteralTypes and the runtime
|
|
8
|
+
representation of data is called Literals.
|
|
9
|
+
|
|
10
|
+
The type system includes:
|
|
11
|
+
- **TypeEngine**: The core engine that manages type transformations and serialization. This is the main entry point for
|
|
12
|
+
for all the internal type transformations and serialization logic.
|
|
13
|
+
- **TypeTransformer**: A class that defines how to transform one type to another. This is extensible
|
|
14
|
+
allowing users to define custom types and transformations.
|
|
15
|
+
- **Renderable**: An interface for types that can be rendered as HTML, that can be outputted to a flyte.report.
|
|
16
|
+
|
|
17
|
+
It is always possible to bypass the type system and use the `FlytePickle` type to serialize any python object
|
|
18
|
+
into a pickle format. The pickle format is not human-readable, but can be passed between flyte tasks that are
|
|
19
|
+
written in python. The Pickled objects cannot be represented in the UI, and may be in-efficient for large datasets.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from importlib.metadata import entry_points
|
|
23
|
+
|
|
24
|
+
from flyte._logging import logger
|
|
25
|
+
|
|
1
26
|
from ._interface import guess_interface
|
|
27
|
+
from ._pickle import FlytePickle
|
|
2
28
|
from ._renderer import Renderable
|
|
3
29
|
from ._string_literals import literal_string_repr
|
|
4
30
|
from ._type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError
|
|
5
31
|
|
|
6
32
|
__all__ = [
|
|
33
|
+
"FlytePickle",
|
|
7
34
|
"Renderable",
|
|
8
35
|
"TypeEngine",
|
|
9
36
|
"TypeTransformer",
|
|
@@ -11,3 +38,15 @@ __all__ = [
|
|
|
11
38
|
"guess_interface",
|
|
12
39
|
"literal_string_repr",
|
|
13
40
|
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _load_custom_type_transformers():
|
|
44
|
+
plugins = entry_points(group="flyte.plugins.types")
|
|
45
|
+
for ep in plugins:
|
|
46
|
+
try:
|
|
47
|
+
logger.info(f"Loading type transformer: {ep.name}")
|
|
48
|
+
loaded = ep.load()
|
|
49
|
+
if callable(loaded):
|
|
50
|
+
loaded()
|
|
51
|
+
except Exception as e:
|
|
52
|
+
logger.warning(f"Failed to load type transformer {ep.name} with error: {e}")
|
flyte/types/_interface.py
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
|
|
1
|
+
import inspect
|
|
2
|
+
from typing import Any, Dict, Iterable, Tuple, Type, cast
|
|
2
3
|
|
|
3
|
-
from
|
|
4
|
+
from flyteidl2.core import interface_pb2, literals_pb2
|
|
5
|
+
from flyteidl2.task import common_pb2
|
|
4
6
|
|
|
5
|
-
from flyte.
|
|
7
|
+
from flyte.models import NativeInterface
|
|
6
8
|
|
|
7
9
|
|
|
8
|
-
def guess_interface(
|
|
10
|
+
def guess_interface(
|
|
11
|
+
interface: interface_pb2.TypedInterface, default_inputs: Iterable[common_pb2.NamedParameter] | None = None
|
|
12
|
+
) -> NativeInterface:
|
|
9
13
|
"""
|
|
10
14
|
Returns the interface of the task with guessed types, as types may not be present in current env.
|
|
11
15
|
"""
|
|
@@ -14,12 +18,23 @@ def guess_interface(interface: interface_pb2.TypedInterface) -> NativeInterface:
|
|
|
14
18
|
if interface is None:
|
|
15
19
|
return NativeInterface({}, {})
|
|
16
20
|
|
|
17
|
-
|
|
21
|
+
default_input_literals: Dict[str, literals_pb2.Literal] = {}
|
|
22
|
+
if default_inputs is not None:
|
|
23
|
+
for param in default_inputs:
|
|
24
|
+
if param.parameter.HasField("default"):
|
|
25
|
+
default_input_literals[param.name] = param.parameter.default
|
|
26
|
+
|
|
27
|
+
guessed_inputs: Dict[str, Tuple[Type[Any], Any] | Any] = {}
|
|
18
28
|
if interface.inputs is not None and len(interface.inputs.variables) > 0:
|
|
19
|
-
|
|
29
|
+
input_types = flyte.types.TypeEngine.guess_python_types(cast(dict, interface.inputs.variables))
|
|
30
|
+
for name, t in input_types.items():
|
|
31
|
+
if name not in default_input_literals:
|
|
32
|
+
guessed_inputs[name] = (t, inspect.Parameter.empty)
|
|
33
|
+
else:
|
|
34
|
+
guessed_inputs[name] = (t, NativeInterface.has_default)
|
|
20
35
|
|
|
21
36
|
guessed_outputs: Dict[str, Type[Any]] = {}
|
|
22
37
|
if interface.outputs is not None and len(interface.outputs.variables) > 0:
|
|
23
38
|
guessed_outputs = flyte.types.TypeEngine.guess_python_types(cast(dict, interface.outputs.variables))
|
|
24
39
|
|
|
25
|
-
return NativeInterface.from_types(guessed_inputs, guessed_outputs)
|
|
40
|
+
return NativeInterface.from_types(guessed_inputs, guessed_outputs, default_input_literals)
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
import os
|
|
3
|
+
import sys
|
|
3
4
|
import typing
|
|
4
5
|
from typing import Type
|
|
5
6
|
|
|
6
7
|
import aiofiles
|
|
7
8
|
import cloudpickle
|
|
8
|
-
from
|
|
9
|
+
from flyteidl2.core import literals_pb2, types_pb2
|
|
9
10
|
|
|
10
11
|
import flyte.storage as storage
|
|
11
|
-
|
|
12
|
+
|
|
13
|
+
from ._type_engine import TypeEngine, TypeTransformer
|
|
12
14
|
|
|
13
15
|
T = typing.TypeVar("T")
|
|
16
|
+
DEFAULT_PICKLE_BYTES_LIMIT = 2**10 * 10 # 10KB
|
|
14
17
|
|
|
15
18
|
|
|
16
19
|
class FlytePickle(typing.Generic[T]):
|
|
@@ -75,8 +78,13 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
|
|
|
75
78
|
...
|
|
76
79
|
|
|
77
80
|
async def to_python_value(self, lv: literals_pb2.Literal, expected_python_type: Type[T]) -> T:
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
if lv.scalar.blob and lv.scalar.blob.uri:
|
|
82
|
+
uri = lv.scalar.blob.uri
|
|
83
|
+
return await FlytePickle.from_pickle(uri)
|
|
84
|
+
elif lv.scalar.binary and lv.scalar.binary.value:
|
|
85
|
+
return cloudpickle.loads(lv.scalar.binary.value)
|
|
86
|
+
else:
|
|
87
|
+
raise ValueError(f"Cannot convert {lv} to {expected_python_type}")
|
|
80
88
|
|
|
81
89
|
async def to_literal(
|
|
82
90
|
self,
|
|
@@ -91,8 +99,15 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
|
|
|
91
99
|
format=self.PYTHON_PICKLE_FORMAT, dimensionality=types_pb2.BlobType.BlobDimensionality.SINGLE
|
|
92
100
|
)
|
|
93
101
|
)
|
|
94
|
-
|
|
95
|
-
|
|
102
|
+
if sys.getsizeof(python_val) > DEFAULT_PICKLE_BYTES_LIMIT:
|
|
103
|
+
remote_path = await FlytePickle.to_pickle(python_val)
|
|
104
|
+
return literals_pb2.Literal(
|
|
105
|
+
scalar=literals_pb2.Scalar(blob=literals_pb2.Blob(metadata=meta, uri=remote_path))
|
|
106
|
+
)
|
|
107
|
+
else:
|
|
108
|
+
return literals_pb2.Literal(
|
|
109
|
+
scalar=literals_pb2.Scalar(binary=literals_pb2.Binary(value=cloudpickle.dumps(python_val)))
|
|
110
|
+
)
|
|
96
111
|
|
|
97
112
|
def guess_python_type(self, literal_type: types_pb2.LiteralType) -> typing.Type[FlytePickle[typing.Any]]:
|
|
98
113
|
if (
|
|
@@ -101,14 +116,27 @@ class FlytePickleTransformer(TypeTransformer[FlytePickle]):
|
|
|
101
116
|
and literal_type.blob.format == FlytePickleTransformer.PYTHON_PICKLE_FORMAT
|
|
102
117
|
):
|
|
103
118
|
return FlytePickle
|
|
119
|
+
if literal_type.simple == types_pb2.SimpleType.BINARY:
|
|
120
|
+
return FlytePickle
|
|
104
121
|
|
|
105
122
|
raise ValueError(f"Transformer {self} cannot reverse {literal_type}")
|
|
106
123
|
|
|
107
124
|
def get_literal_type(self, t: Type[T]) -> types_pb2.LiteralType:
|
|
108
125
|
lt = types_pb2.LiteralType(
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
126
|
+
union_type=types_pb2.UnionType(
|
|
127
|
+
variants=[
|
|
128
|
+
types_pb2.LiteralType(
|
|
129
|
+
blob=types_pb2.BlobType(
|
|
130
|
+
format=self.PYTHON_PICKLE_FORMAT,
|
|
131
|
+
dimensionality=types_pb2.BlobType.BlobDimensionality.SINGLE,
|
|
132
|
+
),
|
|
133
|
+
structure=types_pb2.TypeStructure(tag=self.name),
|
|
134
|
+
),
|
|
135
|
+
types_pb2.LiteralType(
|
|
136
|
+
simple=types_pb2.SimpleType.BINARY, structure=types_pb2.TypeStructure(tag=self.name)
|
|
137
|
+
),
|
|
138
|
+
]
|
|
139
|
+
),
|
|
112
140
|
)
|
|
113
141
|
lt.metadata = {"python_class_name": str(t)}
|
|
114
142
|
return lt
|
flyte/types/_string_literals.py
CHANGED
|
@@ -3,11 +3,10 @@ import json
|
|
|
3
3
|
from typing import Any, Dict, Union
|
|
4
4
|
|
|
5
5
|
import msgpack
|
|
6
|
-
from
|
|
6
|
+
from flyteidl2.core import literals_pb2
|
|
7
|
+
from flyteidl2.task import common_pb2
|
|
7
8
|
from google.protobuf.json_format import MessageToDict
|
|
8
9
|
|
|
9
|
-
from flyte._protos.workflow import run_definition_pb2
|
|
10
|
-
|
|
11
10
|
|
|
12
11
|
def _primitive_to_string(primitive: literals_pb2.Primitive) -> Any:
|
|
13
12
|
"""
|
|
@@ -88,9 +87,9 @@ def _dict_literal_repr(lmd: Dict[str, literals_pb2.Literal]) -> Dict[str, Any]:
|
|
|
88
87
|
def literal_string_repr(
|
|
89
88
|
lm: Union[
|
|
90
89
|
literals_pb2.Literal,
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
90
|
+
common_pb2.NamedLiteral,
|
|
91
|
+
common_pb2.Inputs,
|
|
92
|
+
common_pb2.Outputs,
|
|
94
93
|
literals_pb2.LiteralMap,
|
|
95
94
|
Dict[str, literals_pb2.Literal],
|
|
96
95
|
],
|
|
@@ -105,13 +104,13 @@ def literal_string_repr(
|
|
|
105
104
|
return _literal_string_repr(lm)
|
|
106
105
|
case literals_pb2.LiteralMap():
|
|
107
106
|
return _dict_literal_repr(lm.literals)
|
|
108
|
-
case
|
|
107
|
+
case common_pb2.NamedLiteral():
|
|
109
108
|
lmd = {lm.name: lm.value}
|
|
110
109
|
return _dict_literal_repr(lmd)
|
|
111
|
-
case
|
|
110
|
+
case common_pb2.Inputs():
|
|
112
111
|
lmd = {n.name: n.value for n in lm.literals}
|
|
113
112
|
return _dict_literal_repr(lmd)
|
|
114
|
-
case
|
|
113
|
+
case common_pb2.Outputs():
|
|
115
114
|
lmd = {n.name: n.value for n in lm.literals}
|
|
116
115
|
return _dict_literal_repr(lmd)
|
|
117
116
|
case dict():
|