flyte 2.0.0b19__py3-none-any.whl → 2.0.0b20__py3-none-any.whl

flyte/_internal/controllers/remote/__init__.py

@@ -54,7 +54,5 @@ def create_remote_controller(
 
     controller = RemoteController(
         client_coro=client_coro,
-        workers=10,
-        max_system_retries=5,
     )
     return controller

flyte/_internal/controllers/remote/_controller.py

@@ -117,9 +117,8 @@ class RemoteController(Controller):
     def __init__(
         self,
         client_coro: Awaitable[ClientSet],
-        workers: int,
-        max_system_retries: int,
-        default_parent_concurrency: int = 100,
+        workers: int = 20,
+        max_system_retries: int = 10,
     ):
         """ """
         super().__init__(
@@ -127,6 +126,7 @@ class RemoteController(Controller):
             workers=workers,
             max_system_retries=max_system_retries,
         )
+        default_parent_concurrency = int(os.getenv("_F_P_CNC", "100"))
         self._default_parent_concurrency = default_parent_concurrency
         self._parent_action_semaphore: DefaultDict[str, asyncio.Semaphore] = defaultdict(
             lambda: asyncio.Semaphore(default_parent_concurrency)

flyte/_internal/controllers/remote/_core.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import asyncio
+import os
 import sys
 import threading
 from asyncio import Event
 from typing import Awaitable, Coroutine, Optional
 
 import grpc.aio
+from aiolimiter import AsyncLimiter
 from google.protobuf.wrappers_pb2 import StringValue
 
 import flyte.errors
@@ -32,10 +34,10 @@ class Controller:
     def __init__(
         self,
         client_coro: Awaitable[ClientSet],
-        workers: int = 2,
-        max_system_retries: int = 5,
+        workers: int = 20,
+        max_system_retries: int = 10,
         resource_log_interval_sec: float = 10.0,
-        min_backoff_on_err_sec: float = 0.1,
+        min_backoff_on_err_sec: float = 0.5,
         thread_wait_timeout_sec: float = 5.0,
         enqueue_timeout_sec: float = 5.0,
     ):
@@ -53,14 +55,17 @@ class Controller:
         self._running = False
         self._resource_log_task = None
         self._workers = workers
-        self._max_retries = max_system_retries
+        self._max_retries = int(os.getenv("_F_MAX_RETRIES", max_system_retries))
         self._resource_log_interval = resource_log_interval_sec
         self._min_backoff_on_err = min_backoff_on_err_sec
+        self._max_backoff_on_err = float(os.getenv("_F_MAX_BFF_ON_ERR", "10.0"))
         self._thread_wait_timeout = thread_wait_timeout_sec
         self._client_coro = client_coro
         self._failure_event: Event | None = None
         self._enqueue_timeout = enqueue_timeout_sec
         self._informer_start_wait_timeout = thread_wait_timeout_sec
+        max_qps = int(os.getenv("_F_MAX_QPS", "100"))
+        self._rate_limiter = AsyncLimiter(max_qps, 1.0)
 
         # Thread management
         self._thread = None
@@ -194,15 +199,16 @@ class Controller:
         # We will wait for this to signal that the thread is ready
         # Signal the main thread that we're ready
         logger.debug("Background thread initialization complete")
-        self._thread_ready.set()
         if sys.version_info >= (3, 11):
             async with asyncio.TaskGroup() as tg:
                 for i in range(self._workers):
-                    tg.create_task(self._bg_run())
+                    tg.create_task(self._bg_run(f"worker-{i}"))
+                self._thread_ready.set()
         else:
             tasks = []
             for i in range(self._workers):
-                tasks.append(asyncio.create_task(self._bg_run()))
+                tasks.append(asyncio.create_task(self._bg_run(f"worker-{i}")))
+            self._thread_ready.set()
             await asyncio.gather(*tasks)
 
     def _bg_thread_target(self):
@@ -221,6 +227,7 @@ class Controller:
         except Exception as e:
             logger.error(f"Controller thread encountered an exception: {e}")
             self._set_exception(e)
+            self._failure_event.set()
         finally:
             if self._loop and self._loop.is_running():
                 self._loop.close()
@@ -292,21 +299,22 @@ class Controller:
         started = action.is_started()
         action.mark_cancelled()
         if started:
-            logger.info(f"Cancelling action: {action.name}")
-            try:
-                # TODO add support when the queue service supports aborting actions
-                # await self._queue_service.AbortQueuedAction(
-                #     queue_service_pb2.AbortQueuedActionRequest(action_id=action.action_id),
-                #     wait_for_ready=True,
-                # )
-                logger.info(f"Successfully cancelled action: {action.name}")
-            except grpc.aio.AioRpcError as e:
-                if e.code() in [
-                    grpc.StatusCode.NOT_FOUND,
-                    grpc.StatusCode.FAILED_PRECONDITION,
-                ]:
-                    logger.info(f"Action {action.name} not found, assumed completed or cancelled.")
-                    return
+            async with self._rate_limiter:
+                logger.info(f"Cancelling action: {action.name}")
+                try:
+                    # TODO add support when the queue service supports aborting actions
+                    # await self._queue_service.AbortQueuedAction(
+                    #     queue_service_pb2.AbortQueuedActionRequest(action_id=action.action_id),
+                    #     wait_for_ready=True,
+                    # )
+                    logger.info(f"Successfully cancelled action: {action.name}")
+                except grpc.aio.AioRpcError as e:
+                    if e.code() in [
+                        grpc.StatusCode.NOT_FOUND,
+                        grpc.StatusCode.FAILED_PRECONDITION,
+                    ]:
+                        logger.info(f"Action {action.name} not found, assumed completed or cancelled.")
+                        return
         else:
             # If the action is not started, we have to ensure it does not get launched
             logger.info(f"Action {action.name} is not started, no need to cancel.")
@@ -320,56 +328,69 @@ class Controller:
         Attempt to launch an action.
         """
         if not action.is_started():
-            task: queue_service_pb2.TaskAction | None = None
-            trace: queue_service_pb2.TraceAction | None = None
-            if action.type == "task":
-                if action.task is None:
-                    raise flyte.errors.RuntimeSystemError(
-                        "NoTaskSpec", "Task Spec not found, cannot launch Task Action."
+            async with self._rate_limiter:
+                task: queue_service_pb2.TaskAction | None = None
+                trace: queue_service_pb2.TraceAction | None = None
+                if action.type == "task":
+                    if action.task is None:
+                        raise flyte.errors.RuntimeSystemError(
+                            "NoTaskSpec", "Task Spec not found, cannot launch Task Action."
+                        )
+                    cache_key = None
+                    logger.info(f"Action {action.name} has cache version {action.cache_key}")
+                    if action.cache_key:
+                        cache_key = StringValue(value=action.cache_key)
+
+                    task = queue_service_pb2.TaskAction(
+                        id=task_definition_pb2.TaskIdentifier(
+                            version=action.task.task_template.id.version,
+                            org=action.task.task_template.id.org,
+                            project=action.task.task_template.id.project,
+                            domain=action.task.task_template.id.domain,
+                            name=action.task.task_template.id.name,
+                        ),
+                        spec=action.task,
+                        cache_key=cache_key,
                     )
-                cache_key = None
-                logger.info(f"Action {action.name} has cache version {action.cache_key}")
-                if action.cache_key:
-                    cache_key = StringValue(value=action.cache_key)
-
-                task = queue_service_pb2.TaskAction(
-                    id=task_definition_pb2.TaskIdentifier(
-                        version=action.task.task_template.id.version,
-                        org=action.task.task_template.id.org,
-                        project=action.task.task_template.id.project,
-                        domain=action.task.task_template.id.domain,
-                        name=action.task.task_template.id.name,
-                    ),
-                    spec=action.task,
-                    cache_key=cache_key,
-                )
-            elif action.type == "trace":
-                trace = action.trace
-
-            logger.debug(f"Attempting to launch action: {action.name}")
-            try:
-                await self._queue_service.EnqueueAction(
-                    queue_service_pb2.EnqueueActionRequest(
-                        action_id=action.action_id,
-                        parent_action_name=action.parent_action_name,
-                        task=task,
-                        trace=trace,
-                        input_uri=action.inputs_uri,
-                        run_output_base=action.run_output_base,
-                        group=action.group.name if action.group else None,
-                        # Subject is not used in the current implementation
-                    ),
-                    wait_for_ready=True,
-                    timeout=self._enqueue_timeout,
-                )
-                logger.info(f"Successfully launched action: {action.name}")
-            except grpc.aio.AioRpcError as e:
-                if e.code() == grpc.StatusCode.ALREADY_EXISTS:
-                    logger.info(f"Action {action.name} already exists, continuing to monitor.")
-                    return
-                logger.exception(f"Failed to launch action: {action.name} backing off...")
-                logger.debug(f"Action details: {action}")
-                raise e
+                elif action.type == "trace":
+                    trace = action.trace
+
+                logger.debug(f"Attempting to launch action: {action.name}")
+                try:
+                    await self._queue_service.EnqueueAction(
+                        queue_service_pb2.EnqueueActionRequest(
+                            action_id=action.action_id,
+                            parent_action_name=action.parent_action_name,
+                            task=task,
+                            trace=trace,
+                            input_uri=action.inputs_uri,
+                            run_output_base=action.run_output_base,
+                            group=action.group.name if action.group else None,
+                            # Subject is not used in the current implementation
+                        ),
+                        wait_for_ready=True,
+                        timeout=self._enqueue_timeout,
+                    )
+                    logger.info(f"Successfully launched action: {action.name}")
+                except grpc.aio.AioRpcError as e:
+                    if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                        logger.info(f"Action {action.name} already exists, continuing to monitor.")
+                        return
+                    if e.code() in [
+                        grpc.StatusCode.FAILED_PRECONDITION,
+                        grpc.StatusCode.INVALID_ARGUMENT,
+                        grpc.StatusCode.NOT_FOUND,
+                    ]:
+                        raise flyte.errors.RuntimeSystemError(
+                            e.code().name, f"Precondition failed: {e.details()}"
+                        ) from e
+                    # For all other errors, we will retry with backoff
+                    logger.exception(
+                        f"Failed to launch action: {action.name}, Code: {e.code()}, "
+                        f"Details {e.details()} backing off..."
+                    )
+                    logger.debug(f"Action details: {action}")
+                    raise flyte.errors.SlowDownError(f"Failed to launch action: {e.details()}") from e
 
     @log
     async def _bg_process(self, action: Action):
@@ -397,35 +418,42 @@ class Controller:
             await asyncio.sleep(self._resource_log_interval)
 
     @log
-    async def _bg_run(self):
+    async def _bg_run(self, worker_id: str):
         """Run loop with resource status logging"""
+        logger.info(f"Worker {worker_id} started")
         while self._running:
             logger.debug(f"{threading.current_thread().name} Waiting for resource")
             action = await self._shared_queue.get()
             logger.debug(f"{threading.current_thread().name} Got resource {action.name}")
             try:
                 await self._bg_process(action)
-            except Exception as e:
-                logger.error(f"Error in controller loop: {e}")
-                # TODO we need a better way of handling backoffs currently the entire worker coroutine backs off
-                await asyncio.sleep(self._min_backoff_on_err)
-                action.increment_retries()
+            except flyte.errors.SlowDownError as e:
+                action.retries += 1
                 if action.retries > self._max_retries:
-                    err = flyte.errors.RuntimeSystemError(
-                        code=type(e).__name__,
-                        message=f"Controller failed, system retries {action.retries}"
-                        f" crossed threshold {self._max_retries}",
-                    )
-                    err.__cause__ = e
-                    action.set_client_error(err)
-                    informer = await self._informers.get(
-                        run_name=action.run_name,
-                        parent_action_name=action.parent_action_name,
-                    )
-                    if informer:
-                        await informer.fire_completion_event(action.name)
-                else:
-                    await self._shared_queue.put(action)
+                    raise
+                backoff = min(self._min_backoff_on_err * (2 ** (action.retries - 1)), self._max_backoff_on_err)
+                logger.warning(
+                    f"[{worker_id}] Backing off for {backoff} [retry {action.retries}/{self._max_retries}] "
+                    f"on action {action.name} due to error: {e}"
+                )
+                await asyncio.sleep(backoff)
+                logger.warning(f"[{worker_id}] Retrying action {action.name} after backoff")
+                await self._shared_queue.put(action)
+            except Exception as e:
+                logger.error(f"[{worker_id}] Error in controller loop: {e}")
+                err = flyte.errors.RuntimeSystemError(
+                    code=type(e).__name__,
+                    message=f"Controller failed, system retries {action.retries} crossed threshold {self._max_retries}",
+                    worker=worker_id,
+                )
+                err.__cause__ = e
+                action.set_client_error(err)
+                informer = await self._informers.get(
+                    run_name=action.run_name,
+                    parent_action_name=action.parent_action_name,
+                )
+                if informer:
+                    await informer.fire_completion_event(action.name)
             finally:
                 self._shared_queue.task_done()
 

flyte/_internal/controllers/remote/_informer.py

@@ -132,8 +132,10 @@ class Informer:
         parent_action_name: str,
         shared_queue: Queue,
         client: Optional[StateService] = None,
-        watch_backoff_interval_sec: float = 1.0,
+        min_watch_backoff: float = 1.0,
+        max_watch_backoff: float = 30.0,
         watch_conn_timeout_sec: float = 5.0,
+        max_watch_retries: int = 10,
     ):
         self.name = self.mkname(run_name=run_id.name, parent_action_name=parent_action_name)
         self.parent_action_name = parent_action_name
@@ -144,8 +146,10 @@ class Informer:
         self._running = False
         self._watch_task: asyncio.Task | None = None
         self._ready = asyncio.Event()
-        self._watch_backoff_interval_sec = watch_backoff_interval_sec
+        self._min_watch_backoff = min_watch_backoff
+        self._max_watch_backoff = max_watch_backoff
         self._watch_conn_timeout_sec = watch_conn_timeout_sec
+        self._max_watch_retries = max_watch_retries
 
     @classmethod
     def mkname(cls, *, run_name: str, parent_action_name: str) -> str:
@@ -211,13 +215,16 @@ class Informer:
         """
         # sentinel = False
         retries = 0
-        max_retries = 5
         last_exc = None
         while self._running:
-            if retries >= max_retries:
-                logger.error(f"Informer watch failure retries crossed threshold {retries}/{max_retries}, exiting!")
+            if retries >= self._max_watch_retries:
+                logger.error(
+                    f"Informer watch failure retries crossed threshold {retries}/{self._max_watch_retries}, exiting!"
+                )
                 raise last_exc
             try:
+                if retries >= 1:
+                    logger.warning(f"Informer watch retrying, attempt {retries}/{self._max_watch_retries}")
                 watcher = self._client.Watch(
                     state_service_pb2.WatchRequest(
                         parent_action_id=identifier_pb2.ActionIdentifier(
@@ -252,7 +259,9 @@ class Informer:
                 logger.exception(f"Watch error: {self.name}", exc_info=e)
                 last_exc = e
                 retries += 1
-            await asyncio.sleep(self._watch_backoff_interval_sec)
+            backoff = min(self._min_watch_backoff * (2**retries), self._max_watch_backoff)
+            logger.warning(f"Watch for {self.name} failed, retrying in {backoff} seconds...")
+            await asyncio.sleep(backoff)
 
     @log
     async def start(self, timeout: Optional[float] = None) -> asyncio.Task:

flyte/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.0.0b19'
-__version_tuple__ = version_tuple = (2, 0, 0, 'b19')
+__version__ = version = '2.0.0b20'
+__version_tuple__ = version_tuple = (2, 0, 0, 'b20')
 
-__commit_id__ = commit_id = 'g172a8c6b7'
+__commit_id__ = commit_id = 'g5109b02e4'

flyte/errors.py

@@ -223,3 +223,12 @@ class RunAbortedError(RuntimeUserError):
 
     def __init__(self, message: str):
         super().__init__("RunAbortedError", message, "user")
+
+
+class SlowDownError(RuntimeUserError):
+    """
+    This error is raised when the user tries to access a resource that does not exist or is invalid.
+    """
+
+    def __init__(self, message: str):
+        super().__init__("SlowDownError", message, "user")

{flyte-2.0.0b19.dist-info → flyte-2.0.0b20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyte
-Version: 2.0.0b19
+Version: 2.0.0b20
 Summary: Add your description here
 Author-email: Ketan Umare <kumare3@users.noreply.github.com>
 Requires-Python: >=3.10
@@ -24,6 +24,7 @@ Requires-Dist: toml>=0.10.2
 Requires-Dist: async-lru>=2.0.5
 Requires-Dist: mashumaro
 Requires-Dist: dataclasses_json
+Requires-Dist: aiolimiter>=1.2.1
 Dynamic: license-file
 
 # Flyte 2 SDK 🚀

{flyte-2.0.0b19.dist-info → flyte-2.0.0b20.dist-info}/RECORD

@@ -25,8 +25,8 @@ flyte/_task_plugins.py,sha256=9MH3nFPOH_e8_92BT4sFk4oyAnj6GJFvaPYWaraX7yE,1037
 flyte/_timeout.py,sha256=zx5sFcbYmjJAJbZWSGzzX-BpC9HC7Jfs35T7vVhKwkk,1571
 flyte/_tools.py,sha256=lB3OiJSAhxzSMCYjLUF6nZjlFsmNpaRXtr3_Fefcxbg,747
 flyte/_trace.py,sha256=-BIprs2MbupWl3vsC_Pn33SV3fSVku1rUIsnwfmrIy0,5204
-flyte/_version.py,sha256=nQUNxDZJt__tA11kox5MN0Qw8wDm85cQSW36nTZmT7s,722
-flyte/errors.py,sha256=utl0biQhyYbNdkddVsJAGbGZgUPka_DYnUaHTlQL-Jc,6451
+flyte/_version.py,sha256=DHBNTfUVjKbnarRHJH69S1E8D0BHxMGpAixhCbYE448,722
+flyte/errors.py,sha256=k1-pgz7xm4syViJmPwp7ZDusmqcrlM2uZKrl0com_9I,6707
 flyte/extend.py,sha256=GB4ZedGzKa30vYWRVPOdxEeK62xnUVFY4z2tD6H9eEw,376
 flyte/models.py,sha256=dtaQyU4PhtEr5L39xwiPDVKSYuYKaArRfRNliIfPUf8,16207
 flyte/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -50,12 +50,12 @@ flyte/_internal/__init__.py,sha256=vjXgGzAAjy609YFkAy9_RVPuUlslsHSJBXCLNTVnqOY,1
 flyte/_internal/controllers/__init__.py,sha256=yNALzu-3YVii_D_GlrBoWUpL4PTLYUa7oIKnA7hbee0,4296
 flyte/_internal/controllers/_local_controller.py,sha256=lXFvmxd_ctinJ5z4Fd_bZW0_LcOn0vc12F1try6uYgg,7308
 flyte/_internal/controllers/_trace.py,sha256=ywFg_M2nGrCKYLbh4iVdsVlRtPT1K2S-XZMvDJU8AKU,1499
-flyte/_internal/controllers/remote/__init__.py,sha256=9_azH1eHLqY6VULpDugXi7Kf1kK1ODqEnsQ_3wM6IqU,1919
+flyte/_internal/controllers/remote/__init__.py,sha256=EkDZ4gpp715M5CMnwpFh9s04jkWowndF9RqhXJ34yRA,1869
 flyte/_internal/controllers/remote/_action.py,sha256=zYilKk3yq2jOxUm258xd4TRl1o3T2la4PXFc0xjcytI,7301
 flyte/_internal/controllers/remote/_client.py,sha256=HPbzbfaWZVv5wpOvKNtFXR6COiZDwd1cUJQqi60A7oU,1421
-flyte/_internal/controllers/remote/_controller.py,sha256=IZ74-PdpsDplRAMutvlwFVzrSjQdJJ-naUb2d2SLXKM,25175
-flyte/_internal/controllers/remote/_core.py,sha256=A7GX-V8LqFoF0Llfvxy3xmrRsiIHSJv2z2gdFWTHs48,19053
-flyte/_internal/controllers/remote/_informer.py,sha256=w4p29_dzS_ns762eNBljvnbJLgCm36d1Ogo2ZkgV1yg,14418
+flyte/_internal/controllers/remote/_controller.py,sha256=Mp65OvENDfJ0CpwoAhb1ndCdrJ2hMIccwI0_t8xD7uU,25209
+flyte/_internal/controllers/remote/_core.py,sha256=5qIGFNTilJwtYLQ_KQoawLQ172ih-uH6MfrQKVXqd8k,20828
+flyte/_internal/controllers/remote/_informer.py,sha256=M2uqfe0oz9ySkYK94UFLoTPRt9UoaYQKzmy4lrVOchY,14912
 flyte/_internal/controllers/remote/_service_protocol.py,sha256=B9qbIg6DiGeac-iSccLmX_AL2xUgX4ezNUOiAbSy4V0,1357
 flyte/_internal/imagebuild/__init__.py,sha256=dwXdJ1jMhw9RF8itF7jkPLanvX1yCviSns7hE5eoIts,102
 flyte/_internal/imagebuild/docker_builder.py,sha256=qzKqicgGpWB6wC2DYTc4yh5LJT_hbQ7nacIqnx4bkOQ,21595
@@ -238,11 +238,11 @@ flyte/types/_renderer.py,sha256=ygcCo5l60lHufyQISFddZfWwLlQ8kJAKxUT_XnR_6dY,4818
 flyte/types/_string_literals.py,sha256=NlG1xV8RSA-sZ-n-IFQCAsdB6jXJOAKkHWtnopxVVDk,4231
 flyte/types/_type_engine.py,sha256=IwBdBXr3sso2Y-ZMiIBmXmczfI7GyJMpThXDuQiOqco,94951
 flyte/types/_utils.py,sha256=pbts9E1_2LTdLygAY0UYTLYJ8AsN3BZyviSXvrtcutc,2626
-flyte-2.0.0b19.data/scripts/debug.py,sha256=hnX2tlv9QbqckoT5CJ3c3apJj3tGDpsrdV7ZAsE7j34,911
-flyte-2.0.0b19.data/scripts/runtime.py,sha256=CoMgKQ_pBifnHsNF1MlBGmMrenrKXuMIHTLRF7MKDfA,6193
-flyte-2.0.0b19.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-flyte-2.0.0b19.dist-info/METADATA,sha256=EYDtynzbTo44xv8-onc7OLPeT1giPi5XcW_k_b2SrII,10012
-flyte-2.0.0b19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flyte-2.0.0b19.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
-flyte-2.0.0b19.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
-flyte-2.0.0b19.dist-info/RECORD,,
+flyte-2.0.0b20.data/scripts/debug.py,sha256=hnX2tlv9QbqckoT5CJ3c3apJj3tGDpsrdV7ZAsE7j34,911
+flyte-2.0.0b20.data/scripts/runtime.py,sha256=CoMgKQ_pBifnHsNF1MlBGmMrenrKXuMIHTLRF7MKDfA,6193
+flyte-2.0.0b20.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+flyte-2.0.0b20.dist-info/METADATA,sha256=J44ZHZJFMHKtBfaCx69-eDwqCklCBSgS64uKMJqtfz4,10045
+flyte-2.0.0b20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flyte-2.0.0b20.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
+flyte-2.0.0b20.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
+flyte-2.0.0b20.dist-info/RECORD,,