indexify 0.3.17__py3-none-any.whl → 0.3.19__py3-none-any.whl

indexify/proto/executor_api_pb2_grpc.py

@@ -0,0 +1,227 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import warnings
+
+import grpc
+
+from indexify.proto import executor_api_pb2 as indexify_dot_proto_dot_executor__api__pb2
+
+GRPC_GENERATED_VERSION = "1.70.0"
+GRPC_VERSION = grpc.__version__
+_version_not_supported = False
+
+try:
+    from grpc._utilities import first_version_is_lower
+
+    _version_not_supported = first_version_is_lower(
+        GRPC_VERSION, GRPC_GENERATED_VERSION
+    )
+except ImportError:
+    _version_not_supported = True
+
+if _version_not_supported:
+    raise RuntimeError(
+        f"The grpc package installed is at version {GRPC_VERSION},"
+        + f" but the generated code in indexify/proto/executor_api_pb2_grpc.py depends on"
+        + f" grpcio>={GRPC_GENERATED_VERSION}."
+        + f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}"
+        + f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}."
+    )
+
+
+class ExecutorAPIStub(object):
+    """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
+    Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
+
+    Rename with caution. Existing clients won't find the service if the service name changes. A HTTP2 ingress proxy
+    might use the service name in it HTTP2 path based routing rules. See how gRPC uses service names in its HTTP2 paths
+    at https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.report_executor_state = channel.unary_unary(
+            "/executor_api_pb.ExecutorAPI/report_executor_state",
+            request_serializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateRequest.SerializeToString,
+            response_deserializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateResponse.FromString,
+            _registered_method=True,
+        )
+        self.get_desired_executor_states = channel.unary_stream(
+            "/executor_api_pb.ExecutorAPI/get_desired_executor_states",
+            request_serializer=indexify_dot_proto_dot_executor__api__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
+            response_deserializer=indexify_dot_proto_dot_executor__api__pb2.DesiredExecutorState.FromString,
+            _registered_method=True,
+        )
+        self.report_task_outcome = channel.unary_unary(
+            "/executor_api_pb.ExecutorAPI/report_task_outcome",
+            request_serializer=indexify_dot_proto_dot_executor__api__pb2.ReportTaskOutcomeRequest.SerializeToString,
+            response_deserializer=indexify_dot_proto_dot_executor__api__pb2.ReportTaskOutcomeResponse.FromString,
+            _registered_method=True,
+        )
+
+
+class ExecutorAPIServicer(object):
+    """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
+    Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
+
+    Rename with caution. Existing clients won't find the service if the service name changes. A HTTP2 ingress proxy
+    might use the service name in it HTTP2 path based routing rules. See how gRPC uses service names in its HTTP2 paths
+    at https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
+    """
+
+    def report_executor_state(self, request, context):
+        """Called by Executor every 5 seconds to report that it's still alive and provide its current state.
+
+        Missing 3 reports will result in the Executor being deregistered by Server.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def get_desired_executor_states(self, request, context):
+        """Called by Executor to open a stream of its desired states. When Server wants Executor to change something
+        it puts a message on the stream with the new desired state of the Executor.
+
+        Deprecated HTTP API is used to download the serialized graph and task inputs.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def report_task_outcome(self, request, context):
+        """Report the outcome of a task."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+
+def add_ExecutorAPIServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        "report_executor_state": grpc.unary_unary_rpc_method_handler(
+            servicer.report_executor_state,
+            request_deserializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateRequest.FromString,
+            response_serializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateResponse.SerializeToString,
+        ),
+        "get_desired_executor_states": grpc.unary_stream_rpc_method_handler(
+            servicer.get_desired_executor_states,
+            request_deserializer=indexify_dot_proto_dot_executor__api__pb2.GetDesiredExecutorStatesRequest.FromString,
+            response_serializer=indexify_dot_proto_dot_executor__api__pb2.DesiredExecutorState.SerializeToString,
+        ),
+        "report_task_outcome": grpc.unary_unary_rpc_method_handler(
+            servicer.report_task_outcome,
+            request_deserializer=indexify_dot_proto_dot_executor__api__pb2.ReportTaskOutcomeRequest.FromString,
+            response_serializer=indexify_dot_proto_dot_executor__api__pb2.ReportTaskOutcomeResponse.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        "executor_api_pb.ExecutorAPI", rpc_method_handlers
+    )
+    server.add_generic_rpc_handlers((generic_handler,))
+    server.add_registered_method_handlers(
+        "executor_api_pb.ExecutorAPI", rpc_method_handlers
+    )
+
+
+# This class is part of an EXPERIMENTAL API.
+class ExecutorAPI(object):
+    """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
+    Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
+
+    Rename with caution. Existing clients won't find the service if the service name changes. A HTTP2 ingress proxy
+    might use the service name in it HTTP2 path based routing rules. See how gRPC uses service names in its HTTP2 paths
+    at https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
+    """
+
+    @staticmethod
+    def report_executor_state(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/executor_api_pb.ExecutorAPI/report_executor_state",
+            indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateRequest.SerializeToString,
+            indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True,
+        )
+
+    @staticmethod
+    def get_desired_executor_states(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_stream(
+            request,
+            target,
+            "/executor_api_pb.ExecutorAPI/get_desired_executor_states",
+            indexify_dot_proto_dot_executor__api__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
+            indexify_dot_proto_dot_executor__api__pb2.DesiredExecutorState.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True,
+        )
+
+    @staticmethod
+    def report_task_outcome(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/executor_api_pb.ExecutorAPI/report_task_outcome",
+            indexify_dot_proto_dot_executor__api__pb2.ReportTaskOutcomeRequest.SerializeToString,
+            indexify_dot_proto_dot_executor__api__pb2.ReportTaskOutcomeResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True,
+        )

{indexify-0.3.17.dist-info → indexify-0.3.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.17
+Version: 0.3.19
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.3.17.dist-info → indexify-0.3.19.dist-info}/RECORD

@@ -1,13 +1,14 @@
-indexify/cli/cli.py,sha256=-ngLINNhZ2Req18_dQ1rHEnusY2feQ-9gYxOP8jcorA,8768
+indexify/cli/cli.py,sha256=TgXC_ZZPT-MJA7CWH297Y4_5M5SDt2QslPUoMotp3Cc,8708
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
-indexify/executor/api_objects.py,sha256=oUlH-GQPuPmwgcBzMpI2HehXeElBTCULECk-oHiBHwU,1263
-indexify/executor/downloader.py,sha256=LkvAXfKxddnDzgfmwHcpDB_n795-eVKzn-hLjq4nUEM,9412
-indexify/executor/executor.py,sha256=FTp05YxuKlMUbI99OV7NkL3KuFD12evKcqxzb-fXWBk,14641
-indexify/executor/function_executor/function_executor.py,sha256=s1mc7g6b8ilc98Fp7RFElEBSLJl0UGNQY0iZzCpuR2A,11334
-indexify/executor/function_executor/function_executor_state.py,sha256=b2taGClg0BUnlD_rYGkpom6syXBMUp7UWWrjLrUCwyo,3966
-indexify/executor/function_executor/function_executor_states_container.py,sha256=RclJDJqIr8ywKipPBC6_idnPAqYi0dPa1d4QUAaXqbw,3460
-indexify/executor/function_executor/function_executor_status.py,sha256=U4p1fcdVWlHr7uPY7e7ZSb2_WelUmPeH-WgboQQ9mw4,3336
-indexify/executor/function_executor/health_checker.py,sha256=CPUWvvtJtJCwbEsfr_BOhSKkRki4qOoSk1oeyBosWz0,5464
+indexify/executor/api_objects.py,sha256=qKQMEjr18xIq7yjpnsLPnWXP4KsVQ9O76EsbYoaL_qQ,1507
+indexify/executor/downloader.py,sha256=59FD2LNUQvl6Ul6m59uTsgE0FWU5t6C2XflaaDcc2FA,9459
+indexify/executor/executor.py,sha256=jdsp9vP2q8y6rIAIRmwfTXBGW3D5cgaFjfy_tHetJ3k,16035
+indexify/executor/executor_flavor.py,sha256=uilzDQVVYlQGR1MVnrUC4NevUActDWHdnJkr38M6kTk,118
+indexify/executor/function_executor/function_executor.py,sha256=rmqJrz4yihIefVhfbIIu3K6HFjKU3eEvEy2xFICYPp4,11881
+indexify/executor/function_executor/function_executor_state.py,sha256=ljPm1IrRMJ8hFklwvFp7Xax2HMpUIOHm0DwOxxMcy7U,4336
+indexify/executor/function_executor/function_executor_states_container.py,sha256=wKehM_GXv3i0WkKNS72JVxXq6s60iqiOCX_Gm0qa9pw,3546
+indexify/executor/function_executor/function_executor_status.py,sha256=FUu5PcFC-kkVSCYGwzcEmvpPytoM--XUZ9ylVjv79S4,3416
+indexify/executor/function_executor/health_checker.py,sha256=Fvd1gmrcjyJqP-8vcsUxfnTHQIMNlHeMWCS70PAVr9E,6095
 indexify/executor/function_executor/invocation_state_client.py,sha256=p-xgM4__cHR1ApvMV9hShrGWee_Je0VDhICZUGjpQY4,9644
 indexify/executor/function_executor/metrics/function_executor.py,sha256=TDksxLRJr-P9ZKhF2Orsaxzzb4lVIBxFEjd_9Zv53Ng,6313
 indexify/executor/function_executor/metrics/function_executor_state.py,sha256=qheMhnoiYLiZB7ky5EyegfDy4Mr0Zh83bOE0gJ38YmU,1607
@@ -19,19 +20,22 @@ indexify/executor/function_executor/server/client_configuration.py,sha256=gOywMu
 indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
 indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=cP93a3t1AfGx8qwageNLVdTwG52UOwzYNbbyrPq2TUQ,1692
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
-indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=xbH0-73tBM-i0dh1x_k0fDrNAi8WgDKjqiQ9xveU4Zg,4214
-indexify/executor/function_executor/single_task_runner.py,sha256=iWnJsB2BGqdgAkrlJHbOvSIhVXc88X0AYbB2_o-bB-E,13547
+indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=g1AUbhOoPsdhp_50Ayahdyv1Ix5-nEBE8orOQfkATpM,4470
+indexify/executor/function_executor/single_task_runner.py,sha256=OY3a2znwtuv0_Pqn9r5ScGhekNn1OquznKmaGnQL71k,13979
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
-indexify/executor/function_executor/task_output.py,sha256=SQJSlrknB7Ylf5IOeINfBEgiplS5hAPJh1hYulhyvfU,1962
-indexify/executor/grpc/channel_creator.py,sha256=Z_DU212-wkaU_m-I14OBbWKVeHo3aG5vPmF_ebJaZGc,1849
-indexify/executor/grpc/metrics/channel_creator.py,sha256=k-WArgklmP5WhjcmFmrgRblB7yc3XlaOXO8owRyV-mw,649
+indexify/executor/function_executor/task_output.py,sha256=kBRy7eSYzxZ3FYQ9PhA56VtBmhJ7ywePICMx_Vx5ipo,3011
+indexify/executor/grpc/channel_manager.py,sha256=THamn5VghCxRkXDlu2WEXtC6-SNKGc0xoa718bw9A4k,6257
+indexify/executor/grpc/completed_tasks_container.py,sha256=0JhWBUyEQXurVWZ1UtxoPDLXceIrcnnTBlSmfG3v0Cs,898
+indexify/executor/grpc/function_executor_controller.py,sha256=pfxMgyxLR2FFbLjyY5Yib8RSaI_6YeQOdf_VscXLqN0,18645
+indexify/executor/grpc/metrics/channel_manager.py,sha256=k-WArgklmP5WhjcmFmrgRblB7yc3XlaOXO8owRyV-mw,649
 indexify/executor/grpc/metrics/state_reporter.py,sha256=GggBEjMzQUYIG95LtTS4fUg1u9jYowkaXoUXppAXucs,543
-indexify/executor/grpc/state_reconciler.py,sha256=g7Qi6t79vuldh4y0Ue2mfnU2Jj8J-rsDtQzNZZlbLfE,12973
-indexify/executor/grpc/state_reporter.py,sha256=kk1kqvuUfgNZQriWj2FWjkPyloLcWPq7cCuNCK7mb5I,7669
+indexify/executor/grpc/state_reconciler.py,sha256=XRaZxHy2bX-zIzbJCBg3io_ZlslnGeG61dOk0fhYdJw,12866
+indexify/executor/grpc/state_reporter.py,sha256=7Z3pFqtXucxqAGX3o004uVG6tYWflsnKx1H2ldfnGT8,10043
+indexify/executor/grpc/task_controller.py,sha256=fsUiABPuZhGAjdpSKm0AVNIyoRxlJVFd4XDo3GzXEIQ,18483
 indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sggsR4Yoe_Zc,2746
 indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
 indexify/executor/metrics/task_fetcher.py,sha256=iJEwCLzYr2cuz7hRvNiqaa2nvQP4OrA0hm0iJY0YKG0,736
-indexify/executor/metrics/task_reporter.py,sha256=zUA9RpkSgx5lG_ZqDDuela5VuhtsnC0IKoQcEvHND0Y,730
+indexify/executor/metrics/task_reporter.py,sha256=44VT_c7njXlTtPl4xjlsYrIHpaiVAnvhhQdj56RPU6o,1215
 indexify/executor/metrics/task_runner.py,sha256=ZGFrl7zzfUdgPZnklxRIbnv9wVcHIQRhOGNqn9V2hSk,2047
 indexify/executor/monitoring/function_allowlist.py,sha256=wUGeiv3aAGWMlQXzHXq9O6MVHby6Tu-zY4U0MyWiQu0,683
 indexify/executor/monitoring/handler.py,sha256=Cj1cu_LcsAP0tdviqNhoEtGm4h0OJAxxzW9C2YdNXYU,240
@@ -43,14 +47,14 @@ indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbD
 indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
 indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
-indexify/executor/task_fetcher.py,sha256=NpFfHgaY99bSL-K2D5kcDAMNUG2FArq0-qF_mgF-LBQ,3375
-indexify/executor/task_reporter.py,sha256=mYgwozUO95PEwYMmeeIS0-HfMrO4z3Nhy6IduMsMahM,7367
-indexify/executor/task_runner.py,sha256=1zYH03yS_FaFk9xXBl-ioM74-L2xdW3vHJt522mseds,7073
-indexify/proto/task_scheduler.proto,sha256=kxMIJCj1pXG-fHeJGHXlthZTsB1dy_yvshQLt0UJRTM,5672
-indexify/proto/task_scheduler_pb2.py,sha256=X97JBJZ2n6ToDtUlDjPFV66_vZ05-vO8wPATrpzAonA,9085
-indexify/proto/task_scheduler_pb2.pyi,sha256=aXrB7-eNwgchy2OVlvEfPXtr9EyYoU-sgbdSRVNEI8s,11357
-indexify/proto/task_scheduler_pb2_grpc.py,sha256=STtk9XrBzLbmWdLwpL55Obyf9ehUesfxxysxER32SEE,6854
-indexify-0.3.17.dist-info/METADATA,sha256=hREMWJfSrd4Vcclp2w8fcUnjtvkiXHw6jMCTWECKAtw,1158
-indexify-0.3.17.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.17.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.17.dist-info/RECORD,,
+indexify/executor/task_fetcher.py,sha256=p3iEsWyGi0ZMPAv0183smzOUD1KycQ_dXsyd9mpB9IU,3529
+indexify/executor/task_reporter.py,sha256=59Bgsjflxzo936KMOj70lGpDhwbUlQDuqqfHEy16kcw,11274
+indexify/executor/task_runner.py,sha256=2-f6Uupu_aVPg34G27yuiPTC6eE4XnT-qizpzPryzDI,7134
+indexify/proto/executor_api.proto,sha256=16jEP4VxSaEPWVBx3h2_rel3ltYdgW4SBLvRl-iUe78,8319
+indexify/proto/executor_api_pb2.py,sha256=dW4zAMZ-YU8aeU77-5ciQ0bTNWDt7v4m6CGQ2VdQZZI,12874
+indexify/proto/executor_api_pb2.pyi,sha256=6WHUUjsn5y7jJot_y_h3NXy2L04E8Xo-zpO9_tZJcAA,16851
+indexify/proto/executor_api_pb2_grpc.py,sha256=GGiDtyQlA2382E_ZyKUBYcWNEJHH_RlulieStKfkJXI,9514
+indexify-0.3.19.dist-info/METADATA,sha256=gii0ukZNSxaQ4jF1J1NtsQOSbisC4lCbIV7hjXSFiqk,1158
+indexify-0.3.19.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.19.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.19.dist-info/RECORD,,

indexify/executor/grpc/channel_creator.py

@@ -1,53 +0,0 @@
-import asyncio
-from typing import Any
-
-import grpc.aio
-
-from .metrics.channel_creator import (
-    metric_grpc_server_channel_creation_latency,
-    metric_grpc_server_channel_creation_retries,
-    metric_grpc_server_channel_creations,
-)
-
-_RETRY_INTERVAL_SEC = 5
-_CONNECT_TIMEOUT_SEC = 5
-
-
-class ChannelCreator:
-    def __init__(self, server_address: str, logger: Any):
-        self._logger = logger.bind(module=__name__)
-        self._server_address = server_address
-        self._is_shutdown = False
-
-    async def create(self) -> grpc.aio.Channel:
-        """Creates a channel to the gRPC server.
-
-        Blocks until the channel is ready.
-        Never raises any exceptions.
-        """
-        with metric_grpc_server_channel_creation_latency.time():
-            metric_grpc_server_channel_creations.inc()
-            while not self._is_shutdown:
-                try:
-                    channel = grpc.aio.insecure_channel(self._server_address)
-                    await asyncio.wait_for(
-                        channel.channel_ready(),
-                        timeout=_CONNECT_TIMEOUT_SEC,
-                    )
-                    return channel
-                except Exception:
-                    self._logger.error(
-                        f"failed establishing grpc server channel in {_CONNECT_TIMEOUT_SEC} sec, retrying in {_RETRY_INTERVAL_SEC} sec"
-                    )
-                    try:
-                        await channel.close()
-                    except Exception as e:
-                        self._logger.error(
-                            "failed closing not established channel", exc_info=e
-                        )
-
-                    metric_grpc_server_channel_creation_retries.inc()
-                    await asyncio.sleep(_RETRY_INTERVAL_SEC)
-
-    async def shutdown(self):
-        self._is_shutdown = True

indexify/proto/task_scheduler_pb2.py

@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# NO CHECKED-IN PROTOBUF GENCODE
-# source: indexify/proto/task_scheduler.proto
-# Protobuf Python Version: 5.29.0
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import runtime_version as _runtime_version
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-
-_runtime_version.ValidateProtobufRuntimeVersion(
-    _runtime_version.Domain.PUBLIC, 5, 29, 0, "", "indexify/proto/task_scheduler.proto"
-)
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n#indexify/proto/task_scheduler.proto\x12\x16task_scheduler_service"l\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x34\n\x05model\x18\x02 \x01(\x0e\x32 .task_scheduler_service.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc9\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12\x36\n\x03gpu\x18\x04 \x01(\x0b\x32$.task_scheduler_service.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xf4\x02\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12\x43\n\x0fresource_limits\x18\x08 \x01(\x0b\x32%.task_scheduler_service.HostResourcesH\x06\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limits"\xc6\x01\n\x15\x46unctionExecutorState\x12M\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32\x33.task_scheduler_service.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12\x43\n\x06status\x18\x02 \x01(\x0e\x32..task_scheduler_service.FunctionExecutorStatusH\x01\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_status"\xb3\x03\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x44\n\x0f\x65xecutor_status\x18\x03 \x01(\x0e\x32&.task_scheduler_service.ExecutorStatusH\x02\x88\x01\x01\x12\x42\n\x0e\x66ree_resources\x18\x04 \x01(\x0b\x32%.task_scheduler_service.HostResourcesH\x03\x88\x01\x01\x12\x42\n\x11\x61llowed_functions\x18\x05 \x03(\x0b\x32\'.task_scheduler_service.AllowedFunction\x12O\n\x18\x66unction_executor_states\x18\x06 \x03(\x0b\x32-.task_scheduler_service.FunctionExecutorStateB\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x12\n\x10_executor_statusB\x11\n\x0f_free_resources"s\n\x1aReportExecutorStateRequest\x12\x42\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32%.task_scheduler_service.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\x88\x03\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\tH\x08\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_ms"\x86\x01\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12/\n\x04task\x18\x02 \x01(\x0b\x32\x1c.task_scheduler_service.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xc7\x01\n\x14\x44\x65siredExecutorState\x12O\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32\x33.task_scheduler_service.FunctionExecutorDescription\x12@\n\x10task_allocations\x18\x02 \x03(\x0b\x32&.task_scheduler_service.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xa3\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05\x32\xa6\x02\n\x14TaskSchedulerService\x12\x82\x01\n\x15report_executor_state\x12\x32.task_scheduler_service.ReportExecutorStateRequest\x1a\x33.task_scheduler_service.ReportExecutorStateResponse"\x00\x12\x88\x01\n\x1bget_desired_executor_states\x12\x37.task_scheduler_service.GetDesiredExecutorStatesRequest\x1a,.task_scheduler_service.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
-)
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(
-    DESCRIPTOR, "indexify.proto.task_scheduler_pb2", _globals
-)
-if not _descriptor._USE_C_DESCRIPTORS:
-    DESCRIPTOR._loaded_options = None
-    _globals["_GPUMODEL"]._serialized_start = 2541
-    _globals["_GPUMODEL"]._serialized_end = 2931
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 2934
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 3353
-    _globals["_EXECUTORSTATUS"]._serialized_start = 3356
-    _globals["_EXECUTORSTATUS"]._serialized_end = 3551
-    _globals["_GPURESOURCES"]._serialized_start = 63
-    _globals["_GPURESOURCES"]._serialized_end = 171
-    _globals["_HOSTRESOURCES"]._serialized_start = 174
-    _globals["_HOSTRESOURCES"]._serialized_end = 375
-    _globals["_ALLOWEDFUNCTION"]._serialized_start = 378
-    _globals["_ALLOWEDFUNCTION"]._serialized_end = 565
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 568
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 940
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 943
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1141
-    _globals["_EXECUTORSTATE"]._serialized_start = 1144
-    _globals["_EXECUTORSTATE"]._serialized_end = 1579
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 1581
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 1696
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 1698
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 1727
-    _globals["_TASK"]._serialized_start = 1730
-    _globals["_TASK"]._serialized_end = 2122
-    _globals["_TASKALLOCATION"]._serialized_start = 2125
-    _globals["_TASKALLOCATION"]._serialized_end = 2259
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 2261
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 2336
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 2339
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 2538
-    _globals["_TASKSCHEDULERSERVICE"]._serialized_start = 3554
-    _globals["_TASKSCHEDULERSERVICE"]._serialized_end = 3848
-# @@protoc_insertion_point(module_scope)

indexify/proto/task_scheduler_pb2_grpc.py

@@ -1,170 +0,0 @@
-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-"""Client and server classes corresponding to protobuf-defined services."""
-import warnings
-
-import grpc
-
-from indexify.proto import (
-    task_scheduler_pb2 as indexify_dot_proto_dot_task__scheduler__pb2,
-)
-
-GRPC_GENERATED_VERSION = "1.70.0"
-GRPC_VERSION = grpc.__version__
-_version_not_supported = False
-
-try:
-    from grpc._utilities import first_version_is_lower
-
-    _version_not_supported = first_version_is_lower(
-        GRPC_VERSION, GRPC_GENERATED_VERSION
-    )
-except ImportError:
-    _version_not_supported = True
-
-if _version_not_supported:
-    raise RuntimeError(
-        f"The grpc package installed is at version {GRPC_VERSION},"
-        + f" but the generated code in indexify/proto/task_scheduler_pb2_grpc.py depends on"
-        + f" grpcio>={GRPC_GENERATED_VERSION}."
-        + f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}"
-        + f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}."
-    )
-
-
-class TaskSchedulerServiceStub(object):
-    """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
-    Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
-    """
-
-    def __init__(self, channel):
-        """Constructor.
-
-        Args:
-            channel: A grpc.Channel.
-        """
-        self.report_executor_state = channel.unary_unary(
-            "/task_scheduler_service.TaskSchedulerService/report_executor_state",
-            request_serializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.SerializeToString,
-            response_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.FromString,
-            _registered_method=True,
-        )
-        self.get_desired_executor_states = channel.unary_stream(
-            "/task_scheduler_service.TaskSchedulerService/get_desired_executor_states",
-            request_serializer=indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
-            response_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.FromString,
-            _registered_method=True,
-        )
-
-
-class TaskSchedulerServiceServicer(object):
-    """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
-    Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
-    """
-
-    def report_executor_state(self, request, context):
-        """Called by Executor every 5 seconds to report that it's still alive and provide its current state.
-
-        Missing 3 reports will result in the Executor being deregistered by Server.
-        """
-        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-        context.set_details("Method not implemented!")
-        raise NotImplementedError("Method not implemented!")
-
-    def get_desired_executor_states(self, request, context):
-        """Called by Executor to open a stream of its desired states. When Server wants Executor to change something
-        it puts a message on the stream with the new desired state of the Executor.
-
-        Deprecated HTTP API is used to download the serialized graph and task inputs.
-        """
-        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-        context.set_details("Method not implemented!")
-        raise NotImplementedError("Method not implemented!")
-
-
-def add_TaskSchedulerServiceServicer_to_server(servicer, server):
-    rpc_method_handlers = {
-        "report_executor_state": grpc.unary_unary_rpc_method_handler(
-            servicer.report_executor_state,
-            request_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.FromString,
-            response_serializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.SerializeToString,
-        ),
-        "get_desired_executor_states": grpc.unary_stream_rpc_method_handler(
-            servicer.get_desired_executor_states,
-            request_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.FromString,
-            response_serializer=indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.SerializeToString,
-        ),
-    }
-    generic_handler = grpc.method_handlers_generic_handler(
-        "task_scheduler_service.TaskSchedulerService", rpc_method_handlers
-    )
-    server.add_generic_rpc_handlers((generic_handler,))
-    server.add_registered_method_handlers(
-        "task_scheduler_service.TaskSchedulerService", rpc_method_handlers
-    )
-
-
-# This class is part of an EXPERIMENTAL API.
-class TaskSchedulerService(object):
-    """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
-    Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
-    """
-
-    @staticmethod
-    def report_executor_state(
-        request,
-        target,
-        options=(),
-        channel_credentials=None,
-        call_credentials=None,
-        insecure=False,
-        compression=None,
-        wait_for_ready=None,
-        timeout=None,
-        metadata=None,
-    ):
-        return grpc.experimental.unary_unary(
-            request,
-            target,
-            "/task_scheduler_service.TaskSchedulerService/report_executor_state",
-            indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.SerializeToString,
-            indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.FromString,
-            options,
-            channel_credentials,
-            insecure,
-            call_credentials,
-            compression,
-            wait_for_ready,
-            timeout,
-            metadata,
-            _registered_method=True,
-        )
-
-    @staticmethod
-    def get_desired_executor_states(
-        request,
-        target,
-        options=(),
-        channel_credentials=None,
-        call_credentials=None,
-        insecure=False,
-        compression=None,
-        wait_for_ready=None,
-        timeout=None,
-        metadata=None,
-    ):
-        return grpc.experimental.unary_stream(
-            request,
-            target,
-            "/task_scheduler_service.TaskSchedulerService/get_desired_executor_states",
-            indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
-            indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.FromString,
-            options,
-            channel_credentials,
-            insecure,
-            call_credentials,
-            compression,
-            wait_for_ready,
-            timeout,
-            metadata,
-            _registered_method=True,
-        )