hatchet-sdk 1.2.3__py3-none-any.whl → 1.2.4__py3-none-any.whl

hatchet_sdk/clients/admin.py

@@ -1,7 +1,7 @@
 import asyncio
 import json
 from datetime import datetime
-from typing import Union, cast
+from typing import Generator, TypeVar, Union, cast
 
 import grpc
 from google.protobuf import timestamp_pb2
@@ -27,6 +27,10 @@ from hatchet_sdk.utils.proto_enums import convert_python_enum_to_proto
 from hatchet_sdk.utils.typing import JSONSerializableMapping
 from hatchet_sdk.workflow_run import WorkflowRunRef
 
+T = TypeVar("T")
+
+MAX_BULK_WORKFLOW_RUN_BATCH_SIZE = 1000
+
 
 class ScheduleTriggerWorkflowOptions(BaseModel):
     parent_id: str | None = None
@@ -328,9 +332,6 @@ class AdminClient:
         input: JSONSerializableMapping,
         options: TriggerWorkflowOptions = TriggerWorkflowOptions(),
     ) -> WorkflowRunRef:
-        ## IMPORTANT: The `pooled_workflow_listener` must be created 1) lazily, and not at `init` time, and 2) on the
-        ## main thread. If 1) is not followed, you'll get an error about something being attached to the wrong event
-        ## loop. If 2) is not followed, you'll get an error about the event loop not being set up.
         async with spawn_index_lock:
             request = self._create_workflow_run_request(workflow_name, input, options)
 
@@ -353,70 +354,90 @@ class AdminClient:
             config=self.config,
         )
 
+    def chunk(self, xs: list[T], n: int) -> Generator[list[T], None, None]:
+        for i in range(0, len(xs), n):
+            yield xs[i : i + n]
+
     ## IMPORTANT: Keep this method's signature in sync with the wrapper in the OTel instrumentor
     @tenacity_retry
     def run_workflows(
         self,
         workflows: list[WorkflowRunTriggerConfig],
     ) -> list[WorkflowRunRef]:
-        bulk_request = v0_workflow_protos.BulkTriggerWorkflowRequest(
-            workflows=[
-                self._create_workflow_run_request(
-                    workflow.workflow_name, workflow.input, workflow.options
-                )
-                for workflow in workflows
-            ]
-        )
+        bulk_workflows = [
+            self._create_workflow_run_request(
+                workflow.workflow_name, workflow.input, workflow.options
+            )
+            for workflow in workflows
+        ]
 
-        resp = cast(
-            v0_workflow_protos.BulkTriggerWorkflowResponse,
-            self.v0_client.BulkTriggerWorkflow(
-                bulk_request,
-                metadata=get_metadata(self.token),
-            ),
-        )
+        refs: list[WorkflowRunRef] = []
 
-        return [
-            WorkflowRunRef(
-                workflow_run_id=workflow_run_id,
-                config=self.config,
+        for chunk in self.chunk(bulk_workflows, MAX_BULK_WORKFLOW_RUN_BATCH_SIZE):
+            bulk_request = v0_workflow_protos.BulkTriggerWorkflowRequest(
+                workflows=chunk
             )
-            for workflow_run_id in resp.workflow_run_ids
-        ]
+
+            resp = cast(
+                v0_workflow_protos.BulkTriggerWorkflowResponse,
+                self.v0_client.BulkTriggerWorkflow(
+                    bulk_request,
+                    metadata=get_metadata(self.token),
+                ),
+            )
+
+            refs.extend(
+                [
+                    WorkflowRunRef(
+                        workflow_run_id=workflow_run_id,
+                        config=self.config,
+                    )
+                    for workflow_run_id in resp.workflow_run_ids
+                ]
+            )
+
+        return refs
 
     @tenacity_retry
     async def aio_run_workflows(
         self,
         workflows: list[WorkflowRunTriggerConfig],
     ) -> list[WorkflowRunRef]:
-        ## IMPORTANT: The `pooled_workflow_listener` must be created 1) lazily, and not at `init` time, and 2) on the
-        ## main thread. If 1) is not followed, you'll get an error about something being attached to the wrong event
-        ## loop. If 2) is not followed, you'll get an error about the event loop not being set up.
-        async with spawn_index_lock:
-            bulk_request = v0_workflow_protos.BulkTriggerWorkflowRequest(
-                workflows=[
+        chunks = self.chunk(workflows, MAX_BULK_WORKFLOW_RUN_BATCH_SIZE)
+        refs: list[WorkflowRunRef] = []
+
+        for chunk in chunks:
+            async with spawn_index_lock:
+                bulk_workflows = [
                     self._create_workflow_run_request(
                         workflow.workflow_name, workflow.input, workflow.options
                     )
-                    for workflow in workflows
+                    for workflow in chunk
                 ]
+
+            bulk_request = v0_workflow_protos.BulkTriggerWorkflowRequest(
+                workflows=bulk_workflows
             )
 
-        resp = cast(
-            v0_workflow_protos.BulkTriggerWorkflowResponse,
-            self.v0_client.BulkTriggerWorkflow(
-                bulk_request,
-                metadata=get_metadata(self.token),
-            ),
-        )
+            resp = cast(
+                v0_workflow_protos.BulkTriggerWorkflowResponse,
+                self.v0_client.BulkTriggerWorkflow(
+                    bulk_request,
+                    metadata=get_metadata(self.token),
+                ),
+            )
 
-        return [
-            WorkflowRunRef(
-                workflow_run_id=workflow_run_id,
-                config=self.config,
+            refs.extend(
+                [
+                    WorkflowRunRef(
+                        workflow_run_id=workflow_run_id,
+                        config=self.config,
+                    )
+                    for workflow_run_id in resp.workflow_run_ids
+                ]
             )
-            for workflow_run_id in resp.workflow_run_ids
-        ]
+
+        return refs
 
     def get_workflow_run(self, workflow_run_id: str) -> WorkflowRunRef:
         return WorkflowRunRef(

{hatchet_sdk-1.2.3.dist-info → hatchet_sdk-1.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hatchet-sdk
-Version: 1.2.3
+Version: 1.2.4
 Summary: 
 Author: Alexander Belanger
 Author-email: alexander@hatchet.run

{hatchet_sdk-1.2.3.dist-info → hatchet_sdk-1.2.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
 hatchet_sdk/__init__.py,sha256=o_06wLLKCKRq4uQuCF62yDRb8hTQYYcqPC3FIDNHxuQ,10002
 hatchet_sdk/client.py,sha256=lApcV1-qlvIdiHyGrQJAIlT9qzP96I0mQ2STZ1ZV_2g,1940
-hatchet_sdk/clients/admin.py,sha256=cc-TyZL6uLz3MGpdVKYziiZlmi29twP8MUJ66CboLtU,15160
+hatchet_sdk/clients/admin.py,sha256=1pY3RlJmVjFAfxX6kwVMd7jO_JHxkei6jrz8XUJUaKI,15333
 hatchet_sdk/clients/dispatcher/action_listener.py,sha256=FogjHd6vhj4HeQs--Z-99kPwCfaETo1-EcotjdIkrig,16266
 hatchet_sdk/clients/dispatcher/dispatcher.py,sha256=7tHEoCvL2MKklxbT43oRsQdx9JuLIVCJvFuIUqTHjis,7794
 hatchet_sdk/clients/durable_event_listener.py,sha256=J5Tg8niikms-tEfB8wps_wgsPF7Fx-pYhYExd0lJo7Y,11791
@@ -506,7 +506,7 @@ hatchet_sdk/worker/runner/runner.py,sha256=ZocvR7dDgEvSxcHAWDDaoTasSz3Wio70TbEbB
 hatchet_sdk/worker/runner/utils/capture_logs.py,sha256=nHRPSiDBqzhObM7i2X7t03OupVFnE7kQBdR2Ckgg-2w,2709
 hatchet_sdk/worker/worker.py,sha256=qyHs64H-grF9HR1CgH7MlnoDmTQ8mm4d8basx-ZDyWc,14490
 hatchet_sdk/workflow_run.py,sha256=Q1nTpnWNsFfjWWpx49xXYUHsVbqTnHL6JWnSKoFM3_I,1029
-hatchet_sdk-1.2.3.dist-info/METADATA,sha256=ix70mJ0CDQ0s-8MQRt_1KaVyCozNUXqFzd5iDspc7GE,3571
-hatchet_sdk-1.2.3.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-hatchet_sdk-1.2.3.dist-info/entry_points.txt,sha256=5mTp_AsCWK5raiVxP_MU9eBCgkRGl4OsN6chpHcvm7o,1235
-hatchet_sdk-1.2.3.dist-info/RECORD,,
+hatchet_sdk-1.2.4.dist-info/METADATA,sha256=U6Iloh69aCy5R-Y5V47P61n_TNYHmiQPNLNKyBpmNdg,3571
+hatchet_sdk-1.2.4.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+hatchet_sdk-1.2.4.dist-info/entry_points.txt,sha256=5mTp_AsCWK5raiVxP_MU9eBCgkRGl4OsN6chpHcvm7o,1235
+hatchet_sdk-1.2.4.dist-info/RECORD,,