dbos 0.23.0a9__py3-none-any.whl → 0.23.0a11__py3-none-any.whl

dbos/_conductor/conductor.py

@@ -0,0 +1,194 @@
+import threading
+import time
+import traceback
+from typing import TYPE_CHECKING, Optional
+
+from websockets import ConnectionClosed, ConnectionClosedOK
+from websockets.sync.client import connect
+from websockets.sync.connection import Connection
+
+from dbos._utils import GlobalParams
+from dbos._workflow_commands import list_queued_workflows, list_workflows
+
+from . import protocol as p
+
+if TYPE_CHECKING:
+    from dbos import DBOS
+
+
+class ConductorWebsocket(threading.Thread):
+
+    def __init__(
+        self, dbos: "DBOS", conductor_url: str, conductor_key: str, evt: threading.Event
+    ):
+        super().__init__(daemon=True)
+        self.websocket: Optional[Connection] = None
+        self.evt = evt
+        self.dbos = dbos
+        self.app_name = dbos.config["name"]
+        self.url = (
+            conductor_url.rstrip("/") + f"/websocket/{self.app_name}/{conductor_key}"
+        )
+
+    def run(self) -> None:
+        while not self.evt.is_set():
+            try:
+                with connect(self.url) as websocket:
+                    self.websocket = websocket
+                    while not self.evt.is_set():
+                        message = websocket.recv()
+                        if not isinstance(message, str):
+                            self.dbos.logger.warning(
+                                "Receieved unexpected non-str message"
+                            )
+                            continue
+                        base_message = p.BaseMessage.from_json(message)
+                        type = base_message.type
+                        if type == p.MessageType.EXECUTOR_INFO:
+                            info_response = p.ExecutorInfoResponse(
+                                type=p.MessageType.EXECUTOR_INFO,
+                                request_id=base_message.request_id,
+                                executor_id=GlobalParams.executor_id,
+                                application_version=GlobalParams.app_version,
+                            )
+                            websocket.send(info_response.to_json())
+                            self.dbos.logger.info("Connected to DBOS conductor")
+                        elif type == p.MessageType.RECOVERY:
+                            recovery_message = p.RecoveryRequest.from_json(message)
+                            success = True
+                            try:
+                                self.dbos.recover_pending_workflows(
+                                    recovery_message.executor_ids
+                                )
+                            except Exception as e:
+                                self.dbos.logger.error(
+                                    f"Exception encountered when recovering workflows: {traceback.format_exc()}"
+                                )
+                                success = False
+                            recovery_response = p.RecoveryResponse(
+                                type=p.MessageType.RECOVERY,
+                                request_id=base_message.request_id,
+                                success=success,
+                            )
+                            websocket.send(recovery_response.to_json())
+                        elif type == p.MessageType.CANCEL:
+                            cancel_message = p.CancelRequest.from_json(message)
+                            success = True
+                            try:
+                                self.dbos.cancel_workflow(cancel_message.workflow_id)
+                            except Exception as e:
+                                self.dbos.logger.error(
+                                    f"Exception encountered when cancelling workflow {cancel_message.workflow_id}: {traceback.format_exc()}"
+                                )
+                                success = False
+                            cancel_response = p.CancelResponse(
+                                type=p.MessageType.CANCEL,
+                                request_id=base_message.request_id,
+                                success=success,
+                            )
+                            websocket.send(cancel_response.to_json())
+                        elif type == p.MessageType.RESUME:
+                            resume_message = p.ResumeRequest.from_json(message)
+                            success = True
+                            try:
+                                self.dbos.resume_workflow(resume_message.workflow_id)
+                            except Exception as e:
+                                self.dbos.logger.error(
+                                    f"Exception encountered when resuming workflow {resume_message.workflow_id}: {traceback.format_exc()}"
+                                )
+                                success = False
+                            resume_response = p.ResumeResponse(
+                                type=p.MessageType.RESUME,
+                                request_id=base_message.request_id,
+                                success=success,
+                            )
+                            websocket.send(resume_response.to_json())
+                        elif type == p.MessageType.RESTART:
+                            restart_message = p.RestartRequest.from_json(message)
+                            success = True
+                            try:
+                                self.dbos.restart_workflow(restart_message.workflow_id)
+                            except Exception as e:
+                                self.dbos.logger.error(
+                                    f"Exception encountered when restarting workflow {restart_message.workflow_id}: {traceback.format_exc()}"
+                                )
+                                success = False
+                            restart_response = p.RestartResponse(
+                                type=p.MessageType.RESTART,
+                                request_id=base_message.request_id,
+                                success=success,
+                            )
+                            websocket.send(restart_response.to_json())
+                        elif type == p.MessageType.LIST_WORKFLOWS:
+                            list_workflows_message = p.ListWorkflowsRequest.from_json(
+                                message
+                            )
+                            body = list_workflows_message.body
+                            infos = list_workflows(
+                                self.dbos._sys_db,
+                                workflow_ids=body["workflow_uuids"],
+                                user=body["authenticated_user"],
+                                start_time=body["start_time"],
+                                end_time=body["end_time"],
+                                status=body["status"],
+                                request=False,
+                                app_version=body["application_version"],
+                                name=body["workflow_name"],
+                                limit=body["limit"],
+                                offset=body["offset"],
+                                sort_desc=body["sort_desc"],
+                            )
+                            list_workflows_response = p.ListWorkflowsResponse(
+                                type=p.MessageType.LIST_WORKFLOWS,
+                                request_id=base_message.request_id,
+                                output=[
+                                    p.WorkflowsOutput.from_workflow_information(i)
+                                    for i in infos
+                                ],
+                            )
+                            websocket.send(list_workflows_response.to_json())
+                        elif type == p.MessageType.LIST_QUEUED_WORKFLOWS:
+                            list_queued_workflows_message = (
+                                p.ListQueuedWorkflowsRequest.from_json(message)
+                            )
+                            q_body = list_queued_workflows_message.body
+                            infos = list_queued_workflows(
+                                self.dbos._sys_db,
+                                start_time=q_body["start_time"],
+                                end_time=q_body["end_time"],
+                                status=q_body["status"],
+                                request=False,
+                                name=q_body["workflow_name"],
+                                limit=q_body["limit"],
+                                offset=q_body["offset"],
+                                queue_name=q_body["queue_name"],
+                                sort_desc=q_body["sort_desc"],
+                            )
+                            list_queued_workflows_response = (
+                                p.ListQueuedWorkflowsResponse(
+                                    type=p.MessageType.LIST_QUEUED_WORKFLOWS,
+                                    request_id=base_message.request_id,
+                                    output=[
+                                        p.WorkflowsOutput.from_workflow_information(i)
+                                        for i in infos
+                                    ],
+                                )
+                            )
+                            websocket.send(list_queued_workflows_response.to_json())
+                        else:
+                            self.dbos.logger.warning(f"Unexpected message type: {type}")
+            except ConnectionClosedOK:
+                self.dbos.logger.info("Conductor connection terminated")
+                break
+            except ConnectionClosed as e:
+                self.dbos.logger.warning(
+                    f"Connection to conductor lost. Reconnecting: {e}"
+                )
+                time.sleep(1)
+                continue
+            except Exception as e:
+                self.dbos.logger.error(
+                    f"Unexpected exception in connection to conductor. Reconnecting: {e}"
+                )
+                time.sleep(1)
+                continue

dbos/_conductor/protocol.py

@@ -0,0 +1,186 @@
+import json
+from dataclasses import asdict, dataclass
+from enum import Enum
+from typing import List, Optional, Type, TypedDict, TypeVar
+
+from dbos._workflow_commands import WorkflowInformation
+
+
+class MessageType(str, Enum):
+    EXECUTOR_INFO = "executor_info"
+    RECOVERY = "recovery"
+    CANCEL = "cancel"
+    LIST_WORKFLOWS = "list_workflows"
+    LIST_QUEUED_WORKFLOWS = "list_queued_workflows"
+    RESUME = "resume"
+    RESTART = "restart"
+
+
+T = TypeVar("T", bound="BaseMessage")
+
+
+@dataclass
+class BaseMessage:
+    type: MessageType
+    request_id: str
+
+    @classmethod
+    def from_json(cls: Type[T], json_str: str) -> T:
+        """
+        Safely load a JSON into a dataclass, loading only the
+        attributes specified in the dataclass.
+        """
+        data = json.loads(json_str)
+        all_annotations = {}
+        for base_cls in cls.__mro__:
+            if hasattr(base_cls, "__annotations__"):
+                all_annotations.update(base_cls.__annotations__)
+        kwargs = {k: v for k, v in data.items() if k in all_annotations}
+        return cls(**kwargs)
+
+    def to_json(self) -> str:
+        dict_data = asdict(self)
+        return json.dumps(dict_data)
+
+
+@dataclass
+class ExecutorInfoRequest(BaseMessage):
+    pass
+
+
+@dataclass
+class ExecutorInfoResponse(BaseMessage):
+    executor_id: str
+    application_version: str
+
+
+@dataclass
+class RecoveryRequest(BaseMessage):
+    executor_ids: List[str]
+
+
+@dataclass
+class RecoveryResponse(BaseMessage):
+    success: bool
+
+
+@dataclass
+class CancelRequest(BaseMessage):
+    workflow_id: str
+
+
+@dataclass
+class CancelResponse(BaseMessage):
+    success: bool
+
+
+@dataclass
+class ResumeRequest(BaseMessage):
+    workflow_id: str
+
+
+@dataclass
+class ResumeResponse(BaseMessage):
+    success: bool
+
+
+@dataclass
+class RestartRequest(BaseMessage):
+    workflow_id: str
+
+
+@dataclass
+class RestartResponse(BaseMessage):
+    success: bool
+
+
+class ListWorkflowsBody(TypedDict):
+    workflow_uuids: List[str]
+    workflow_name: Optional[str]
+    authenticated_user: Optional[str]
+    start_time: Optional[str]
+    end_time: Optional[str]
+    status: Optional[str]
+    application_version: Optional[str]
+    limit: Optional[int]
+    offset: Optional[int]
+    sort_desc: bool
+
+
+@dataclass
+class WorkflowsOutput:
+    WorkflowUUID: str
+    Status: Optional[str]
+    WorkflowName: Optional[str]
+    WorkflowClassName: Optional[str]
+    WorkflowConfigName: Optional[str]
+    AuthenticatedUser: Optional[str]
+    AssumedRole: Optional[str]
+    AuthenticatedRoles: Optional[str]
+    Input: Optional[str]
+    Output: Optional[str]
+    Request: Optional[str]
+    Error: Optional[str]
+    CreatedAt: Optional[str]
+    UpdatedAt: Optional[str]
+    QueueName: Optional[str]
+    ApplicationVersion: Optional[str]
+
+    @classmethod
+    def from_workflow_information(cls, info: WorkflowInformation) -> "WorkflowsOutput":
+        # Convert fields to strings as needed
+        created_at_str = str(info.created_at) if info.created_at is not None else None
+        updated_at_str = str(info.updated_at) if info.updated_at is not None else None
+        inputs_str = str(info.input) if info.input is not None else None
+        outputs_str = str(info.output) if info.output is not None else None
+        request_str = str(info.request) if info.request is not None else None
+
+        return cls(
+            WorkflowUUID=info.workflow_id,
+            Status=info.status,
+            WorkflowName=info.workflow_name,
+            WorkflowClassName=info.workflow_class_name,
+            WorkflowConfigName=info.workflow_config_name,
+            AuthenticatedUser=info.authenticated_user,
+            AssumedRole=info.assumed_role,
+            AuthenticatedRoles=info.authenticated_roles,
+            Input=inputs_str,
+            Output=outputs_str,
+            Request=request_str,
+            Error=info.error,
+            CreatedAt=created_at_str,
+            UpdatedAt=updated_at_str,
+            QueueName=info.queue_name,
+            ApplicationVersion=info.app_version,
+        )
+
+
+@dataclass
+class ListWorkflowsRequest(BaseMessage):
+    body: ListWorkflowsBody
+
+
+@dataclass
+class ListWorkflowsResponse(BaseMessage):
+    output: List[WorkflowsOutput]
+
+
+class ListQueuedWorkflowsBody(TypedDict):
+    workflow_name: Optional[str]
+    start_time: Optional[str]
+    end_time: Optional[str]
+    status: Optional[str]
+    queue_name: Optional[str]
+    limit: Optional[int]
+    offset: Optional[int]
+    sort_desc: bool
+
+
+@dataclass
+class ListQueuedWorkflowsRequest(BaseMessage):
+    body: ListQueuedWorkflowsBody
+
+
+@dataclass
+class ListQueuedWorkflowsResponse(BaseMessage):
+    output: List[WorkflowsOutput]

dbos/_dbos.py

@@ -9,6 +9,7 @@ import os
 import sys
 import threading
 import traceback
+import uuid
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from logging import Logger
@@ -32,6 +33,7 @@ from typing import (
 
 from opentelemetry.trace import Span
 
+from dbos._conductor.conductor import ConductorWebsocket
 from dbos._utils import GlobalParams
 
 from ._classproperty import classproperty
@@ -258,6 +260,8 @@ class DBOS:
         config: Optional[ConfigFile] = None,
         fastapi: Optional["FastAPI"] = None,
         flask: Optional["Flask"] = None,
+        conductor_url: Optional[str] = None,
+        conductor_key: Optional[str] = None,
     ) -> DBOS:
         global _dbos_global_instance
         global _dbos_global_registry
@@ -273,7 +277,7 @@ class DBOS:
                 config = _dbos_global_registry.config
 
             _dbos_global_instance = super().__new__(cls)
-            _dbos_global_instance.__init__(fastapi=fastapi, config=config, flask=flask)  # type: ignore
+            _dbos_global_instance.__init__(fastapi=fastapi, config=config, flask=flask, conductor_url=conductor_url, conductor_key=conductor_key)  # type: ignore
         else:
             if (config is not None and _dbos_global_instance.config is not config) or (
                 _dbos_global_instance.fastapi is not fastapi
@@ -301,6 +305,8 @@ class DBOS:
         config: Optional[ConfigFile] = None,
         fastapi: Optional["FastAPI"] = None,
         flask: Optional["Flask"] = None,
+        conductor_url: Optional[str] = None,
+        conductor_key: Optional[str] = None,
     ) -> None:
         if hasattr(self, "_initialized") and self._initialized:
             return
@@ -324,6 +330,9 @@ class DBOS:
         self.flask: Optional["Flask"] = flask
         self._executor_field: Optional[ThreadPoolExecutor] = None
         self._background_threads: List[threading.Thread] = []
+        self.conductor_url: Optional[str] = conductor_url
+        self.conductor_key: Optional[str] = conductor_key
+        self.conductor_websocket: Optional[ConductorWebsocket] = None
 
         # If using FastAPI, set up middleware and lifecycle events
         if self.fastapi is not None:
@@ -399,6 +408,9 @@ class DBOS:
             self._debug_mode = debug_mode
             if GlobalParams.app_version == "":
                 GlobalParams.app_version = self._registry.compute_app_version()
+            if self.conductor_key is not None:
+                GlobalParams.executor_id = str(uuid.uuid4())
+            dbos_logger.info(f"Executor ID: {GlobalParams.executor_id}")
             dbos_logger.info(f"Application version: {GlobalParams.app_version}")
             self._executor_field = ThreadPoolExecutor(max_workers=64)
             self._sys_db_field = SystemDatabase(self.config, debug_mode=debug_mode)
@@ -451,6 +463,22 @@ class DBOS:
             bg_queue_thread.start()
             self._background_threads.append(bg_queue_thread)
 
+            # Start the conductor thread if requested
+            if self.conductor_key is not None:
+                if self.conductor_url is None:
+                    dbos_domain = os.environ.get("DBOS_DOMAIN", "cloud.dbos.dev")
+                    self.conductor_url = f"wss://{dbos_domain}/conductor/v1alpha1"
+                evt = threading.Event()
+                self.stop_events.append(evt)
+                self.conductor_websocket = ConductorWebsocket(
+                    self,
+                    conductor_url=self.conductor_url,
+                    conductor_key=self.conductor_key,
+                    evt=evt,
+                )
+                self.conductor_websocket.start()
+                self._background_threads.append(self.conductor_websocket)
+
             # Grab any pollers that were deferred and start them
             for evt, func, args, kwargs in self._registry.pollers:
                 self.stop_events.append(evt)
@@ -501,6 +529,11 @@ class DBOS:
         if self._admin_server_field is not None:
             self._admin_server_field.stop()
             self._admin_server_field = None
+        if (
+            self.conductor_websocket is not None
+            and self.conductor_websocket.websocket is not None
+        ):
+            self.conductor_websocket.websocket.close()
         # CB - This needs work, some things ought to stop before DBs are tossed out,
         #  on the other hand it hangs to move it
         if self._executor_field is not None:
@@ -864,12 +897,14 @@ class DBOS:
     @classmethod
     def cancel_workflow(cls, workflow_id: str) -> None:
         """Cancel a workflow by ID."""
+        dbos_logger.info(f"Cancelling workflow: {workflow_id}")
         _get_dbos_instance()._sys_db.cancel_workflow(workflow_id)
         _get_or_create_dbos_registry().cancel_workflow(workflow_id)
 
     @classmethod
     def resume_workflow(cls, workflow_id: str) -> WorkflowHandle[Any]:
         """Resume a workflow by ID."""
+        dbos_logger.info(f"Resuming workflow: {workflow_id}")
         _get_dbos_instance()._sys_db.resume_workflow(workflow_id)
         _get_or_create_dbos_registry().clear_workflow_cancelled(workflow_id)
         return execute_workflow_by_id(_get_dbos_instance(), workflow_id, False)

dbos/_recovery.py

@@ -27,8 +27,9 @@ def startup_recovery_thread(
                     pending_workflow.queue_name
                     and pending_workflow.queue_name != "_dbos_internal_queue"
                 ):
-                    dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
-                    continue
+                    cleared = dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
+                    if cleared:
+                        continue
                 execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
                 pending_workflows.remove(pending_workflow)
         except DBOSWorkflowFunctionNotFoundError:
@@ -46,7 +47,6 @@ def recover_pending_workflows(
     """Attempt to recover pending workflows for a list of specific executors and return workflow handles for them."""
     workflow_handles: List["WorkflowHandle[Any]"] = []
     for executor_id in executor_ids:
-        dbos.logger.debug(f"Recovering pending workflows for executor: {executor_id}")
         pending_workflows = dbos._sys_db.get_pending_workflows(
             executor_id, GlobalParams.app_version
         )
@@ -56,10 +56,15 @@ def recover_pending_workflows(
                 and pending_workflow.queue_name != "_dbos_internal_queue"
             ):
                 try:
-                    dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
-                    workflow_handles.append(
-                        dbos.retrieve_workflow(pending_workflow.workflow_uuid)
-                    )
+                    cleared = dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
+                    if cleared:
+                        workflow_handles.append(
+                            dbos.retrieve_workflow(pending_workflow.workflow_uuid)
+                        )
+                    else:
+                        workflow_handles.append(
+                            execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
+                        )
                 except Exception as e:
                     dbos.logger.error(e)
             else:
@@ -67,6 +72,6 @@ def recover_pending_workflows(
                     execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
                 )
         dbos.logger.info(
-            f"Recovering {len(pending_workflows)} workflows from version {GlobalParams.app_version}"
+            f"Recovering {len(pending_workflows)} workflows for executor {executor_id} from version {GlobalParams.app_version}"
         )
     return workflow_handles

dbos/_sys_db.py

@@ -124,7 +124,9 @@ class GetWorkflowsInput:
         self.offset: Optional[int] = (
             None  # Offset into the matching records for pagination
         )
-        self.sort_desc: bool = False  # If true, sort by created_at in DESC order. Default false (in ASC order).
+        self.sort_desc: bool = (
+            False  # If true, sort by created_at in DESC order. Default false (in ASC order).
+        )
 
 
 class GetQueuedWorkflowsInput(TypedDict):
@@ -1460,21 +1462,35 @@ class SystemDatabase:
                     .values(completed_at_epoch_ms=int(time.time() * 1000))
                 )
 
-    def clear_queue_assignment(self, workflow_id: str) -> None:
+
+    def clear_queue_assignment(self, workflow_id: str) -> bool:
         if self._debug_mode:
             raise Exception("called clear_queue_assignment in debug mode")
-        with self.engine.begin() as c:
-            c.execute(
-                sa.update(SystemSchema.workflow_queue)
-                .where(SystemSchema.workflow_queue.c.workflow_uuid == workflow_id)
-                .values(executor_id=None, started_at_epoch_ms=None)
-            )
-            c.execute(
-                sa.update(SystemSchema.workflow_status)
-                .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
-                .values(executor_id=None, status=WorkflowStatusString.ENQUEUED.value)
-            )
 
+        with self.engine.connect() as conn:
+            with conn.begin() as transaction:
+                res = conn.execute(
+                    sa.update(SystemSchema.workflow_queue)
+                    .where(SystemSchema.workflow_queue.c.workflow_uuid == workflow_id)
+                    .values(executor_id=None, started_at_epoch_ms=None)
+                )
+
+                # If no rows were affected, the workflow is not anymore in the queue
+                if res.rowcount == 0:
+                    transaction.rollback()
+                    return False
+
+                res = conn.execute(
+                    sa.update(SystemSchema.workflow_status)
+                    .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
+                    .values(executor_id=None, status=WorkflowStatusString.ENQUEUED.value)
+                )
+                if res.rowcount == 0:
+                    # This should never happen
+                    raise Exception(
+                        f"UNREACHABLE: Workflow {workflow_id} is found in the workflow_queue table but not found in the workflow_status table"
+                    )
+                return True
 
 def reset_system_database(config: ConfigFile) -> None:
     sysdb_name = (

dbos/_workflow_commands.py

@@ -1,10 +1,6 @@
 from typing import List, Optional, cast
 
-import typer
-
 from . import _serialization
-from ._dbos_config import ConfigFile
-from ._logger import dbos_logger
 from ._sys_db import (
     GetQueuedWorkflowsInput,
     GetWorkflowsInput,

dbos/cli/cli.py

@@ -75,7 +75,10 @@ def start() -> None:
 
             # If the child is still running, force kill it
             if process.poll() is None:
-                os.killpg(os.getpgid(process.pid), signal.SIGKILL)
+                try:
+                    os.killpg(os.getpgid(process.pid), signal.SIGKILL)
+                except Exception:
+                    pass
 
             # Exit immediately
             os._exit(process.returncode if process.returncode is not None else 1)

{dbos-0.23.0a9.dist-info → dbos-0.23.0a11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.23.0a9
+Version: 0.23.0a11
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT
@@ -23,6 +23,7 @@ Requires-Dist: docker>=7.1.0
 Requires-Dist: cryptography>=43.0.3
 Requires-Dist: rich>=13.9.4
 Requires-Dist: pyjwt>=2.10.1
+Requires-Dist: websockets>=15.0
 Description-Content-Type: text/markdown
 
 

{dbos-0.23.0a9.dist-info → dbos-0.23.0a11.dist-info}/RECORD

@@ -1,7 +1,7 @@
-dbos-0.23.0a9.dist-info/METADATA,sha256=KCH9_w-I-4MaqoTS0g8Zm4lAPxsWD71dOQwe4jyT3Ww,5523
-dbos-0.23.0a9.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-dbos-0.23.0a9.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
-dbos-0.23.0a9.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
+dbos-0.23.0a11.dist-info/METADATA,sha256=pm213i_FtTxsXr1cfO0_iP5bcU0E60Dzmq9GWXYtFMQ,5556
+dbos-0.23.0a11.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+dbos-0.23.0a11.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
+dbos-0.23.0a11.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
 dbos/__init__.py,sha256=CxRHBHEthPL4PZoLbZhp3rdm44-KkRTT2-7DkK9d4QQ,724
 dbos/__main__.py,sha256=P7jAr-7L9XE5mrsQ7i4b-bLr2ap1tCQfhMByLCRWDj0,568
 dbos/_admin_server.py,sha256=YiVn5lywz2Vg8_juyNHOYl0HVEy48--7b4phwK7r92o,5732
@@ -10,11 +10,13 @@ dbos/_classproperty.py,sha256=f0X-_BySzn3yFDRKB2JpCbLYQ9tLwt1XftfshvY7CBs,626
 dbos/_cloudutils/authentication.py,sha256=V0fCWQN9stCkhbuuxgPTGpvuQcDqfU3KAxPAh01vKW4,5007
 dbos/_cloudutils/cloudutils.py,sha256=YC7jGsIopT0KveLsqbRpQk2KlRBk-nIRC_UCgep4f3o,7797
 dbos/_cloudutils/databases.py,sha256=_shqaqSvhY4n2ScgQ8IP5PDZvzvcx3YBKV8fj-cxhSY,8543
+dbos/_conductor/conductor.py,sha256=SkTQDMBabzgrgJDDDwJSkF0eLIFzBe-_DIPL8OrNjA4,10067
+dbos/_conductor/protocol.py,sha256=mlAtcE2aT_egBiof_oyVVZPHTvWvU_U7-yu0fuNGKV0,4643
 dbos/_context.py,sha256=Ue5qu3rzLfRmPkz-UUZi9ZS8iXpapRN0NTM4mbA2QmQ,17738
 dbos/_core.py,sha256=UQb068FT59Op-F5RmtxreSeSQ1_wljOso0dQCUOPrC4,37528
 dbos/_croniter.py,sha256=XHAyUyibs_59sJQfSNWkP7rqQY6_XrlfuuCxk4jYqek,47559
 dbos/_db_wizard.py,sha256=6tfJaCRa1NtkUdNW75a2yvi_mEgnPJ9C1HP2zPG1hCU,8067
-dbos/_dbos.py,sha256=jNh1R4b13a4g1VXeSPaC2d3J2JXiF6YNpCBek9hkcbM,39393
+dbos/_dbos.py,sha256=0kX3fgdTqAn-eMKSbh73LFVR08YPMoB030g4dzvc9Yk,41150
 dbos/_dbos_config.py,sha256=_VETbEsMZ66563A8sX05B_coKz2BrILbIm9H5BmnPmk,9572
 dbos/_debug.py,sha256=wcvjM2k4BrK7mlYjImUZXNBUB00fPGjQrNimZXlj76c,1491
 dbos/_error.py,sha256=xqB7b7g5AF_OwOvqLKLXL1xldn2gAtORix2ZC2B8zK0,5089
@@ -34,7 +36,7 @@ dbos/_migrations/versions/d76646551a6c_workflow_queue.py,sha256=G942nophZ2uC2vc4
 dbos/_migrations/versions/eab0cc1d9a14_job_queue.py,sha256=uvhFOtqbBreCePhAxZfIT0qCAI7BiZTou9wt6QnbY7c,1412
 dbos/_outcome.py,sha256=FDMgWVjZ06vm9xO-38H17mTqBImUYQxgKs_bDCSIAhE,6648
 dbos/_queue.py,sha256=I2gBc7zQ4G0vyDDBnKwIFzxtqfD7DxHO2IZ41brFSOM,2927
-dbos/_recovery.py,sha256=Gk-ug-ppAMAFO72ZiGrxdbMx1gLmbAjciLCam_O_zV4,2847
+dbos/_recovery.py,sha256=QIi0xQToYOpUlAzwTsuDonsWbGrI9HZBR2qVyOS2IJU,3077
 dbos/_registrations.py,sha256=_zy6k944Ll8QwqU12Kr3OP23ukVtm8axPNN1TS_kJRc,6717
 dbos/_request.py,sha256=cX1B3Atlh160phgS35gF1VEEV4pD126c9F3BDgBmxZU,929
 dbos/_roles.py,sha256=iOsgmIAf1XVzxs3gYWdGRe1B880YfOw5fpU7Jwx8_A8,2271
@@ -43,7 +45,7 @@ dbos/_schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_schemas/application_database.py,sha256=KeyoPrF7hy_ODXV7QNike_VFSD74QBRfQ76D7QyE9HI,966
 dbos/_schemas/system_database.py,sha256=rwp4EvCSaXcUoMaRczZCvETCxGp72k3-hvLyGUDkih0,5163
 dbos/_serialization.py,sha256=YCYv0qKAwAZ1djZisBC7khvKqG-5OcIv9t9EC5PFIog,1743
-dbos/_sys_db.py,sha256=NVd4OCyEfAvokyewNX35qUN8nj8L3x_j2vzA1t4BNp0,63817
+dbos/_sys_db.py,sha256=nJVosEwdc_mkP9cJQb0uFpOOLOX8KxkOs-pij0vu83c,64467
 dbos/_templates/dbos-db-starter/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
 dbos/_templates/dbos-db-starter/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_templates/dbos-db-starter/__package/main.py,sha256=eI0SS9Nwj-fldtiuSzIlIG6dC91GXXwdRsoHxv6S_WI,2719
@@ -56,11 +58,11 @@ dbos/_templates/dbos-db-starter/migrations/versions/2024_07_31_180642_init.py,sh
 dbos/_templates/dbos-db-starter/start_postgres_docker.py,sha256=lQVLlYO5YkhGPEgPqwGc7Y8uDKse9HsWv5fynJEFJHM,1681
 dbos/_tracer.py,sha256=_Id9j9kCrptSNpEpLiRk_g5VPp-DrTWP1WNZInd5BA4,2439
 dbos/_utils.py,sha256=wjOJzxN66IzL9p4dwcEmQACRQah_V09G6mJI2exQfOM,155
-dbos/_workflow_commands.py,sha256=Z1PwprvR_A8PXV2FNhcMrvV8B4NlDI9dc5naMeeNKGw,4774
+dbos/_workflow_commands.py,sha256=CEzR5XghoZscbc2RHb9G-7Eoo4MMuzfeTo-QBZu4VPY,4690
 dbos/cli/_github_init.py,sha256=Y_bDF9gfO2jB1id4FV5h1oIxEJRWyqVjhb7bNEa5nQ0,3224
 dbos/cli/_template_init.py,sha256=AfuMaO8bmr9WsPNHr6j2cp7kjVVZDUpH7KpbTg0hhFs,2722
-dbos/cli/cli.py,sha256=ohrXoRshwxOwN-gFag_RW2yoNjwcCVv6nTYQdaArwek,15506
+dbos/cli/cli.py,sha256=ThomRytw7EP5iOcrjEgwnpaWgXNTLfnFEBBvCGHxtJs,15590
 dbos/dbos-config.schema.json,sha256=X5TpXNcARGceX0zQs0fVgtZW_Xj9uBbY5afPt9Rz9yk,5741
 dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
 version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
-dbos-0.23.0a9.dist-info/RECORD,,
+dbos-0.23.0a11.dist-info/RECORD,,