dbos 0.26.0a13__tar.gz → 0.26.0a14__tar.gz

{dbos-0.26.0a13 → dbos-0.26.0a14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.26.0a13
+Version: 0.26.0a14
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.26.0a13 → dbos-0.26.0a14}/dbos/_core.py

@@ -365,7 +365,9 @@ def _execute_workflow_wthread(
                 if isinstance(result, Immediate):
                     return cast(Immediate[R], result)()
                 else:
-                    return asyncio.run(cast(Pending[R], result)())
+                    return dbos._background_event_loop.submit_coroutine(
+                        cast(Pending[R], result)()
+                    )
             except Exception:
                 dbos.logger.error(
                     f"Exception encountered in asynchronous workflow: {traceback.format_exc()}"

{dbos-0.26.0a13 → dbos-0.26.0a14}/dbos/_dbos.py

@@ -4,7 +4,6 @@ import asyncio
 import atexit
 import hashlib
 import inspect
-import json
 import os
 import sys
 import threading
@@ -31,7 +30,6 @@ from typing import (
 
 from opentelemetry.trace import Span
 
-from dbos import _serialization
 from dbos._conductor.conductor import ConductorWebsocket
 from dbos._utils import INTERNAL_QUEUE_NAME, GlobalParams
 from dbos._workflow_commands import (
@@ -112,6 +110,7 @@ from ._error import (
     DBOSException,
     DBOSNonExistentWorkflowError,
 )
+from ._event_loop import BackgroundEventLoop
 from ._logger import add_otlp_to_all_loggers, config_logger, dbos_logger, init_logger
 from ._sys_db import SystemDatabase
 from ._workflow_commands import WorkflowStatus, get_workflow
@@ -341,6 +340,7 @@ class DBOS:
         self.conductor_url: Optional[str] = conductor_url
         self.conductor_key: Optional[str] = conductor_key
         self.conductor_websocket: Optional[ConductorWebsocket] = None
+        self._background_event_loop: BackgroundEventLoop = BackgroundEventLoop()
 
         init_logger()
 
@@ -451,6 +451,7 @@ class DBOS:
             dbos_logger.info(f"Executor ID: {GlobalParams.executor_id}")
             dbos_logger.info(f"Application version: {GlobalParams.app_version}")
             self._executor_field = ThreadPoolExecutor(max_workers=64)
+            self._background_event_loop.start()
             self._sys_db_field = SystemDatabase(
                 self.config["database"], debug_mode=debug_mode
             )
@@ -568,6 +569,7 @@ class DBOS:
         self._initialized = False
         for event in self.stop_events:
             event.set()
+        self._background_event_loop.stop()
         if self._sys_db_field is not None:
             self._sys_db_field.destroy()
             self._sys_db_field = None

dbos-0.26.0a14/dbos/_event_loop.py

@@ -0,0 +1,67 @@
+import asyncio
+import threading
+from typing import Any, Coroutine, Optional, TypeVar
+
+
+class BackgroundEventLoop:
+    """
+    This is the event loop to which DBOS submits any coroutines that are not started from within an event loop.
+    In particular, coroutines submitted to queues (such as from scheduled workflows) run on this event loop.
+    """
+
+    def __init__(self) -> None:
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._thread: Optional[threading.Thread] = None
+        self._running = False
+        self._ready = threading.Event()
+
+    def start(self) -> None:
+        if self._running:
+            return
+
+        self._thread = threading.Thread(target=self._run_event_loop, daemon=True)
+        self._thread.start()
+        self._ready.wait()  # Wait until the loop is running
+
+    def stop(self) -> None:
+        if not self._running or self._loop is None or self._thread is None:
+            return
+
+        asyncio.run_coroutine_threadsafe(self._shutdown(), self._loop)
+        self._thread.join()
+        self._running = False
+
+    def _run_event_loop(self) -> None:
+        self._loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self._loop)
+
+        self._running = True
+        self._ready.set()  # Signal that the loop is ready
+
+        try:
+            self._loop.run_forever()
+        finally:
+            self._loop.close()
+
+    async def _shutdown(self) -> None:
+        if self._loop is None:
+            raise RuntimeError("Event loop not started")
+        tasks = [
+            task
+            for task in asyncio.all_tasks(self._loop)
+            if task is not asyncio.current_task(self._loop)
+        ]
+
+        for task in tasks:
+            task.cancel()
+
+        await asyncio.gather(*tasks, return_exceptions=True)
+        self._loop.stop()
+
+    T = TypeVar("T")
+
+    def submit_coroutine(self, coro: Coroutine[Any, Any, T]) -> T:
+        """Submit a coroutine to the background event loop"""
+        if self._loop is None:
+            raise RuntimeError("Event loop not started")
+        return asyncio.run_coroutine_threadsafe(coro, self._loop).result()

{dbos-0.26.0a13 → dbos-0.26.0a14}/pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "0.26.0a13"
+version = "0.26.0a14"
 
 [project.license]
 text = "MIT"

{dbos-0.26.0a13 → dbos-0.26.0a14}/tests/test_async.py

@@ -7,8 +7,8 @@ import pytest
 import sqlalchemy as sa
 
 # Public API
-from dbos import DBOS, SetWorkflowID
-from dbos._dbos import WorkflowHandleAsync
+from dbos import DBOS, Queue, SetWorkflowID
+from dbos._dbos import WorkflowHandle, WorkflowHandleAsync
 from dbos._dbos_config import ConfigFile
 from dbos._error import DBOSException
 
@@ -56,6 +56,15 @@ async def test_async_workflow(dbos: DBOS) -> None:
     assert step_counter == 1
     assert txn_counter == 1
 
+    # Test DBOS.start_workflow_async
+    handle = await DBOS.start_workflow_async(test_workflow, "alice", "bob")
+    assert (await handle.get_result()) == "alicetxn21bobstep2"
+
+    # Test DBOS.start_workflow. Not recommended for async workflows,
+    # but needed for backwards compatibility.
+    sync_handle = DBOS.start_workflow(test_workflow, "alice", "bob")
+    assert sync_handle.get_result() == "alicetxn31bobstep3"  # type: ignore
+
 
 @pytest.mark.asyncio
 async def test_async_step(dbos: DBOS) -> None:
@@ -160,10 +169,11 @@ async def test_send_recv_async(dbos: DBOS) -> None:
     none_uuid = str(uuid.uuid4())
     none_handle = None
     with SetWorkflowID(none_uuid):
-        none_handle = dbos.start_workflow(test_recv_timeout, 10.0)
+        none_handle = await dbos.start_workflow_async(test_recv_timeout, 10.0)
     await test_send_none(none_uuid)
     begin_time = time.time()
-    assert none_handle.get_result() is None
+    result = await none_handle.get_result()  # type: ignore
+    assert result is None
     duration = time.time() - begin_time
     assert duration < 1.0  # None is from the received message, not from the timeout.
 
@@ -400,3 +410,59 @@ async def test_retrieve_workflow_async(dbos: DBOS) -> None:
     wfstatus = await handle.get_status()
     assert wfstatus.status == "SUCCESS"
     assert wfstatus.workflow_id == wfuuid
+
+
+def test_unawaited_workflow(dbos: DBOS) -> None:
+    input = 5
+    child_id = str(uuid.uuid4())
+    queue = Queue("test_queue")
+
+    @DBOS.workflow()
+    async def child_workflow(x: int) -> int:
+        await asyncio.sleep(0.1)
+        return x
+
+    @DBOS.workflow()
+    async def parent_workflow(x: int) -> None:
+        with SetWorkflowID(child_id):
+            await DBOS.start_workflow_async(child_workflow, x)
+
+    assert queue.enqueue(parent_workflow, input).get_result() is None
+    handle: WorkflowHandle[int] = DBOS.retrieve_workflow(
+        child_id, existing_workflow=False
+    )
+    assert handle.get_result() == 5
+
+
+def test_unawaited_workflow_exception(dbos: DBOS) -> None:
+    child_id = str(uuid.uuid4())
+    queue = Queue("test_queue")
+
+    @DBOS.workflow()
+    async def child_workflow(s: str) -> int:
+        await asyncio.sleep(0.1)
+        raise Exception(s)
+
+    @DBOS.workflow()
+    async def parent_workflow(s: str) -> None:
+        with SetWorkflowID(child_id):
+            await DBOS.start_workflow_async(child_workflow, s)
+
+    # Verify the unawaited child properly throws an exception
+    input = "alice"
+    assert queue.enqueue(parent_workflow, input).get_result() is None
+    handle: WorkflowHandle[int] = DBOS.retrieve_workflow(
+        child_id, existing_workflow=False
+    )
+    with pytest.raises(Exception) as exc_info:
+        handle.get_result()
+    assert input in str(exc_info.value)
+
+    # Verify it works if run again
+    input = "bob"
+    child_id = str(uuid.uuid4())
+    assert queue.enqueue(parent_workflow, input).get_result() is None
+    handle = DBOS.retrieve_workflow(child_id, existing_workflow=False)
+    with pytest.raises(Exception) as exc_info:
+        handle.get_result()
+    assert input in str(exc_info.value)