dbos 0.16.0a2__tar.gz → 0.17.0__tar.gz

{dbos-0.16.0a2 → dbos-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.16.0a2
+Version: 0.17.0
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.16.0a2 → dbos-0.17.0}/dbos/_core.py

@@ -21,7 +21,7 @@ from typing import (
     overload,
 )
 
-from dbos._outcome import Immediate, Outcome, Pending
+from dbos._outcome import Immediate, NoResult, Outcome, Pending
 
 from ._app_db import ApplicationDatabase, TransactionResultInternal
 
@@ -719,7 +719,7 @@ def decorate_step(
                 finally:
                     dbos._sys_db.record_operation_result(step_output)
 
-            def check_existing_result() -> Optional[R]:
+            def check_existing_result() -> Union[NoResult, R]:
                 ctx = assert_current_dbos_context()
                 recorded_output = dbos._sys_db.check_operation_execution(
                     ctx.workflow_id, ctx.function_id
@@ -743,7 +743,7 @@ def decorate_step(
                     dbos.logger.debug(
                         f"Running step, id: {ctx.function_id}, name: {attributes['name']}"
                     )
-                    return None
+                    return NoResult()
 
             stepOutcome = Outcome[R].make(functools.partial(func, *args, **kwargs))
             if retries_allowed:

{dbos-0.16.0a2 → dbos-0.17.0}/dbos/_dbos.py

@@ -45,7 +45,7 @@ from ._core import (
     start_workflow,
     workflow_wrapper,
 )
-from ._queue import Queue, _queue_thread
+from ._queue import Queue, queue_thread
 from ._recovery import recover_pending_workflows, startup_recovery_thread
 from ._registrations import (
     DEFAULT_MAX_RECOVERY_ATTEMPTS,
@@ -283,6 +283,7 @@ class DBOS:
         self.flask: Optional["Flask"] = flask
         self._executor_field: Optional[ThreadPoolExecutor] = None
         self._background_threads: List[threading.Thread] = []
+        self._executor_id: str = os.environ.get("DBOS__VMID", "local")
 
         # If using FastAPI, set up middleware and lifecycle events
         if self.fastapi is not None:
@@ -383,7 +384,7 @@ class DBOS:
             evt = threading.Event()
             self.stop_events.append(evt)
             bg_queue_thread = threading.Thread(
-                target=_queue_thread, args=(evt, self), daemon=True
+                target=queue_thread, args=(evt, self), daemon=True
             )
             bg_queue_thread.start()
             self._background_threads.append(bg_queue_thread)

{dbos-0.16.0a2 → dbos-0.17.0}/dbos/_outcome.py

@@ -8,6 +8,16 @@ T = TypeVar("T")
 R = TypeVar("R")
 
 
+class NoResult:
+    _instance: Optional["NoResult"] = None
+    __slots__ = ()
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "NoResult":
+        if not cls._instance:
+            cls._instance = super(NoResult, cls).__new__(cls, *args, **kwargs)
+        return cls._instance
+
+
 # define Outcome protocol w/ common composition methods
 class Outcome(Protocol[T]):
 
@@ -28,7 +38,9 @@ class Outcome(Protocol[T]):
         exceeded_retries: Callable[[int], BaseException],
     ) -> "Outcome[T]": ...
 
-    def intercept(self, interceptor: Callable[[], Optional[T]]) -> "Outcome[T]": ...
+    def intercept(
+        self, interceptor: Callable[[], Union[NoResult, T]]
+    ) -> "Outcome[T]": ...
 
     def __call__(self) -> Union[T, Coroutine[Any, Any, T]]: ...
 
@@ -58,11 +70,15 @@ class Immediate(Outcome[T]):
         return Immediate(lambda: before()(self._func))
 
     @staticmethod
-    def _intercept(func: Callable[[], T], interceptor: Callable[[], Optional[T]]) -> T:
+    def _intercept(
+        func: Callable[[], T], interceptor: Callable[[], Union[NoResult, T]]
+    ) -> T:
         intercepted = interceptor()
-        return intercepted if intercepted else func()
+        return intercepted if not isinstance(intercepted, NoResult) else func()
 
-    def intercept(self, interceptor: Callable[[], Optional[T]]) -> "Immediate[T]":
+    def intercept(
+        self, interceptor: Callable[[], Union[NoResult, T]]
+    ) -> "Immediate[T]":
         return Immediate[T](lambda: Immediate._intercept(self._func, interceptor))
 
     @staticmethod
@@ -151,12 +167,12 @@ class Pending(Outcome[T]):
     @staticmethod
     async def _intercept(
         func: Callable[[], Coroutine[Any, Any, T]],
-        interceptor: Callable[[], Optional[T]],
+        interceptor: Callable[[], Union[NoResult, T]],
     ) -> T:
         intercepted = await asyncio.to_thread(interceptor)
-        return intercepted if intercepted else await func()
+        return intercepted if not isinstance(intercepted, NoResult) else await func()
 
-    def intercept(self, interceptor: Callable[[], Optional[T]]) -> "Pending[T]":
+    def intercept(self, interceptor: Callable[[], Union[NoResult, T]]) -> "Pending[T]":
         return Pending[T](lambda: Pending._intercept(self._func, interceptor))
 
     @staticmethod

{dbos-0.16.0a2 → dbos-0.17.0}/dbos/_queue.py

@@ -51,13 +51,13 @@ class Queue:
         return start_workflow(dbos, func, self.name, False, *args, **kwargs)
 
 
-def _queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
+def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
     while not stop_event.is_set():
         if stop_event.wait(timeout=1):
             return
         for _, queue in dbos._registry.queue_info_map.items():
             try:
-                wf_ids = dbos._sys_db.start_queued_workflows(queue)
+                wf_ids = dbos._sys_db.start_queued_workflows(queue, dbos._executor_id)
                 for id in wf_ids:
                     execute_workflow_by_id(dbos, id)
             except Exception:

{dbos-0.16.0a2 → dbos-0.17.0}/dbos/_sys_db.py

@@ -1104,7 +1104,7 @@ class SystemDatabase:
                 .on_conflict_do_nothing()
             )
 
-    def start_queued_workflows(self, queue: "Queue") -> List[str]:
+    def start_queued_workflows(self, queue: "Queue", executor_id: str) -> List[str]:
         start_time_ms = int(time.time() * 1000)
         if queue.limiter is not None:
             limiter_period_ms = int(queue.limiter["period"] * 1000)
@@ -1159,7 +1159,7 @@ class SystemDatabase:
                     if len(ret_ids) + num_recent_queries >= queue.limiter["limit"]:
                         break
 
-                # To start a function, first set its status to PENDING
+                # To start a function, first set its status to PENDING and update its executor ID
                 c.execute(
                     SystemSchema.workflow_status.update()
                     .where(SystemSchema.workflow_status.c.workflow_uuid == id)
@@ -1167,7 +1167,10 @@ class SystemDatabase:
                         SystemSchema.workflow_status.c.status
                         == WorkflowStatusString.ENQUEUED.value
                     )
-                    .values(status=WorkflowStatusString.PENDING.value)
+                    .values(
+                        status=WorkflowStatusString.PENDING.value,
+                        executor_id=executor_id,
+                    )
                 )
 
                 # Then give it a start time

{dbos-0.16.0a2 → dbos-0.17.0}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "0.16.0a2"
+version = "0.17.0"
 
 [project.license]
 text = "MIT"

{dbos-0.16.0a2 → dbos-0.17.0}/tests/test_dbos.py

@@ -318,6 +318,7 @@ def test_temp_workflow_errors(dbos: DBOS) -> None:
 
 def test_recovery_workflow(dbos: DBOS) -> None:
     txn_counter: int = 0
+    txn_return_none_counter: int = 0
     wf_counter: int = 0
 
     @DBOS.workflow()
@@ -325,6 +326,8 @@ def test_recovery_workflow(dbos: DBOS) -> None:
         nonlocal wf_counter
         wf_counter += 1
         res = test_transaction(var2)
+        should_be_none = test_transaction_return_none()
+        assert should_be_none is None
         return res + var
 
     @DBOS.transaction()
@@ -334,6 +337,13 @@ def test_recovery_workflow(dbos: DBOS) -> None:
         txn_counter += 1
         return var2 + str(rows[0][0])
 
+    @DBOS.transaction()
+    def test_transaction_return_none() -> None:
+        nonlocal txn_return_none_counter
+        DBOS.sql_session.execute(sa.text("SELECT 1")).fetchall()
+        txn_return_none_counter += 1
+        return
+
     wfuuid = str(uuid.uuid4())
     with SetWorkflowID(wfuuid):
         assert test_workflow("bob", "bob") == "bob1bob"
@@ -367,6 +377,123 @@ def test_recovery_workflow(dbos: DBOS) -> None:
     assert workflow_handles[0].get_result() == "bob1bob"
     assert wf_counter == 2
     assert txn_counter == 1
+    assert txn_return_none_counter == 1
+
+    # Test that there was a recovery attempt of this
+    stat = workflow_handles[0].get_status()
+    assert stat
+    assert stat.recovery_attempts == 1
+
+
+def test_recovery_workflow_step(dbos: DBOS) -> None:
+    step_counter: int = 0
+    wf_counter: int = 0
+
+    @DBOS.workflow()
+    def test_workflow(var: str, var2: str) -> str:
+        nonlocal wf_counter
+        wf_counter += 1
+        should_be_none = test_step(var2)
+        assert should_be_none is None
+        return var
+
+    @DBOS.step()
+    def test_step(var2: str) -> None:
+        nonlocal step_counter
+        step_counter += 1
+        print(f"I'm a test_step {var2}!")
+        return
+
+    wfuuid = str(uuid.uuid4())
+    with SetWorkflowID(wfuuid):
+        assert test_workflow("bob", "bob") == "bob"
+
+    dbos._sys_db.wait_for_buffer_flush()
+    # Change the workflow status to pending
+    dbos._sys_db.update_workflow_status(
+        {
+            "workflow_uuid": wfuuid,
+            "status": "PENDING",
+            "name": test_workflow.__qualname__,
+            "class_name": None,
+            "config_name": None,
+            "output": None,
+            "error": None,
+            "executor_id": None,
+            "app_id": None,
+            "app_version": None,
+            "request": None,
+            "recovery_attempts": None,
+            "authenticated_user": None,
+            "authenticated_roles": None,
+            "assumed_role": None,
+            "queue_name": None,
+        }
+    )
+
+    # Recovery should execute the workflow again but skip the transaction
+    workflow_handles = DBOS.recover_pending_workflows()
+    assert len(workflow_handles) == 1
+    assert workflow_handles[0].get_result() == "bob"
+    assert wf_counter == 2
+    assert step_counter == 1
+
+    # Test that there was a recovery attempt of this
+    stat = workflow_handles[0].get_status()
+    assert stat
+    assert stat.recovery_attempts == 1
+
+
+def test_workflow_returns_none(dbos: DBOS) -> None:
+    wf_counter: int = 0
+
+    @DBOS.workflow()
+    def test_workflow(var: str, var2: str) -> None:
+        nonlocal wf_counter
+        wf_counter += 1
+        assert var == var2 == "bob"
+        return
+
+    wfuuid = str(uuid.uuid4())
+    with SetWorkflowID(wfuuid):
+        assert test_workflow("bob", "bob") is None
+    assert wf_counter == 1
+
+    dbos._sys_db.wait_for_buffer_flush()
+    with SetWorkflowID(wfuuid):
+        assert test_workflow("bob", "bob") is None
+    assert wf_counter == 2
+
+    handle: WorkflowHandle[None] = DBOS.retrieve_workflow(wfuuid)
+    assert handle.get_result() == None
+    assert wf_counter == 2
+
+    # Change the workflow status to pending
+    dbos._sys_db.update_workflow_status(
+        {
+            "workflow_uuid": wfuuid,
+            "status": "PENDING",
+            "name": test_workflow.__qualname__,
+            "class_name": None,
+            "config_name": None,
+            "output": None,
+            "error": None,
+            "executor_id": None,
+            "app_id": None,
+            "app_version": None,
+            "request": None,
+            "recovery_attempts": None,
+            "authenticated_user": None,
+            "authenticated_roles": None,
+            "assumed_role": None,
+            "queue_name": None,
+        }
+    )
+
+    workflow_handles = DBOS.recover_pending_workflows()
+    assert len(workflow_handles) == 1
+    assert workflow_handles[0].get_result() is None
+    assert wf_counter == 3
 
     # Test that there was a recovery attempt of this
     stat = workflow_handles[0].get_status()