dbos 0.22.0a11__tar.gz → 0.23.0a2__tar.gz

{dbos-0.22.0a11 → dbos-0.23.0a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.22.0a11
+Version: 0.23.0a2
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_context.py

@@ -12,6 +12,8 @@ from typing import List, Literal, Optional, Type, TypedDict
 from opentelemetry.trace import Span, Status, StatusCode
 from sqlalchemy.orm import Session
 
+from dbos._utils import GlobalParams
+
 from ._logger import dbos_logger
 from ._request import Request
 from ._tracer import dbos_tracer
@@ -48,7 +50,7 @@ class TracedAttributes(TypedDict, total=False):
 
 class DBOSContext:
     def __init__(self) -> None:
-        self.executor_id = os.environ.get("DBOS__VMID", "local")
+        self.executor_id = GlobalParams.executor_id
         self.app_id = os.environ.get("DBOS__APPID", "")
 
         self.logger = dbos_logger

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_core.py

@@ -22,6 +22,7 @@ from typing import (
 )
 
 from dbos._outcome import Immediate, NoResult, Outcome, Pending
+from dbos._utils import GlobalParams
 
 from ._app_db import ApplicationDatabase, TransactionResultInternal
 
@@ -51,6 +52,7 @@ from ._error import (
     DBOSMaxStepRetriesExceeded,
     DBOSNonExistentWorkflowError,
     DBOSRecoveryError,
+    DBOSWorkflowCancelledError,
     DBOSWorkflowConflictIDError,
     DBOSWorkflowFunctionNotFoundError,
 )
@@ -163,7 +165,7 @@ def _init_workflow(
         "output": None,
         "error": None,
         "app_id": ctx.app_id,
-        "app_version": dbos.app_version,
+        "app_version": GlobalParams.app_version,
         "executor_id": ctx.executor_id,
         "request": (
             _serialization.serialize(ctx.request) if ctx.request is not None else None
@@ -224,6 +226,8 @@ def _get_wf_invoke_func(
             )
             output = wf_handle.get_result()
             return output
+        except DBOSWorkflowCancelledError as error:
+            raise
         except Exception as error:
             status["status"] = "ERROR"
             status["error"] = _serialization.serialize_exception(error)
@@ -539,6 +543,13 @@ def decorate_transaction(
                 raise DBOSException(
                     f"Function {func.__name__} invoked before DBOS initialized"
                 )
+
+            ctx = assert_current_dbos_context()
+            if dbosreg.is_workflow_cancelled(ctx.workflow_id):
+                raise DBOSWorkflowCancelledError(
+                    f"Workflow {ctx.workflow_id} is cancelled. Aborting transaction {func.__name__}."
+                )
+
             dbos = dbosreg.dbos
             with dbos._app_db.sessionmaker() as session:
                 attributes: TracedAttributes = {
@@ -560,6 +571,12 @@ def decorate_transaction(
                     backoff_factor = 1.5
                     max_retry_wait_seconds = 2.0
                     while True:
+
+                        if dbosreg.is_workflow_cancelled(ctx.workflow_id):
+                            raise DBOSWorkflowCancelledError(
+                                f"Workflow {ctx.workflow_id} is cancelled. Aborting transaction {func.__name__}."
+                            )
+
                         has_recorded_error = False
                         txn_error: Optional[Exception] = None
                         try:
@@ -710,6 +727,13 @@ def decorate_step(
                 "operationType": OperationType.STEP.value,
             }
 
+            # Check if the workflow is cancelled
+            ctx = assert_current_dbos_context()
+            if dbosreg.is_workflow_cancelled(ctx.workflow_id):
+                raise DBOSWorkflowCancelledError(
+                    f"Workflow {ctx.workflow_id} is cancelled. Aborting step {func.__name__}."
+                )
+
             attempts = max_attempts if retries_allowed else 1
             max_retry_interval_seconds: float = 3600  # 1 Hour
 
@@ -800,6 +824,7 @@ def decorate_step(
             ctx = get_local_dbos_context()
             if ctx and ctx.is_step():
                 # Call the original function directly
+
                 return func(*args, **kwargs)
             if ctx and ctx.is_within_workflow():
                 assert ctx.is_workflow(), "Steps must be called from within workflows"

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_dbos.py

@@ -32,6 +32,8 @@ from typing import (
 
 from opentelemetry.trace import Span
 
+from dbos._utils import GlobalParams
+
 from ._classproperty import classproperty
 from ._core import (
     TEMP_SEND_WF_NAME,
@@ -155,6 +157,7 @@ class DBOSRegistry:
         self.pollers: list[RegisteredJob] = []
         self.dbos: Optional[DBOS] = None
         self.config: Optional[ConfigFile] = None
+        self.workflow_cancelled_map: dict[str, bool] = {}
 
     def register_wf_function(self, name: str, wrapped_func: F, functype: str) -> None:
         if name in self.function_type_map:
@@ -197,6 +200,15 @@ class DBOSRegistry:
         else:
             self.instance_info_map[fn] = inst
 
+    def cancel_workflow(self, workflow_id: str) -> None:
+        self.workflow_cancelled_map[workflow_id] = True
+
+    def is_workflow_cancelled(self, workflow_id: str) -> bool:
+        return self.workflow_cancelled_map.get(workflow_id, False)
+
+    def clear_workflow_cancelled(self, workflow_id: str) -> None:
+        self.workflow_cancelled_map.pop(workflow_id, None)
+
     def compute_app_version(self) -> str:
         """
         An application's version is computed from a hash of the source of its workflows.
@@ -280,6 +292,8 @@ class DBOS:
         if destroy_registry:
             global _dbos_global_registry
             _dbos_global_registry = None
+        GlobalParams.app_version = os.environ.get("DBOS__APPVERSION", "")
+        GlobalParams.executor_id = os.environ.get("DBOS__VMID", "local")
 
     def __init__(
         self,
@@ -309,8 +323,6 @@ class DBOS:
         self.flask: Optional["Flask"] = flask
         self._executor_field: Optional[ThreadPoolExecutor] = None
         self._background_threads: List[threading.Thread] = []
-        self._executor_id: str = os.environ.get("DBOS__VMID", "local")
-        self.app_version: str = os.environ.get("DBOS__APPVERSION", "")
 
         # If using FastAPI, set up middleware and lifecycle events
         if self.fastapi is not None:
@@ -379,10 +391,9 @@ class DBOS:
                 dbos_logger.warning(f"DBOS was already launched")
                 return
             self._launched = True
-            if self.app_version == "":
-                self.app_version = self._registry.compute_app_version()
-            dbos_logger.info(f"Application version: {self.app_version}")
-            dbos_tracer.app_version = self.app_version
+            if GlobalParams.app_version == "":
+                GlobalParams.app_version = self._registry.compute_app_version()
+            dbos_logger.info(f"Application version: {GlobalParams.app_version}")
             self._executor_field = ThreadPoolExecutor(max_workers=64)
             self._sys_db_field = SystemDatabase(self.config)
             self._app_db_field = ApplicationDatabase(self.config)
@@ -392,15 +403,15 @@ class DBOS:
             self._admin_server_field = AdminServer(dbos=self, port=admin_port)
 
             workflow_ids = self._sys_db.get_pending_workflows(
-                self._executor_id, self.app_version
+                GlobalParams.executor_id, GlobalParams.app_version
             )
             if (len(workflow_ids)) > 0:
                 self.logger.info(
-                    f"Recovering {len(workflow_ids)} workflows from application version {self.app_version}"
+                    f"Recovering {len(workflow_ids)} workflows from application version {GlobalParams.app_version}"
                 )
             else:
                 self.logger.info(
-                    f"No workflows to recover from application version {self.app_version}"
+                    f"No workflows to recover from application version {GlobalParams.app_version}"
                 )
 
             self._executor.submit(startup_recovery_thread, self, workflow_ids)
@@ -446,7 +457,7 @@ class DBOS:
             # to enable their export in DBOS Cloud
             for handler in dbos_logger.handlers:
                 handler.flush()
-            add_otlp_to_all_loggers(self.app_version)
+            add_otlp_to_all_loggers()
         except Exception:
             dbos_logger.error(f"DBOS failed to launch: {traceback.format_exc()}")
             raise
@@ -844,11 +855,13 @@ class DBOS:
     def cancel_workflow(cls, workflow_id: str) -> None:
         """Cancel a workflow by ID."""
         _get_dbos_instance()._sys_db.cancel_workflow(workflow_id)
+        _get_or_create_dbos_registry().cancel_workflow(workflow_id)
 
     @classmethod
     def resume_workflow(cls, workflow_id: str) -> WorkflowHandle[Any]:
         """Resume a workflow by ID."""
         _get_dbos_instance()._sys_db.resume_workflow(workflow_id)
+        _get_or_create_dbos_registry().clear_workflow_cancelled(workflow_id)
         return execute_workflow_by_id(_get_dbos_instance(), workflow_id, False)
 
     @classproperty

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_error.py

@@ -36,6 +36,7 @@ class DBOSErrorCode(Enum):
     MaxStepRetriesExceeded = 7
     NotAuthorized = 8
     ConflictingWorkflowError = 9
+    WorkflowCancelled = 10
     ConflictingRegistrationError = 25
 
 
@@ -130,6 +131,16 @@ class DBOSMaxStepRetriesExceeded(DBOSException):
         )
 
 
+class DBOSWorkflowCancelledError(DBOSException):
+    """Exception raised when the workflow has already been cancelled."""
+
+    def __init__(self, msg: str) -> None:
+        super().__init__(
+            msg,
+            dbos_error_code=DBOSErrorCode.WorkflowCancelled.value,
+        )
+
+
 class DBOSConflictingRegistrationError(DBOSException):
     """Exception raised when conflicting decorators are applied to the same function."""
 

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_logger.py

@@ -8,6 +8,8 @@ from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
 from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 from opentelemetry.sdk.resources import Resource
 
+from dbos._utils import GlobalParams
+
 if TYPE_CHECKING:
     from ._dbos_config import ConfigFile
 
@@ -19,13 +21,11 @@ class DBOSLogTransformer(logging.Filter):
     def __init__(self) -> None:
         super().__init__()
         self.app_id = os.environ.get("DBOS__APPID", "")
-        self.app_version = os.environ.get("DBOS__APPVERSION", "")
-        self.executor_id = os.environ.get("DBOS__VMID", "local")
 
     def filter(self, record: Any) -> bool:
         record.applicationID = self.app_id
-        record.applicationVersion = self.app_version
-        record.executorID = self.executor_id
+        record.applicationVersion = GlobalParams.app_version
+        record.executorID = GlobalParams.executor_id
         return True
 
 
@@ -86,9 +86,8 @@ def config_logger(config: "ConfigFile") -> None:
         dbos_logger.addFilter(_otlp_transformer)
 
 
-def add_otlp_to_all_loggers(app_version: str) -> None:
+def add_otlp_to_all_loggers() -> None:
     if _otlp_handler is not None and _otlp_transformer is not None:
-        _otlp_transformer.app_version = app_version
         root = logging.root
 
         root.addHandler(_otlp_handler)

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_queue.py

@@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Optional, TypedDict
 from psycopg import errors
 from sqlalchemy.exc import OperationalError
 
+from dbos._utils import GlobalParams
+
 from ._core import P, R, execute_workflow_by_id, start_workflow
 
 if TYPE_CHECKING:
@@ -71,7 +73,9 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
             return
         for _, queue in dbos._registry.queue_info_map.items():
             try:
-                wf_ids = dbos._sys_db.start_queued_workflows(queue, dbos._executor_id)
+                wf_ids = dbos._sys_db.start_queued_workflows(
+                    queue, GlobalParams.executor_id
+                )
                 for id in wf_ids:
                     execute_workflow_by_id(dbos, id)
             except OperationalError as e:

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_recovery.py

@@ -4,6 +4,8 @@ import time
 import traceback
 from typing import TYPE_CHECKING, Any, List
 
+from dbos._utils import GlobalParams
+
 from ._core import execute_workflow_by_id
 from ._error import DBOSWorkflowFunctionNotFoundError
 from ._sys_db import GetPendingWorkflowsOutput
@@ -45,7 +47,7 @@ def recover_pending_workflows(
     for executor_id in executor_ids:
         dbos.logger.debug(f"Recovering pending workflows for executor: {executor_id}")
         pending_workflows = dbos._sys_db.get_pending_workflows(
-            executor_id, dbos.app_version
+            executor_id, GlobalParams.app_version
         )
         for pending_workflow in pending_workflows:
             if (
@@ -64,6 +66,6 @@ def recover_pending_workflows(
                     execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
                 )
         dbos.logger.info(
-            f"Recovering {len(pending_workflows)} workflows from version {dbos.app_version}"
+            f"Recovering {len(pending_workflows)} workflows from version {GlobalParams.app_version}"
         )
     return workflow_handles

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_sys_db.py

@@ -27,6 +27,8 @@ from alembic.config import Config
 from sqlalchemy.exc import DBAPIError
 from sqlalchemy.sql import func
 
+from dbos._utils import GlobalParams
+
 from . import _serialization
 from ._dbos_config import ConfigFile
 from ._error import (
@@ -192,9 +194,7 @@ class SystemDatabase:
             port=config["database"]["port"],
             database="postgres",
             # fills the "application_name" column in pg_stat_activity
-            query={
-                "application_name": f"dbos_transact_{os.environ.get('DBOS__VMID', 'local')}"
-            },
+            query={"application_name": f"dbos_transact_{GlobalParams.executor_id}"},
         )
         engine = sa.create_engine(postgres_db_url)
         with engine.connect() as conn:
@@ -214,9 +214,7 @@ class SystemDatabase:
             port=config["database"]["port"],
             database=sysdb_name,
             # fills the "application_name" column in pg_stat_activity
-            query={
-                "application_name": f"dbos_transact_{os.environ.get('DBOS__VMID', 'local')}"
-            },
+            query={"application_name": f"dbos_transact_{GlobalParams.executor_id}"},
         )
 
         # Create a connection pool for the system database

{dbos-0.22.0a11 → dbos-0.23.0a2}/dbos/_tracer.py

@@ -7,6 +7,8 @@ from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
 from opentelemetry.trace import Span
 
+from dbos._utils import GlobalParams
+
 from ._dbos_config import ConfigFile
 
 if TYPE_CHECKING:
@@ -17,8 +19,6 @@ class DBOSTracer:
 
     def __init__(self) -> None:
         self.app_id = os.environ.get("DBOS__APPID", None)
-        self.app_version = os.environ.get("DBOS__APPVERSION", None)
-        self.executor_id = os.environ.get("DBOS__VMID", "local")
         self.provider: Optional[TracerProvider] = None
 
     def config(self, config: ConfigFile) -> None:
@@ -51,8 +51,8 @@ class DBOSTracer:
         context = trace.set_span_in_context(parent) if parent else None
         span: Span = tracer.start_span(name=attributes["name"], context=context)
         attributes["applicationID"] = self.app_id
-        attributes["applicationVersion"] = self.app_version
-        attributes["executorID"] = self.executor_id
+        attributes["applicationVersion"] = GlobalParams.app_version
+        attributes["executorID"] = GlobalParams.executor_id
         for k, v in attributes.items():
             if k != "name" and v is not None and isinstance(v, (str, bool, int, float)):
                 span.set_attribute(k, v)

dbos-0.23.0a2/dbos/_utils.py

@@ -0,0 +1,6 @@
+import os
+
+
+class GlobalParams:
+    app_version: str = os.environ.get("DBOS__APPVERSION", "")
+    executor_id: str = os.environ.get("DBOS__VMID", "local")

{dbos-0.22.0a11 → dbos-0.23.0a2}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "0.22.0a11"
+version = "0.23.0a2"
 
 [project.license]
 text = "MIT"

{dbos-0.22.0a11 → dbos-0.23.0a2}/tests/test_admin_server.py

@@ -6,6 +6,7 @@ import requests
 
 # Public API
 from dbos import DBOS, ConfigFile, Queue, SetWorkflowID, _workflow_commands
+from dbos._utils import GlobalParams
 
 
 def test_admin_endpoints(dbos: DBOS) -> None:
@@ -59,10 +60,13 @@ def test_admin_endpoints(dbos: DBOS) -> None:
         assert event.is_set(), "Event is not set!"
 
 
-def test_admin_recovery(dbos: DBOS) -> None:
+def test_admin_recovery(config: ConfigFile) -> None:
     os.environ["DBOS__VMID"] = "testexecutor"
     os.environ["DBOS__APPVERSION"] = "testversion"
     os.environ["DBOS__APPID"] = "testappid"
+    DBOS.destroy(destroy_registry=True)
+    dbos = DBOS(config=config)
+    DBOS.launch()
 
     step_counter: int = 0
     wf_counter: int = 0

{dbos-0.22.0a11 → dbos-0.23.0a2}/tests/test_dbos.py

@@ -19,6 +19,7 @@ from dbos._context import assert_current_dbos_context, get_local_dbos_context
 from dbos._error import DBOSConflictingRegistrationError, DBOSMaxStepRetriesExceeded
 from dbos._schemas.system_database import SystemSchema
 from dbos._sys_db import GetWorkflowsInput
+from dbos._utils import GlobalParams
 
 
 def test_simple_workflow(dbos: DBOS) -> None:
@@ -1240,11 +1241,12 @@ def test_app_version(config: ConfigFile) -> None:
     DBOS.launch()
 
     # Verify that app version is correctly set to a hex string
-    app_version = dbos.app_version
+    app_version = GlobalParams.app_version
     assert len(app_version) > 0
     assert is_hex(app_version)
 
     DBOS.destroy(destroy_registry=True)
+    assert GlobalParams.app_version == ""
     dbos = DBOS(config=config)
 
     @DBOS.workflow()
@@ -1258,7 +1260,7 @@ def test_app_version(config: ConfigFile) -> None:
     DBOS.launch()
 
     # Verify stability--the same workflow source produces the same app version.
-    assert dbos.app_version == app_version
+    assert GlobalParams.app_version == app_version
 
     DBOS.destroy(destroy_registry=True)
     dbos = DBOS(config=config)
@@ -1269,7 +1271,7 @@ def test_app_version(config: ConfigFile) -> None:
 
     # Verify that changing the workflow source changes the workflow version
     DBOS.launch()
-    assert dbos.app_version != app_version
+    assert GlobalParams.app_version != app_version
 
     # Verify that version can be overriden with an environment variable
     app_version = "12345"
@@ -1283,7 +1285,7 @@ def test_app_version(config: ConfigFile) -> None:
         return x
 
     DBOS.launch()
-    assert dbos.app_version == app_version
+    assert GlobalParams.app_version == app_version
 
     del os.environ["DBOS__APPVERSION"]
 

{dbos-0.22.0a11 → dbos-0.23.0a2}/tests/test_spans.py

@@ -9,6 +9,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanE
 
 from dbos import DBOS
 from dbos._tracer import dbos_tracer
+from dbos._utils import GlobalParams
 
 
 def test_spans(dbos: DBOS) -> None:
@@ -36,7 +37,7 @@ def test_spans(dbos: DBOS) -> None:
 
     for span in spans:
         assert span.attributes is not None
-        assert span.attributes["applicationVersion"] == dbos.app_version
+        assert span.attributes["applicationVersion"] == GlobalParams.app_version
         assert span.context is not None
 
     assert spans[0].name == test_step.__name__
@@ -74,7 +75,7 @@ async def test_spans_async(dbos: DBOS) -> None:
 
     for span in spans:
         assert span.attributes is not None
-        assert span.attributes["applicationVersion"] == dbos.app_version
+        assert span.attributes["applicationVersion"] == GlobalParams.app_version
         assert span.context is not None
 
     assert spans[0].name == test_step.__name__
@@ -111,7 +112,7 @@ def test_temp_wf_fastapi(dbos_fastapi: Tuple[DBOS, FastAPI]) -> None:
 
     for span in spans:
         assert span.attributes is not None
-        assert span.attributes["applicationVersion"] == dbos.app_version
+        assert span.attributes["applicationVersion"] == GlobalParams.app_version
         assert span.context is not None
 
     assert spans[0].name == test_step_endpoint.__name__

dbos-0.23.0a2/tests/test_workflow_cancel.py

@@ -0,0 +1,145 @@
+import threading
+import time
+import uuid
+from datetime import datetime, timedelta, timezone
+
+# Public API
+from dbos import (
+    DBOS,
+    ConfigFile,
+    Queue,
+    SetWorkflowID,
+    WorkflowStatusString,
+    _workflow_commands,
+)
+
+
+def test_basic(dbos: DBOS, config: ConfigFile) -> None:
+
+    steps_completed = 0
+
+    @DBOS.step()
+    def step_one() -> None:
+        nonlocal steps_completed
+        steps_completed += 1
+        print("Step one completed!")
+
+    @DBOS.step()
+    def step_two() -> None:
+        nonlocal steps_completed
+        steps_completed += 1
+        print("Step two completed!")
+
+    @DBOS.workflow()
+    def simple_workflow() -> None:
+        step_one()
+        dbos.sleep(1)
+        step_two()
+        print("Executed Simple workflow")
+        return
+
+    # run the workflow
+    simple_workflow()
+    time.sleep(1)  # wait for the workflow to complete
+    assert (
+        steps_completed == 2
+    ), f"Expected steps_completed to be 2, but got {steps_completed}"
+
+
+def test_two_steps_cancel(dbos: DBOS, config: ConfigFile) -> None:
+
+    steps_completed = 0
+
+    @DBOS.step()
+    def step_one() -> None:
+        nonlocal steps_completed
+        steps_completed += 1
+        print("Step one completed!")
+
+    @DBOS.step()
+    def step_two() -> None:
+        nonlocal steps_completed
+        steps_completed += 1
+        print("Step two completed!")
+
+    @DBOS.workflow()
+    def simple_workflow() -> None:
+        step_one()
+        dbos.sleep(2)
+        step_two()
+        print("Executed Simple workflow")
+        return
+
+    # run the workflow
+    try:
+        wfuuid = str(uuid.uuid4())
+        with SetWorkflowID(wfuuid):
+            simple_workflow()
+
+        dbos.cancel_workflow(wfuuid)
+    except Exception as e:
+        # time.sleep(1)  # wait for the workflow to complete
+        assert (
+            steps_completed == 1
+        ), f"Expected steps_completed to be 1, but got {steps_completed}"
+
+    dbos.resume_workflow(wfuuid)
+    time.sleep(1)
+
+    assert (
+        steps_completed == 2
+    ), f"Expected steps_completed to be 2, but got {steps_completed}"
+
+
+def test_two_transactions_cancel(dbos: DBOS, config: ConfigFile) -> None:
+
+    tr_completed = 0
+
+    @DBOS.transaction()
+    def transaction_one() -> None:
+        nonlocal tr_completed
+        tr_completed += 1
+        print("Transaction one completed!")
+
+    @DBOS.transaction()
+    def transaction_two() -> None:
+        nonlocal tr_completed
+        tr_completed += 1
+        print("Step two completed!")
+
+    @DBOS.workflow()
+    def simple_workflow() -> None:
+        transaction_one()
+        dbos.sleep(2)
+        transaction_two()
+        print("Executed Simple workflow")
+        return
+
+    # run the workflow
+    try:
+        wfuuid = str(uuid.uuid4())
+        with SetWorkflowID(wfuuid):
+            simple_workflow()
+
+        dbos.cancel_workflow(wfuuid)
+    except Exception as e:
+        # time.sleep(1)  # wait for the workflow to complete
+        assert (
+            tr_completed == 1
+        ), f"Expected tr_completed to be 1, but got {tr_completed}"
+
+    dbos.resume_workflow(wfuuid)
+    time.sleep(1)
+
+    assert (
+        tr_completed == 2
+    ), f"Expected steps_completed to be 2, but got {tr_completed}"
+
+    # resume it a 2nd time
+
+    dbos.resume_workflow(wfuuid)
+    time.sleep(1)
+
+    assert (
+        tr_completed == 2
+    ), f"Expected steps_completed to be 2, but got {tr_completed}"