dbos 0.21.0a3__tar.gz → 0.21.0a4__tar.gz

{dbos-0.21.0a3 → dbos-0.21.0a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.21.0a3
+Version: 0.21.0a4
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.21.0a3 → dbos-0.21.0a4}/dbos/_dbos.py

@@ -800,14 +800,13 @@ class DBOS:
     @classmethod
     def cancel_workflow(cls, workflow_id: str) -> None:
         """Cancel a workflow by ID."""
-        _get_dbos_instance()._sys_db.set_workflow_status(
-            workflow_id, WorkflowStatusString.CANCELLED
-        )
+        _get_dbos_instance()._sys_db.cancel_workflow(workflow_id)
 
     @classmethod
-    def resume_workflow(cls, workflow_id: str) -> None:
+    def resume_workflow(cls, workflow_id: str) -> WorkflowHandle[Any]:
         """Resume a workflow by ID."""
-        execute_workflow_by_id(_get_dbos_instance(), workflow_id, False)
+        _get_dbos_instance()._sys_db.resume_workflow(workflow_id)
+        return execute_workflow_by_id(_get_dbos_instance(), workflow_id, False)
 
     @classproperty
     def logger(cls) -> Logger:

{dbos-0.21.0a3 → dbos-0.21.0a4}/dbos/_sys_db.py

@@ -390,20 +390,55 @@ class SystemDatabase:
         if status["workflow_uuid"] in self._temp_txn_wf_ids:
             self._exported_temp_txn_wf_status.add(status["workflow_uuid"])
 
-    def set_workflow_status(
+    def cancel_workflow(
         self,
-        workflow_uuid: str,
-        status: WorkflowStatusString,
+        workflow_id: str,
     ) -> None:
         with self.engine.begin() as c:
-            stmt = (
+            # Remove the workflow from the queues table so it does not block the table
+            c.execute(
+                sa.delete(SystemSchema.workflow_queue).where(
+                    SystemSchema.workflow_queue.c.workflow_uuid == workflow_id
+                )
+            )
+            # Set the workflow's status to CANCELLED
+            c.execute(
                 sa.update(SystemSchema.workflow_status)
-                .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid)
+                .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
                 .values(
-                    status=status,
+                    status=WorkflowStatusString.CANCELLED.value,
+                )
+            )
+
+    def resume_workflow(
+        self,
+        workflow_id: str,
+    ) -> None:
+        with self.engine.begin() as c:
+            # Check the status of the workflow. If it is complete, do nothing.
+            row = c.execute(
+                sa.select(
+                    SystemSchema.workflow_status.c.status,
+                ).where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
+            ).fetchone()
+            if (
+                row is None
+                or row[0] == WorkflowStatusString.SUCCESS.value
+                or row[0] == WorkflowStatusString.ERROR.value
+            ):
+                return
+            # Remove the workflow from the queues table so resume can safely be called on an ENQUEUED workflow
+            c.execute(
+                sa.delete(SystemSchema.workflow_queue).where(
+                    SystemSchema.workflow_queue.c.workflow_uuid == workflow_id
                 )
             )
-            c.execute(stmt)
+            # Set the workflow's status to PENDING and clear its recovery attempts.
+            c.execute(
+                sa.update(SystemSchema.workflow_status)
+                .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
+                .values(status=WorkflowStatusString.PENDING.value, recovery_attempts=0)
+            )
 
     def get_workflow_status(
         self, workflow_uuid: str

{dbos-0.21.0a3 → dbos-0.21.0a4}/dbos/_workflow_commands.py

@@ -1,23 +1,14 @@
-import importlib
-import os
-import sys
-from typing import Any, List, Optional, cast
+from typing import List, Optional, cast
 
 import typer
-from rich import print
 
-from dbos import DBOS
-
-from . import _serialization, load_config
-from ._core import execute_workflow_by_id
-from ._dbos_config import ConfigFile, _is_valid_app_name
+from . import _serialization
+from ._dbos_config import ConfigFile
 from ._sys_db import (
     GetWorkflowsInput,
     GetWorkflowsOutput,
     SystemDatabase,
     WorkflowStatuses,
-    WorkflowStatusInternal,
-    WorkflowStatusString,
 )
 
 
@@ -41,7 +32,7 @@ class WorkflowInformation:
     queue_name: Optional[str]
 
 
-def _list_workflows(
+def list_workflows(
     config: ConfigFile,
     li: int,
     user: Optional[str],
@@ -91,17 +82,13 @@ def _list_workflows(
             sys_db.destroy()
 
 
-def _get_workflow(
+def get_workflow(
     config: ConfigFile, uuid: str, request: bool
 ) -> Optional[WorkflowInformation]:
-    sys_db = None
-
     try:
         sys_db = SystemDatabase(config)
-
         info = _get_workflow_info(sys_db, uuid, request)
         return info
-
     except Exception as e:
         typer.echo(f"Error getting workflow: {e}")
         return None
@@ -110,18 +97,13 @@ def _get_workflow(
             sys_db.destroy()
 
 
-def _cancel_workflow(config: ConfigFile, uuid: str) -> None:
-    # config = load_config()
-    sys_db = None
-
+def cancel_workflow(config: ConfigFile, uuid: str) -> None:
     try:
         sys_db = SystemDatabase(config)
-        sys_db.set_workflow_status(uuid, WorkflowStatusString.CANCELLED)
-        return
-
+        sys_db.cancel_workflow(uuid)
     except Exception as e:
         typer.echo(f"Failed to connect to DBOS system database: {e}")
-        return None
+        raise e
     finally:
         if sys_db:
             sys_db.destroy()

{dbos-0.21.0a3 → dbos-0.21.0a4}/dbos/cli/cli.py

@@ -19,7 +19,7 @@ from .. import load_config
 from .._app_db import ApplicationDatabase
 from .._dbos_config import _is_valid_app_name
 from .._sys_db import SystemDatabase, reset_system_database
-from .._workflow_commands import _cancel_workflow, _get_workflow, _list_workflows
+from .._workflow_commands import cancel_workflow, get_workflow, list_workflows
 from ..cli._github_init import create_template_from_github
 from ._template_init import copy_template, get_project_name, get_templates_directory
 
@@ -282,7 +282,7 @@ def list(
     ] = None,
 ) -> None:
     config = load_config()
-    workflows = _list_workflows(
+    workflows = list_workflows(
         config, limit, user, starttime, endtime, status, request, appversion
     )
     print(jsonpickle.encode(workflows, unpicklable=False))
@@ -301,7 +301,7 @@ def get(
     ] = True,
 ) -> None:
     config = load_config()
-    print(jsonpickle.encode(_get_workflow(config, uuid, request), unpicklable=False))
+    print(jsonpickle.encode(get_workflow(config, uuid, request), unpicklable=False))
 
 
 @workflow.command(
@@ -315,7 +315,7 @@ def cancel(
     ] = None,
 ) -> None:
     config = load_config()
-    _cancel_workflow(config, uuid)
+    cancel_workflow(config, uuid)
     print(f"Workflow {uuid} has been cancelled")
 
 

{dbos-0.21.0a3 → dbos-0.21.0a4}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "0.21.0a3"
+version = "0.21.0a4"
 
 [project.license]
 text = "MIT"

{dbos-0.21.0a3 → dbos-0.21.0a4}/tests/test_admin_server.py

@@ -151,54 +151,59 @@ runtimeConfig:
 
 
 def test_admin_workflow_resume(dbos: DBOS, config: ConfigFile) -> None:
+    counter: int = 0
 
     @DBOS.workflow()
     def simple_workflow() -> None:
-        print("Executed Simple workflow")
-        return
+        nonlocal counter
+        counter += 1
 
-    # run the workflow
+    # Run the workflow and flush its results
     simple_workflow()
-    time.sleep(1)
+    assert counter == 1
+    dbos._sys_db.wait_for_buffer_flush()
 
-    # get the workflow list
-    output = _workflow_commands._list_workflows(
+    # Verify the workflow has succeeded
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, None, False, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
-
     assert output[0] != None, "Expected output to be not None"
-
     wfUuid = output[0].workflowUUID
-
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
     assert info is not None, "Expected output to be not None"
-
     assert info.status == "SUCCESS", f"Expected status to be SUCCESS"
 
+    # Cancel the workflow. Verify it was cancelled
     response = requests.post(
         f"http://localhost:3001/workflows/{wfUuid}/cancel", json=[], timeout=5
     )
     assert response.status_code == 204
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
+    assert info is not None
+    assert info.status == "CANCELLED", f"Expected status to be CANCELLED"
 
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
-    if info is not None:
-        assert info.status == "CANCELLED", f"Expected status to be CANCELLED"
-    else:
-        assert False, "Expected info to be not None"
-
+    # Resume the workflow. Verify that it succeeds again.
     response = requests.post(
         f"http://localhost:3001/workflows/{wfUuid}/resume", json=[], timeout=5
     )
     assert response.status_code == 204
+    dbos._sys_db.wait_for_buffer_flush()
+    assert counter == 2
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
+    assert info is not None
+    assert info.status == "SUCCESS", f"Expected status to be SUCCESS"
 
-    time.sleep(1)
-
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
-    if info is not None:
-        assert info.status == "SUCCESS", f"Expected status to be SUCCESS"
-    else:
-        assert False, "Expected info to be not None"
+    # Resume the workflow. Verify it does not run and status remains SUCCESS
+    response = requests.post(
+        f"http://localhost:3001/workflows/{wfUuid}/resume", json=[], timeout=5
+    )
+    assert response.status_code == 204
+    dbos._sys_db.wait_for_buffer_flush()
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
+    assert info is not None
+    assert info.status == "SUCCESS", f"Expected status to be SUCCESS"
+    assert counter == 2
 
 
 def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
@@ -213,7 +218,7 @@ def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
     time.sleep(1)
 
     # get the workflow list
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, None, False, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
@@ -222,7 +227,7 @@ def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
 
     wfUuid = output[0].workflowUUID
 
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
     assert info is not None, "Expected output to be not None"
 
     assert info.status == "SUCCESS", f"Expected status to be SUCCESS"
@@ -232,7 +237,7 @@ def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
     )
     assert response.status_code == 204
 
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
     if info is not None:
         assert info.status == "CANCELLED", f"Expected status to be CANCELLED"
     else:
@@ -245,13 +250,13 @@ def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
 
     time.sleep(1)
 
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
     if info is not None:
         assert info.status == "CANCELLED", f"Expected status to be CANCELLED"
     else:
         assert False, "Expected info to be not None"
 
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, None, False, None
     )
     assert len(output) == 2, f"Expected list length to be 2, but got {len(output)}"
@@ -261,7 +266,7 @@ def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
     else:
         new_wfUuid = output[0].workflowUUID
 
-    info = _workflow_commands._get_workflow(config, new_wfUuid, True)
+    info = _workflow_commands.get_workflow(config, new_wfUuid, True)
     if info is not None:
         assert info.status == "SUCCESS", f"Expected status to be SUCCESS"
     else:

{dbos-0.21.0a3 → dbos-0.21.0a4}/tests/test_failures.py

@@ -189,60 +189,41 @@ def test_dead_letter_queue(dbos: DBOS) -> None:
         recovery_count += 1
         event.wait()
 
-    handle = DBOS.start_workflow(dead_letter_workflow)
+    # Start a workflow that blocks forever
+    wfid = str(uuid.uuid4())
+    with SetWorkflowID(wfid):
+        handle = DBOS.start_workflow(dead_letter_workflow)
 
+    # Attempt to recover the blocked workflow the maximum number of times
     for i in range(max_recovery_attempts):
         DBOS.recover_pending_workflows()
         assert recovery_count == i + 2
 
+    # Verify an additional attempt (either through recovery or through a direct call) throws a DLQ error
+    # and puts the workflow in the DLQ status.
     with pytest.raises(Exception) as exc_info:
         DBOS.recover_pending_workflows()
     assert exc_info.errisinstance(DBOSDeadLetterQueueError)
     assert handle.get_status().status == WorkflowStatusString.RETRIES_EXCEEDED.value
-
-    event.set()
-    assert handle.get_result() == None
-    dbos._sys_db.wait_for_buffer_flush()
-    assert handle.get_status().status == WorkflowStatusString.SUCCESS.value
-
-
-def test_enqueued_dead_letter_queue(dbos: DBOS) -> None:
-    function_started_event = threading.Event()
-    event = threading.Event()
-    max_concurrency = 1
-    max_recovery_attempts = 10
-    recovery_count = 0
-
-    @DBOS.workflow(max_recovery_attempts=max_recovery_attempts)
-    def dead_letter_workflow() -> None:
-        function_started_event.set()
-        nonlocal recovery_count
-        recovery_count += 1
-        event.wait()
-
-    @DBOS.workflow()
-    def regular_workflow() -> None:
-        return
-
-    queue = Queue("test_queue", concurrency=max_concurrency)
-    handle = queue.enqueue(dead_letter_workflow)
-    function_started_event.wait()
-
-    for i in range(max_recovery_attempts):
-        DBOS.recover_pending_workflows()
-        assert recovery_count == i + 2
-
-    regular_handle = queue.enqueue(regular_workflow)
-
     with pytest.raises(Exception) as exc_info:
-        DBOS.recover_pending_workflows()
+        with SetWorkflowID(wfid):
+            dead_letter_workflow()
     assert exc_info.errisinstance(DBOSDeadLetterQueueError)
-    assert handle.get_status().status == WorkflowStatusString.RETRIES_EXCEEDED.value
 
-    assert regular_handle.get_result() == None
+    # Resume the workflow. Verify it returns to PENDING status without error.
+    resumed_handle = dbos.resume_workflow(wfid)
+    assert (
+        handle.get_status().status
+        == resumed_handle.get_status().status
+        == WorkflowStatusString.PENDING.value
+    )
+
+    # Verify the workflow can recover again without error.
+    DBOS.recover_pending_workflows()
 
+    # Complete the blocked workflow
     event.set()
-    assert handle.get_result() == None
+    assert handle.get_result() == resumed_handle.get_result() == None
     dbos._sys_db.wait_for_buffer_flush()
     assert handle.get_status().status == WorkflowStatusString.SUCCESS.value
 

{dbos-0.21.0a3 → dbos-0.21.0a4}/tests/test_queue.py

@@ -17,6 +17,7 @@ from dbos import (
     SetWorkflowID,
     WorkflowHandle,
 )
+from dbos._error import DBOSDeadLetterQueueError
 from dbos._schemas.system_database import SystemSchema
 from dbos._sys_db import WorkflowStatusString
 from tests.conftest import default_config
@@ -581,3 +582,190 @@ def test_duplicate_workflow_id(dbos: DBOS, caplog: pytest.LogCaptureFixture) ->
 
     # Reset logging
     logging.getLogger("dbos").propagate = original_propagate
+
+
+def test_queue_recovery(dbos: DBOS) -> None:
+    step_counter: int = 0
+    queued_steps = 5
+
+    wfid = str(uuid.uuid4())
+    queue = Queue("test_queue")
+    step_events = [threading.Event() for _ in range(queued_steps)]
+    event = threading.Event()
+
+    @DBOS.workflow()
+    def test_workflow() -> list[int]:
+        assert DBOS.workflow_id == wfid
+        handles = []
+        for i in range(queued_steps):
+            h = queue.enqueue(test_step, i)
+            handles.append(h)
+        return [h.get_result() for h in handles]
+
+    @DBOS.step()
+    def test_step(i: int) -> int:
+        nonlocal step_counter
+        step_counter += 1
+        step_events[i].set()
+        event.wait()
+        return i
+
+    # Start the workflow. Wait for all five steps to start. Verify that they started.
+    with SetWorkflowID(wfid):
+        original_handle = DBOS.start_workflow(test_workflow)
+    for e in step_events:
+        e.wait()
+    assert step_counter == 5
+
+    # Recover the workflow, then resume it.
+    recovery_handles = DBOS.recover_pending_workflows()
+    event.set()
+    # There should be one handle for the workflow and another for each queued step.
+    assert len(recovery_handles) == queued_steps + 1
+    # Verify that both the recovered and original workflows complete correctly.
+    for h in recovery_handles:
+        if h.get_workflow_id() == wfid:
+            assert h.get_result() == [0, 1, 2, 3, 4]
+    assert original_handle.get_result() == [0, 1, 2, 3, 4]
+    # Each step should start twice, once originally and once in recovery.
+    assert step_counter == 10
+
+    # Rerun the workflow. Because each step is complete, none should start again.
+    with SetWorkflowID(wfid):
+        assert test_workflow() == [0, 1, 2, 3, 4]
+    assert step_counter == 10
+
+    # Verify all queue entries eventually get cleaned up.
+    assert queue_entries_are_cleaned_up(dbos)
+
+
+def test_cancelling_queued_workflows(dbos: DBOS) -> None:
+    start_event = threading.Event()
+    blocking_event = threading.Event()
+
+    @DBOS.workflow()
+    def stuck_workflow() -> None:
+        start_event.set()
+        blocking_event.wait()
+
+    @DBOS.workflow()
+    def regular_workflow() -> None:
+        return
+
+    # Enqueue both the blocked workflow and a regular workflow on a queue with concurrency 1
+    queue = Queue("test_queue", concurrency=1)
+    wfid = str(uuid.uuid4())
+    with SetWorkflowID(wfid):
+        blocked_handle = queue.enqueue(stuck_workflow)
+    regular_handle = queue.enqueue(regular_workflow)
+
+    # Verify that the blocked workflow starts and is PENDING while the regular workflow remains ENQUEUED.
+    start_event.wait()
+    assert blocked_handle.get_status().status == WorkflowStatusString.PENDING.value
+    assert regular_handle.get_status().status == WorkflowStatusString.ENQUEUED.value
+
+    # Cancel the blocked workflow. Verify this lets the regular workflow run.
+    dbos.cancel_workflow(wfid)
+    assert blocked_handle.get_status().status == WorkflowStatusString.CANCELLED.value
+    assert regular_handle.get_result() == None
+
+    # Complete the blocked workflow
+    blocking_event.set()
+    assert blocked_handle.get_result() == None
+
+    # Verify all queue entries eventually get cleaned up.
+    assert queue_entries_are_cleaned_up(dbos)
+
+
+def test_resuming_queued_workflows(dbos: DBOS) -> None:
+    start_event = threading.Event()
+    blocking_event = threading.Event()
+
+    @DBOS.workflow()
+    def stuck_workflow() -> None:
+        start_event.set()
+        blocking_event.wait()
+
+    @DBOS.workflow()
+    def regular_workflow() -> None:
+        return
+
+    # Enqueue a blocked workflow and two regular workflows on a queue with concurrency 1
+    queue = Queue("test_queue", concurrency=1)
+    wfid = str(uuid.uuid4())
+    blocked_handle = queue.enqueue(stuck_workflow)
+    with SetWorkflowID(wfid):
+        regular_handle_1 = queue.enqueue(regular_workflow)
+    regular_handle_2 = queue.enqueue(regular_workflow)
+
+    # Verify that the blocked workflow starts and is PENDING while the regular workflows remain ENQUEUED.
+    start_event.wait()
+    assert blocked_handle.get_status().status == WorkflowStatusString.PENDING.value
+    assert regular_handle_1.get_status().status == WorkflowStatusString.ENQUEUED.value
+    assert regular_handle_2.get_status().status == WorkflowStatusString.ENQUEUED.value
+
+    # Resume a regular workflow. Verify it completes.
+    dbos.resume_workflow(wfid)
+    assert regular_handle_1.get_result() == None
+
+    # Complete the blocked workflow. Verify the second regular workflow also completes.
+    blocking_event.set()
+    assert blocked_handle.get_result() == None
+    assert regular_handle_2.get_result() == None
+
+    # Verify all queue entries eventually get cleaned up.
+    assert queue_entries_are_cleaned_up(dbos)
+
+
+def test_dlq_enqueued_workflows(dbos: DBOS) -> None:
+    start_event = threading.Event()
+    blocking_event = threading.Event()
+    max_recovery_attempts = 10
+    recovery_count = 0
+
+    @DBOS.workflow(max_recovery_attempts=max_recovery_attempts)
+    def blocked_workflow() -> None:
+        start_event.set()
+        nonlocal recovery_count
+        recovery_count += 1
+        blocking_event.wait()
+
+    @DBOS.workflow()
+    def regular_workflow() -> None:
+        return
+
+    # Enqueue both the blocked workflow and a regular workflow on a queue with concurrency 1
+    queue = Queue("test_queue", concurrency=1)
+    blocked_handle = queue.enqueue(blocked_workflow)
+    regular_handle = queue.enqueue(regular_workflow)
+
+    # Verify that the blocked workflow starts and is PENDING while the regular workflow remains ENQUEUED.
+    start_event.wait()
+    assert blocked_handle.get_status().status == WorkflowStatusString.PENDING.value
+    assert regular_handle.get_status().status == WorkflowStatusString.ENQUEUED.value
+
+    # Attempt to recover the blocked workflow the maximum number of times
+    for i in range(max_recovery_attempts):
+        DBOS.recover_pending_workflows()
+        assert recovery_count == i + 2
+
+    # Verify an additional recovery throws a DLQ error and puts the workflow in the DLQ status.
+    with pytest.raises(Exception) as exc_info:
+        DBOS.recover_pending_workflows()
+    assert exc_info.errisinstance(DBOSDeadLetterQueueError)
+    assert (
+        blocked_handle.get_status().status
+        == WorkflowStatusString.RETRIES_EXCEEDED.value
+    )
+
+    # Verify the blocked workflow entering the DLQ lets the regular workflow run
+    assert regular_handle.get_result() == None
+
+    # Complete the blocked workflow
+    blocking_event.set()
+    assert blocked_handle.get_result() == None
+    dbos._sys_db.wait_for_buffer_flush()
+    assert blocked_handle.get_status().status == WorkflowStatusString.SUCCESS.value
+
+    # Verify all queue entries eventually get cleaned up.
+    assert queue_entries_are_cleaned_up(dbos)

{dbos-0.21.0a3 → dbos-0.21.0a4}/tests/test_workflow_cmds.py

@@ -31,7 +31,7 @@ def test_list_workflow(dbos: DBOS, config: ConfigFile) -> None:
     simple_workflow()
     time.sleep(1)  # wait for the workflow to complete
     # get the workflow list
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, None, False, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
@@ -54,7 +54,7 @@ def test_list_workflow_limit(dbos: DBOS, config: ConfigFile) -> None:
     simple_workflow()
     time.sleep(1)  # wait for the workflow to complete
     # get the workflow list
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 2, None, None, None, None, False, None
     )
     assert len(output) == 2, f"Expected list length to be 1, but got {len(output)}"
@@ -72,12 +72,12 @@ def test_list_workflow_status(dbos: DBOS, config: ConfigFile) -> None:
     simple_workflow()
     time.sleep(1)  # wait for the workflow to complete
     # get the workflow list
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, "PENDING", False, None
     )
     assert len(output) == 0, f"Expected list length to be 0, but got {len(output)}"
 
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, "SUCCESS", False, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
@@ -102,7 +102,7 @@ def test_list_workflow_start_end_times(dbos: DBOS, config: ConfigFile) -> None:
     endtime = datetime.now().isoformat()
     print(endtime)
 
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, starttime, endtime, None, False, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
@@ -110,7 +110,7 @@ def test_list_workflow_start_end_times(dbos: DBOS, config: ConfigFile) -> None:
     newstarttime = (now - timedelta(seconds=30)).isoformat()
     newendtime = starttime
 
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, newstarttime, newendtime, None, False, None
     )
     assert len(output) == 0, f"Expected list length to be 0, but got {len(output)}"
@@ -141,19 +141,19 @@ def test_list_workflow_end_times_positive(dbos: DBOS, config: ConfigFile) -> Non
     # get the workflow list
     time_3 = datetime.now().isoformat()
 
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, time_0, time_1, None, False, None
     )
 
     assert len(output) == 0, f"Expected list length to be 0, but got {len(output)}"
 
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, time_1, time_2, None, False, None
     )
 
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
 
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, time_1, time_3, None, False, None
     )
     assert len(output) == 2, f"Expected list length to be 2, but got {len(output)}"
@@ -171,7 +171,7 @@ def test_get_workflow(dbos: DBOS, config: ConfigFile) -> None:
     simple_workflow()
     time.sleep(1)  # wait for the workflow to complete
     # get the workflow list
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, None, False, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
@@ -180,7 +180,7 @@ def test_get_workflow(dbos: DBOS, config: ConfigFile) -> None:
 
     wfUuid = output[0].workflowUUID
 
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
     assert info is not None, "Expected output to be not None"
 
     if info is not None:
@@ -199,7 +199,7 @@ def test_cancel_workflow(dbos: DBOS, config: ConfigFile) -> None:
     # run the workflow
     simple_workflow()
     # get the workflow list
-    output = _workflow_commands._list_workflows(
+    output = _workflow_commands.list_workflows(
         config, 10, None, None, None, None, False, None
     )
     # assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
@@ -208,9 +208,9 @@ def test_cancel_workflow(dbos: DBOS, config: ConfigFile) -> None:
     assert output[0] != None, "Expected output to be not None"
     wfUuid = output[0].workflowUUID
 
-    _workflow_commands._cancel_workflow(config, wfUuid)
+    _workflow_commands.cancel_workflow(config, wfUuid)
 
-    info = _workflow_commands._get_workflow(config, wfUuid, True)
+    info = _workflow_commands.get_workflow(config, wfUuid, True)
     assert info is not None, "Expected info to be not None"
     if info is not None:
         assert info.status == "CANCELLED", f"Expected status to be CANCELLED"