dbos 0.21.0a4__tar.gz → 0.21.0a7__tar.gz

{dbos-0.21.0a4 → dbos-0.21.0a7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.21.0a4
+Version: 0.21.0a7
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.21.0a4 → dbos-0.21.0a7}/dbos/_dbos.py

@@ -56,7 +56,7 @@ from ._registrations import (
 )
 from ._roles import default_required_roles, required_roles
 from ._scheduler import ScheduledWorkflow, scheduled
-from ._sys_db import WorkflowStatusString, reset_system_database
+from ._sys_db import reset_system_database
 from ._tracer import dbos_tracer
 
 if TYPE_CHECKING:
@@ -613,6 +613,7 @@ class DBOS:
             workflow_id=workflow_id,
             status=stat["status"],
             name=stat["name"],
+            executor_id=stat["executor_id"],
             recovery_attempts=stat["recovery_attempts"],
             class_name=stat["class_name"],
             config_name=stat["config_name"],
@@ -909,6 +910,7 @@ class WorkflowStatus:
         workflow_id(str):  The ID of the workflow execution
         status(str):  The status of the execution, from `WorkflowStatusString`
         name(str): The workflow function name
+        executor_id(str): The ID of the executor running the workflow
         class_name(str): For member functions, the name of the class containing the workflow function
         config_name(str): For instance member functions, the name of the class instance for the execution
         queue_name(str): For workflows that are or were queued, the queue name
@@ -922,6 +924,7 @@ class WorkflowStatus:
     workflow_id: str
     status: str
     name: str
+    executor_id: Optional[str]
     class_name: Optional[str]
     config_name: Optional[str]
     queue_name: Optional[str]

{dbos-0.21.0a4 → dbos-0.21.0a7}/dbos/_dbos_config.py

@@ -123,7 +123,10 @@ def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
 
 
 def load_config(
-    config_file_path: str = DBOS_CONFIG_PATH, *, use_db_wizard: bool = True
+    config_file_path: str = DBOS_CONFIG_PATH,
+    *,
+    use_db_wizard: bool = True,
+    silent: bool = False,
 ) -> ConfigFile:
     """
     Load the DBOS `ConfigFile` from the specified path (typically `dbos-config.yaml`).
@@ -188,18 +191,19 @@ def load_config(
     # Load the DB connection file. Use its values for missing fields from dbos-config.yaml. Use defaults otherwise.
     data = cast(ConfigFile, data)
     db_connection = load_db_connection()
-    if data["database"].get("hostname"):
-        print(
-            "[bold blue]Loading database connection parameters from dbos-config.yaml[/bold blue]"
-        )
-    elif db_connection.get("hostname"):
-        print(
-            "[bold blue]Loading database connection parameters from .dbos/db_connection[/bold blue]"
-        )
-    else:
-        print(
-            "[bold blue]Using default database connection parameters (localhost)[/bold blue]"
-        )
+    if not silent:
+        if data["database"].get("hostname"):
+            print(
+                "[bold blue]Loading database connection parameters from dbos-config.yaml[/bold blue]"
+            )
+        elif db_connection.get("hostname"):
+            print(
+                "[bold blue]Loading database connection parameters from .dbos/db_connection[/bold blue]"
+            )
+        else:
+            print(
+                "[bold blue]Using default database connection parameters (localhost)[/bold blue]"
+            )
 
     data["database"]["hostname"] = (
         data["database"].get("hostname") or db_connection.get("hostname") or "localhost"

dbos-0.21.0a7/dbos/_recovery.py

@@ -0,0 +1,70 @@
+import os
+import threading
+import time
+import traceback
+from typing import TYPE_CHECKING, Any, List
+
+from ._core import execute_workflow_by_id
+from ._error import DBOSWorkflowFunctionNotFoundError
+from ._sys_db import GetPendingWorkflowsOutput
+
+if TYPE_CHECKING:
+    from ._dbos import DBOS, WorkflowHandle
+
+
+def startup_recovery_thread(
+    dbos: "DBOS", pending_workflows: List[GetPendingWorkflowsOutput]
+) -> None:
+    """Attempt to recover local pending workflows on startup using a background thread."""
+    stop_event = threading.Event()
+    dbos.stop_events.append(stop_event)
+    while not stop_event.is_set() and len(pending_workflows) > 0:
+        try:
+            for pending_workflow in list(pending_workflows):
+                if (
+                    pending_workflow.queue_name
+                    and pending_workflow.queue_name != "_dbos_internal_queue"
+                ):
+                    dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
+                    continue
+                execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
+                pending_workflows.remove(pending_workflow)
+        except DBOSWorkflowFunctionNotFoundError:
+            time.sleep(1)
+        except Exception as e:
+            dbos.logger.error(
+                f"Exception encountered when recovering workflows: {traceback.format_exc()}"
+            )
+            raise e
+
+
+def recover_pending_workflows(
+    dbos: "DBOS", executor_ids: List[str] = ["local"]
+) -> List["WorkflowHandle[Any]"]:
+    workflow_handles: List["WorkflowHandle[Any]"] = []
+    for executor_id in executor_ids:
+        if executor_id == "local" and os.environ.get("DBOS__VMID"):
+            dbos.logger.debug(
+                f"Skip local recovery because it's running in a VM: {os.environ.get('DBOS__VMID')}"
+            )
+        dbos.logger.debug(f"Recovering pending workflows for executor: {executor_id}")
+        pending_workflows = dbos._sys_db.get_pending_workflows(executor_id)
+        for pending_workflow in pending_workflows:
+            if (
+                pending_workflow.queue_name
+                and pending_workflow.queue_name != "_dbos_internal_queue"
+            ):
+                try:
+                    dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
+                    workflow_handles.append(
+                        dbos.retrieve_workflow(pending_workflow.workflow_uuid)
+                    )
+                except Exception as e:
+                    dbos.logger.error(e)
+            else:
+                workflow_handles.append(
+                    execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
+                )
+
+    dbos.logger.info("Recovered pending workflows")
+    return workflow_handles

{dbos-0.21.0a4 → dbos-0.21.0a7}/dbos/_sys_db.py

@@ -126,11 +126,26 @@ class GetWorkflowsInput:
         )
 
 
+class GetQueuedWorkflowsInput(TypedDict):
+    queue_name: Optional[str]
+    status: Optional[str]
+    start_time: Optional[str]  # Timestamp in ISO 8601 format
+    end_time: Optional[str]  # Timestamp in ISO 8601 format
+    limit: Optional[int]  # Return up to this many workflows IDs.
+    name: Optional[str]  # The name of the workflow function
+
+
 class GetWorkflowsOutput:
     def __init__(self, workflow_uuids: List[str]):
         self.workflow_uuids = workflow_uuids
 
 
+class GetPendingWorkflowsOutput:
+    def __init__(self, *, workflow_uuid: str, queue_name: Optional[str] = None):
+        self.workflow_uuid: str = workflow_uuid
+        self.queue_name: Optional[str] = queue_name
+
+
 class WorkflowInformation(TypedDict, total=False):
     workflow_uuid: str
     status: WorkflowStatuses  # The status of the workflow.
@@ -456,6 +471,7 @@ class SystemDatabase:
                     SystemSchema.workflow_status.c.authenticated_roles,
                     SystemSchema.workflow_status.c.assumed_role,
                     SystemSchema.workflow_status.c.queue_name,
+                    SystemSchema.workflow_status.c.executor_id,
                 ).where(SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid)
             ).fetchone()
             if row is None:
@@ -470,7 +486,7 @@ class SystemDatabase:
                 "error": None,
                 "app_id": None,
                 "app_version": None,
-                "executor_id": None,
+                "executor_id": row[10],
                 "request": row[2],
                 "recovery_attempts": row[3],
                 "authenticated_user": row[6],
@@ -656,9 +672,8 @@ class SystemDatabase:
 
     def get_workflows(self, input: GetWorkflowsInput) -> GetWorkflowsOutput:
         query = sa.select(SystemSchema.workflow_status.c.workflow_uuid).order_by(
-            SystemSchema.workflow_status.c.created_at.desc()
+            SystemSchema.workflow_status.c.created_at.asc()
         )
-
         if input.name:
             query = query.where(SystemSchema.workflow_status.c.name == input.name)
         if input.authenticated_user:
@@ -692,16 +707,73 @@ class SystemDatabase:
 
         return GetWorkflowsOutput(workflow_uuids)
 
-    def get_pending_workflows(self, executor_id: str) -> list[str]:
+    def get_queued_workflows(
+        self, input: GetQueuedWorkflowsInput
+    ) -> GetWorkflowsOutput:
+
+        query = (
+            sa.select(SystemSchema.workflow_queue.c.workflow_uuid)
+            .join(
+                SystemSchema.workflow_status,
+                SystemSchema.workflow_queue.c.workflow_uuid
+                == SystemSchema.workflow_status.c.workflow_uuid,
+            )
+            .order_by(SystemSchema.workflow_status.c.created_at.asc())
+        )
+
+        if input.get("name"):
+            query = query.where(SystemSchema.workflow_status.c.name == input["name"])
+
+        if input.get("queue_name"):
+            query = query.where(
+                SystemSchema.workflow_queue.c.queue_name == input["queue_name"]
+            )
+
+        if input.get("status"):
+            query = query.where(
+                SystemSchema.workflow_status.c.status == input["status"]
+            )
+        if "start_time" in input and input["start_time"] is not None:
+            query = query.where(
+                SystemSchema.workflow_status.c.created_at
+                >= datetime.datetime.fromisoformat(input["start_time"]).timestamp()
+                * 1000
+            )
+        if "end_time" in input and input["end_time"] is not None:
+            query = query.where(
+                SystemSchema.workflow_status.c.created_at
+                <= datetime.datetime.fromisoformat(input["end_time"]).timestamp() * 1000
+            )
+        if input.get("limit"):
+            query = query.limit(input["limit"])
+
+        with self.engine.begin() as c:
+            rows = c.execute(query)
+        workflow_uuids = [row[0] for row in rows]
+
+        return GetWorkflowsOutput(workflow_uuids)
+
+    def get_pending_workflows(
+        self, executor_id: str
+    ) -> list[GetPendingWorkflowsOutput]:
         with self.engine.begin() as c:
             rows = c.execute(
-                sa.select(SystemSchema.workflow_status.c.workflow_uuid).where(
+                sa.select(
+                    SystemSchema.workflow_status.c.workflow_uuid,
+                    SystemSchema.workflow_status.c.queue_name,
+                ).where(
                     SystemSchema.workflow_status.c.status
                     == WorkflowStatusString.PENDING.value,
                     SystemSchema.workflow_status.c.executor_id == executor_id,
                 )
             ).fetchall()
-            return [row[0] for row in rows]
+            return [
+                GetPendingWorkflowsOutput(
+                    workflow_uuid=row.workflow_uuid,
+                    queue_name=row.queue_name,
+                )
+                for row in rows
+            ]
 
     def record_operation_result(
         self, result: OperationResultInternal, conn: Optional[sa.Connection] = None
@@ -1321,6 +1393,19 @@ class SystemDatabase:
                     .values(completed_at_epoch_ms=int(time.time() * 1000))
                 )
 
+    def clear_queue_assignment(self, workflow_id: str) -> None:
+        with self.engine.begin() as c:
+            c.execute(
+                sa.update(SystemSchema.workflow_queue)
+                .where(SystemSchema.workflow_queue.c.workflow_uuid == workflow_id)
+                .values(executor_id=None, started_at_epoch_ms=None)
+            )
+            c.execute(
+                sa.update(SystemSchema.workflow_status)
+                .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
+                .values(executor_id=None, status=WorkflowStatusString.ENQUEUED.value)
+            )
+
 
 def reset_system_database(config: ConfigFile) -> None:
     sysdb_name = (

{dbos-0.21.0a4 → dbos-0.21.0a7}/dbos/_workflow_commands.py

@@ -5,6 +5,7 @@ import typer
 from . import _serialization
 from ._dbos_config import ConfigFile
 from ._sys_db import (
+    GetQueuedWorkflowsInput,
     GetWorkflowsInput,
     GetWorkflowsOutput,
     SystemDatabase,
@@ -19,8 +20,8 @@ class WorkflowInformation:
     workflowClassName: Optional[str]
     workflowConfigName: Optional[str]
     input: Optional[_serialization.WorkflowInputs]  # JSON (jsonpickle)
-    output: Optional[str]  # JSON (jsonpickle)
-    error: Optional[str]  # JSON (jsonpickle)
+    output: Optional[str] = None  # JSON (jsonpickle)
+    error: Optional[str] = None  # JSON (jsonpickle)
     executor_id: Optional[str]
     app_version: Optional[str]
     app_id: Optional[str]
@@ -34,17 +35,15 @@ class WorkflowInformation:
 
 def list_workflows(
     config: ConfigFile,
-    li: int,
+    limit: int,
     user: Optional[str],
     starttime: Optional[str],
     endtime: Optional[str],
     status: Optional[str],
     request: bool,
     appversion: Optional[str],
+    name: Optional[str],
 ) -> List[WorkflowInformation]:
-
-    sys_db = None
-
     try:
         sys_db = SystemDatabase(config)
 
@@ -55,24 +54,55 @@ def list_workflows(
         if status is not None:
             input.status = cast(WorkflowStatuses, status)
         input.application_version = appversion
-        input.limit = li
+        input.limit = limit
+        input.name = name
 
         output: GetWorkflowsOutput = sys_db.get_workflows(input)
-
         infos: List[WorkflowInformation] = []
+        for workflow_id in output.workflow_uuids:
+            info = _get_workflow_info(
+                sys_db, workflow_id, request
+            )  # Call the method for each ID
+            if info is not None:
+                infos.append(info)
+
+        return infos
+    except Exception as e:
+        typer.echo(f"Error listing workflows: {e}")
+        return []
+    finally:
+        if sys_db:
+            sys_db.destroy()
 
-        if output.workflow_uuids is None:
-            typer.echo("No workflows found")
-            return {}
 
+def list_queued_workflows(
+    config: ConfigFile,
+    limit: Optional[int] = None,
+    start_time: Optional[str] = None,
+    end_time: Optional[str] = None,
+    queue_name: Optional[str] = None,
+    status: Optional[str] = None,
+    name: Optional[str] = None,
+    request: bool = False,
+) -> List[WorkflowInformation]:
+    try:
+        sys_db = SystemDatabase(config)
+        input: GetQueuedWorkflowsInput = {
+            "queue_name": queue_name,
+            "start_time": start_time,
+            "end_time": end_time,
+            "status": status,
+            "limit": limit,
+            "name": name,
+        }
+        output: GetWorkflowsOutput = sys_db.get_queued_workflows(input)
+        infos: List[WorkflowInformation] = []
         for workflow_id in output.workflow_uuids:
             info = _get_workflow_info(
                 sys_db, workflow_id, request
             )  # Call the method for each ID
-
             if info is not None:
                 infos.append(info)
-
         return infos
     except Exception as e:
         typer.echo(f"Error listing workflows: {e}")

{dbos-0.21.0a4 → dbos-0.21.0a7}/dbos/cli/cli.py

@@ -19,14 +19,21 @@ from .. import load_config
 from .._app_db import ApplicationDatabase
 from .._dbos_config import _is_valid_app_name
 from .._sys_db import SystemDatabase, reset_system_database
-from .._workflow_commands import cancel_workflow, get_workflow, list_workflows
+from .._workflow_commands import (
+    cancel_workflow,
+    get_workflow,
+    list_queued_workflows,
+    list_workflows,
+)
 from ..cli._github_init import create_template_from_github
 from ._template_init import copy_template, get_project_name, get_templates_directory
 
 app = typer.Typer()
 workflow = typer.Typer()
+queue = typer.Typer()
 
 app.add_typer(workflow, name="workflow", help="Manage DBOS workflows")
+workflow.add_typer(queue, name="queue", help="Manage enqueued workflows")
 
 
 def _on_windows() -> bool:
@@ -272,18 +279,22 @@ def list(
             help="Retrieve workflows with this application version",
         ),
     ] = None,
+    name: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--name",
+            "-n",
+            help="Retrieve workflows with this name",
+        ),
+    ] = None,
     request: Annotated[
         bool,
         typer.Option("--request", help="Retrieve workflow request information"),
     ] = True,
-    appdir: Annotated[
-        typing.Optional[str],
-        typer.Option("--app-dir", "-d", help="Specify the application root directory"),
-    ] = None,
 ) -> None:
-    config = load_config()
+    config = load_config(silent=True)
     workflows = list_workflows(
-        config, limit, user, starttime, endtime, status, request, appversion
+        config, limit, user, starttime, endtime, status, request, appversion, name
     )
     print(jsonpickle.encode(workflows, unpicklable=False))
 
@@ -291,16 +302,12 @@ def list(
 @workflow.command(help="Retrieve the status of a workflow")
 def get(
     uuid: Annotated[str, typer.Argument()],
-    appdir: Annotated[
-        typing.Optional[str],
-        typer.Option("--app-dir", "-d", help="Specify the application root directory"),
-    ] = None,
     request: Annotated[
         bool,
         typer.Option("--request", help="Retrieve workflow request information"),
     ] = True,
 ) -> None:
-    config = load_config()
+    config = load_config(silent=True)
     print(jsonpickle.encode(get_workflow(config, uuid, request), unpicklable=False))
 
 
@@ -309,10 +316,6 @@ def get(
 )
 def cancel(
     uuid: Annotated[str, typer.Argument()],
-    appdir: Annotated[
-        typing.Optional[str],
-        typer.Option("--app-dir", "-d", help="Specify the application root directory"),
-    ] = None,
 ) -> None:
     config = load_config()
     cancel_workflow(config, uuid)
@@ -363,5 +366,70 @@ def restart(
         print(f"Failed to resume workflow {uuid}. Status code: {response.status_code}")
 
 
+@queue.command(name="list", help="List enqueued functions for your application")
+def list_queue(
+    limit: Annotated[
+        typing.Optional[int],
+        typer.Option("--limit", "-l", help="Limit the results returned"),
+    ] = None,
+    start_time: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--start-time",
+            "-s",
+            help="Retrieve functions starting after this timestamp (ISO 8601 format)",
+        ),
+    ] = None,
+    end_time: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--end-time",
+            "-e",
+            help="Retrieve functions starting before this timestamp (ISO 8601 format)",
+        ),
+    ] = None,
+    status: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--status",
+            "-S",
+            help="Retrieve functions with this status (PENDING, SUCCESS, ERROR, RETRIES_EXCEEDED, ENQUEUED, or CANCELLED)",
+        ),
+    ] = None,
+    queue_name: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--queue-name",
+            "-q",
+            help="Retrieve functions on this queue",
+        ),
+    ] = None,
+    name: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--name",
+            "-n",
+            help="Retrieve functions on this queue",
+        ),
+    ] = None,
+    request: Annotated[
+        bool,
+        typer.Option("--request", help="Retrieve workflow request information"),
+    ] = True,
+) -> None:
+    config = load_config(silent=True)
+    workflows = list_queued_workflows(
+        config=config,
+        limit=limit,
+        start_time=start_time,
+        end_time=end_time,
+        queue_name=queue_name,
+        status=status,
+        request=request,
+        name=name,
+    )
+    print(jsonpickle.encode(workflows, unpicklable=False))
+
+
 if __name__ == "__main__":
     app()

{dbos-0.21.0a4 → dbos-0.21.0a7}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "0.21.0a4"
+version = "0.21.0a7"
 
 [project.license]
 text = "MIT"

{dbos-0.21.0a4 → dbos-0.21.0a7}/tests/test_admin_server.py

@@ -165,7 +165,7 @@ def test_admin_workflow_resume(dbos: DBOS, config: ConfigFile) -> None:
 
     # Verify the workflow has succeeded
     output = _workflow_commands.list_workflows(
-        config, 10, None, None, None, None, False, None
+        config, 10, None, None, None, None, False, None, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
     assert output[0] != None, "Expected output to be not None"
@@ -219,7 +219,7 @@ def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
 
     # get the workflow list
     output = _workflow_commands.list_workflows(
-        config, 10, None, None, None, None, False, None
+        config, 10, None, None, None, None, False, None, None
     )
     assert len(output) == 1, f"Expected list length to be 1, but got {len(output)}"
 
@@ -257,7 +257,7 @@ def test_admin_workflow_restart(dbos: DBOS, config: ConfigFile) -> None:
         assert False, "Expected info to be not None"
 
     output = _workflow_commands.list_workflows(
-        config, 10, None, None, None, None, False, None
+        config, 10, None, None, None, None, False, None, None
     )
     assert len(output) == 2, f"Expected list length to be 2, but got {len(output)}"
 

{dbos-0.21.0a4 → dbos-0.21.0a7}/tests/test_dbos.py

@@ -905,10 +905,10 @@ def test_send_recv_temp_wf(dbos: DBOS) -> None:
 
     wfs = dbos._sys_db.get_workflows(gwi)
     assert len(wfs.workflow_uuids) == 2
-    assert wfs.workflow_uuids[1] == dest_uuid
-    assert wfs.workflow_uuids[0] != dest_uuid
+    assert wfs.workflow_uuids[0] == dest_uuid
+    assert wfs.workflow_uuids[1] != dest_uuid
 
-    wfi = dbos._sys_db.get_workflow_info(wfs.workflow_uuids[0], False)
+    wfi = dbos._sys_db.get_workflow_info(wfs.workflow_uuids[1], False)
     assert wfi
     assert wfi["name"] == "<temp>.temp_send_workflow"
 

{dbos-0.21.0a4 → dbos-0.21.0a7}/tests/test_failures.py

@@ -9,7 +9,7 @@ from psycopg.errors import SerializationFailure
 from sqlalchemy.exc import InvalidRequestError, OperationalError
 
 # Public API
-from dbos import DBOS, GetWorkflowsInput, Queue, SetWorkflowID
+from dbos import DBOS, GetWorkflowsInput, SetWorkflowID
 from dbos._error import DBOSDeadLetterQueueError, DBOSException
 from dbos._sys_db import WorkflowStatusString
 

{dbos-0.21.0a4 → dbos-0.21.0a7}/tests/test_package.py

@@ -8,6 +8,7 @@ import time
 import urllib.error
 import urllib.request
 
+import requests
 import sqlalchemy as sa
 import yaml
 
@@ -147,3 +148,48 @@ def test_reset(postgres_db_engine: sa.Engine) -> None:
                 )
             ).scalar()
             assert result == 0
+
+
+def test_list_commands() -> None:
+    app_name = "reset-app"
+    with tempfile.TemporaryDirectory() as temp_path:
+        subprocess.check_call(
+            ["dbos", "init", app_name, "--template", "dbos-toolbox"],
+            cwd=temp_path,
+        )
+        subprocess.check_call(["dbos", "reset", "-y"], cwd=temp_path)
+        subprocess.check_call(["dbos", "migrate"], cwd=temp_path)
+
+        # Get some workflows enqueued on the toolbox, then kill the toolbox
+        process = subprocess.Popen(["dbos", "start"], cwd=temp_path)
+        try:
+            session = requests.Session()
+            for i in range(10):
+                try:
+                    session.get(
+                        "http://localhost:8000/queue", timeout=1
+                    ).raise_for_status()
+                    break
+                except requests.exceptions.Timeout:
+                    break
+                except requests.exceptions.ConnectionError as e:
+                    if i == 9:
+                        raise
+                    print(f"Attempt {i+1} failed: {e}. Retrying in 1 second...")
+                    time.sleep(1)
+            time.sleep(1)  # So the queued workflows can start
+        finally:
+            os.kill(process.pid, signal.SIGINT)
+            process.wait()
+
+        # Verify the output is valid JSON
+        output = subprocess.check_output(["dbos", "workflow", "list"], cwd=temp_path)
+        data = json.loads(output)
+        assert isinstance(data, list) and len(data) == 10
+
+        # Verify the output is valid JSON
+        output = subprocess.check_output(
+            ["dbos", "workflow", "queue", "list"], cwd=temp_path
+        )
+        data = json.loads(output)
+        assert isinstance(data, list) and len(data) == 10