dbos 1.6.0a5__tar.gz → 1.7.0a3__tar.gz

{dbos-1.6.0a5 → dbos-1.7.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 1.6.0a5
+Version: 1.7.0a3
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-1.6.0a5 → dbos-1.7.0a3}/dbos/_admin_server.py

@@ -3,12 +3,14 @@ from __future__ import annotations
 import json
 import re
 import threading
+from dataclasses import asdict
 from functools import partial
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict
 
 from dbos._workflow_commands import garbage_collect, global_timeout
 
+from ._conductor import protocol as conductor_protocol
 from ._context import SetWorkflowID
 from ._error import DBOSException
 from ._logger import dbos_logger
@@ -326,20 +328,24 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
 
     def _handle_workflows(self, filters: Dict[str, Any]) -> None:
         workflows = self.dbos.list_workflows(
-            workflow_ids=filters.get("workflow_ids"),
-            name=filters.get("name"),
+            workflow_ids=filters.get("workflow_uuids"),
+            user=filters.get("authenticated_user"),
             start_time=filters.get("start_time"),
             end_time=filters.get("end_time"),
             status=filters.get("status"),
             app_version=filters.get("application_version"),
+            name=filters.get("workflow_name"),
             limit=filters.get("limit"),
             offset=filters.get("offset"),
             sort_desc=filters.get("sort_desc", False),
             workflow_id_prefix=filters.get("workflow_id_prefix"),
         )
-
+        workflows_output = [
+            conductor_protocol.WorkflowsOutput.from_workflow_information(i)
+            for i in workflows
+        ]
         response_body = json.dumps(
-            [workflow.__dict__ for workflow in workflows]
+            [workflow.__dict__ for workflow in workflows_output]
         ).encode("utf-8")
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
@@ -349,18 +355,21 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
 
     def _handle_queued_workflows(self, filters: Dict[str, Any]) -> None:
         workflows = self.dbos.list_queued_workflows(
-            queue_name=filters.get("queue_name"),
-            name=filters.get("name"),
             start_time=filters.get("start_time"),
             end_time=filters.get("end_time"),
             status=filters.get("status"),
+            name=filters.get("workflow_name"),
             limit=filters.get("limit"),
             offset=filters.get("offset"),
+            queue_name=filters.get("queue_name"),
             sort_desc=filters.get("sort_desc", False),
         )
-
+        workflows_output = [
+            conductor_protocol.WorkflowsOutput.from_workflow_information(i)
+            for i in workflows
+        ]
         response_body = json.dumps(
-            [workflow.__dict__ for workflow in workflows]
+            [workflow.__dict__ for workflow in workflows_output]
         ).encode("utf-8")
         self.send_response(200)
         self.send_header("Content-Type", "application/json")

{dbos-1.6.0a5 → dbos-1.7.0a3}/dbos/_dbos.py

@@ -7,7 +7,6 @@ import inspect
 import os
 import sys
 import threading
-import traceback
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from logging import Logger
@@ -28,6 +27,7 @@ from typing import (
 )
 
 from opentelemetry.trace import Span
+from rich import print
 
 from dbos._conductor.conductor import ConductorWebsocket
 from dbos._sys_db import WorkflowStatus
@@ -517,6 +517,16 @@ class DBOS:
 
             dbos_logger.info("DBOS launched!")
 
+            if self.conductor_key is None and os.environ.get("DBOS__CLOUD") != "true":
+                # Hint the user to open the URL to register and set up Conductor
+                app_name = self._config["name"]
+                conductor_registration_url = (
+                    f"https://console.dbos.dev/self-host?appname={app_name}"
+                )
+                print(
+                    f"[bold]To view and manage workflows, connect to DBOS Conductor at:[/bold] [bold blue]{conductor_registration_url}[/bold blue]"
+                )
+
             # Flush handlers and add OTLP to all loggers if enabled
             # to enable their export in DBOS Cloud
             for handler in dbos_logger.handlers:

{dbos-1.6.0a5 → dbos-1.7.0a3}/dbos/_queue.py

@@ -1,3 +1,4 @@
+import random
 import threading
 from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, TypedDict
 
@@ -94,8 +95,12 @@ class Queue:
 
 
 def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
+    polling_interval = 1.0
+    min_polling_interval = 1.0
+    max_polling_interval = 120.0
     while not stop_event.is_set():
-        if stop_event.wait(timeout=1):
+        # Wait for the polling interval with jitter
+        if stop_event.wait(timeout=polling_interval * random.uniform(0.95, 1.05)):
             return
         queues = dict(dbos._registry.queue_info_map)
         for _, queue in queues.items():
@@ -106,12 +111,22 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
                 for id in wf_ids:
                     execute_workflow_by_id(dbos, id)
             except OperationalError as e:
-                # Ignore serialization error
-                if not isinstance(
+                if isinstance(
                     e.orig, (errors.SerializationFailure, errors.LockNotAvailable)
                 ):
+                    # If a serialization error is encountered, increase the polling interval
+                    polling_interval = min(
+                        max_polling_interval,
+                        polling_interval * 2.0,
+                    )
+                    dbos.logger.warning(
+                        f"Contention detected in queue thread for {queue.name}. Increasing polling interval to {polling_interval:.2f}."
+                    )
+                else:
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
             except Exception as e:
                 if not stop_event.is_set():
                     # Only print the error if the thread is not stopping
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
+        # Attempt to scale back the polling interval on each iteration
+        polling_interval = max(min_polling_interval, polling_interval * 0.9)

{dbos-1.6.0a5 → dbos-1.7.0a3}/dbos/_sys_db.py

@@ -1650,7 +1650,7 @@ class SystemDatabase:
                     return []
 
             # Compute max_tasks, the number of workflows that can be dequeued given local and global concurrency limits,
-            max_tasks = float("inf")
+            max_tasks = 100  # To minimize contention with large queues, never dequeue more than 100 tasks
             if queue.worker_concurrency is not None or queue.concurrency is not None:
                 # Count how many workflows on this queue are currently PENDING both locally and globally.
                 pending_tasks_query = (
@@ -1694,6 +1694,7 @@ class SystemDatabase:
 
             # Retrieve the first max_tasks workflows in the queue.
             # Only retrieve workflows of the local version (or without version set)
+            skip_locks = queue.concurrency is None
             query = (
                 sa.select(
                     SystemSchema.workflow_status.c.workflow_uuid,
@@ -1711,7 +1712,10 @@ class SystemDatabase:
                         SystemSchema.workflow_status.c.application_version.is_(None),
                     )
                 )
-                .with_for_update(nowait=True)  # Error out early
+                # Unless global concurrency is set, use skip_locked to only select
+                # rows that can be locked. If global concurrency is set, use no_wait
+                # to ensure all processes have a consistent view of the table.
+                .with_for_update(skip_locked=skip_locks, nowait=(not skip_locks))
             )
             if queue.priority_enabled:
                 query = query.order_by(
@@ -1720,9 +1724,7 @@ class SystemDatabase:
                 )
             else:
                 query = query.order_by(SystemSchema.workflow_status.c.created_at.asc())
-            # Apply limit only if max_tasks is finite
-            if max_tasks != float("inf"):
-                query = query.limit(int(max_tasks))
+            query = query.limit(int(max_tasks))
 
             rows = c.execute(query).fetchall()
 

{dbos-1.6.0a5 → dbos-1.7.0a3}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "1.6.0a5"
+version = "1.7.0a3"
 
 [project.license]
 text = "MIT"

{dbos-1.6.0a5 → dbos-1.7.0a3}/tests/test_admin_server.py

@@ -3,7 +3,8 @@ import socket
 import threading
 import time
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict
 
 import pytest
 import requests
@@ -462,13 +463,13 @@ def test_list_workflows(dbos: DBOS) -> None:
         pass
 
     @DBOS.workflow()
-    def test_workflow_2() -> None:
-        pass
+    def test_workflow_2(my_time: datetime) -> str:
+        return DBOS.workflow_id + " completed at " + my_time.isoformat()
 
     # Start workflows
     handle_1 = DBOS.start_workflow(test_workflow_1)
     time.sleep(2)  # Sleep for 2 seconds between workflows
-    handle_2 = DBOS.start_workflow(test_workflow_2)
+    handle_2 = DBOS.start_workflow(test_workflow_2, datetime.now())
 
     # Wait for workflows to complete
     handle_1.get_result()
@@ -492,8 +493,8 @@ def test_list_workflows(dbos: DBOS) -> None:
     ).isoformat()
 
     # Test POST /workflows with filters
-    filters = {
-        "workflow_ids": workflow_ids,
+    filters: Dict[str, Any] = {
+        "workflow_uuids": workflow_ids,
         "start_time": start_time_filter,
     }
     response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
@@ -501,7 +502,24 @@ def test_list_workflows(dbos: DBOS) -> None:
 
     workflows = response.json()
     assert len(workflows) == 1, f"Expected 1 workflows, but got {len(workflows)}"
-    assert workflows[0]["workflow_id"] == handle_2.workflow_id, "Workflow ID mismatch"
+
+    # Make sure it contains all the expected fields
+    assert workflows[0]["WorkflowUUID"] == handle_2.workflow_id, "Workflow ID mismatch"
+    assert workflows[0]["WorkflowName"] == test_workflow_2.__qualname__
+    assert workflows[0]["Status"] == "SUCCESS"
+    assert workflows[0]["WorkflowClassName"] is None
+    assert workflows[0]["WorkflowConfigName"] is None
+    assert workflows[0]["AuthenticatedUser"] is None
+    assert workflows[0]["AssumedRole"] is None
+    assert workflows[0]["AuthenticatedRoles"] is None
+    assert workflows[0]["Input"] is not None and len(workflows[0]["Input"]) > 0
+    assert workflows[0]["Output"] is not None and len(workflows[0]["Output"]) > 0
+    assert workflows[0]["Error"] is None
+    assert workflows[0]["CreatedAt"] is not None and len(workflows[0]["CreatedAt"]) > 0
+    assert workflows[0]["UpdatedAt"] is not None and len(workflows[0]["UpdatedAt"]) > 0
+    assert workflows[0]["QueueName"] is None
+    assert workflows[0]["ApplicationVersion"] == GlobalParams.app_version
+    assert workflows[0]["ExecutorID"] == GlobalParams.executor_id
 
     # Test POST /workflows without filters
     response = requests.post("http://localhost:3001/workflows", json={}, timeout=5)
@@ -512,7 +530,106 @@ def test_list_workflows(dbos: DBOS) -> None:
         workflows_list
     ), f"Expected {len(workflows_list)} workflows, but got {len(workflows)}"
     for workflow in workflows:
-        assert workflow["workflow_id"] in workflow_ids, "Workflow ID mismatch"
+        assert workflow["WorkflowUUID"] in workflow_ids, "Workflow ID mismatch"
+
+    # Verify sort_desc inverts the order
+    filters = {
+        "sort_desc": True,
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == len(workflows_list)
+    assert (
+        workflows[0]["WorkflowUUID"] == handle_2.workflow_id
+    ), "First workflow should be the last one started"
+
+    # Test all filters
+    filters = {
+        "workflow_uuids": ["not-a-valid-uuid"],
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 0, "Expected no workflows for invalid UUID"
+
+    filters = {
+        "workflow_uuids": [handle_1.workflow_id, handle_2.workflow_id],
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 2
+
+    filters = {
+        "authenticated_user": "no-user",
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 0
+
+    filters = {
+        "workflow_name": test_workflow_1.__qualname__,
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 1
+    assert workflows[0]["WorkflowUUID"] == handle_1.workflow_id
+
+    filters = {
+        "end_time": (datetime.now(timezone.utc) - timedelta(minutes=10)).isoformat()
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 0
+
+    filters = {
+        "start_time": (datetime.now(timezone.utc) + timedelta(minutes=10)).isoformat(),
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 0
+
+    filters = {
+        "status": ["SUCCESS", "CANCELLED"],
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 2
+
+    filters = {
+        "application_version": GlobalParams.app_version,
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 2
+
+    filters = {
+        "limit": 1,
+        "offset": 1,
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 1
+    assert workflows[0]["WorkflowUUID"] == handle_2.workflow_id
+
+    filters = {
+        "workflow_id_prefix": handle_1.workflow_id[
+            :10
+        ],  # First 10 characters of the workflow name
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+    workflows = response.json()
+    assert len(workflows) == 1
+    assert workflows[0]["WorkflowUUID"] == handle_1.workflow_id
 
 
 def test_get_workflow_by_id(dbos: DBOS) -> None:
@@ -606,15 +723,15 @@ def test_queued_workflows_endpoint(dbos: DBOS) -> None:
     test_queue2 = Queue("test-queue-2", concurrency=1)
 
     @DBOS.workflow()
-    def blocking_workflow() -> str:
+    def blocking_workflow(i: int) -> str:
         while True:
             time.sleep(0.1)
 
     # Enqueue some workflows to create queued entries
-    handles = []
-    handles.append(test_queue1.enqueue(blocking_workflow))
-    handles.append(test_queue1.enqueue(blocking_workflow))
-    handles.append(test_queue2.enqueue(blocking_workflow))
+    handles: list[WorkflowHandle[str]] = []
+    handles.append(test_queue1.enqueue(blocking_workflow, 1))
+    handles.append(test_queue1.enqueue(blocking_workflow, 2))
+    handles.append(test_queue2.enqueue(blocking_workflow, 3))
 
     # Test basic queued workflows endpoint
     response = requests.post("http://localhost:3001/queues", json={}, timeout=5)
@@ -628,16 +745,101 @@ def test_queued_workflows_endpoint(dbos: DBOS) -> None:
         len(queued_workflows) == 3
     ), f"Expected 3 queued workflows, got {len(queued_workflows)}"
 
-    # Test with filters
-    filters = {"queue_name": "test-queue-1", "limit": 1}
+    # Make sure it contains all the expected fields
+    assert queued_workflows[0]["WorkflowName"] == blocking_workflow.__qualname__
+    assert (
+        queued_workflows[0]["WorkflowUUID"] == handles[0].workflow_id
+    ), "Workflow ID mismatch"
+    assert (
+        queued_workflows[0]["Status"] == "ENQUEUED"
+        or queued_workflows[0]["Status"] == "PENDING"
+    )
+    assert queued_workflows[0]["WorkflowClassName"] is None
+    assert queued_workflows[0]["WorkflowConfigName"] is None
+    assert queued_workflows[0]["AuthenticatedUser"] is None
+    assert queued_workflows[0]["AssumedRole"] is None
+    assert queued_workflows[0]["AuthenticatedRoles"] is None
+    assert (
+        queued_workflows[0]["Input"] is not None
+        and len(queued_workflows[0]["Input"]) > 0
+    )
+    assert "1" in queued_workflows[0]["Input"]
+    assert queued_workflows[0]["Output"] is None
+    assert queued_workflows[0]["Error"] is None
+    assert (
+        queued_workflows[0]["CreatedAt"] is not None
+        and len(queued_workflows[0]["CreatedAt"]) > 0
+    )
+    assert (
+        queued_workflows[0]["UpdatedAt"] is not None
+        and len(queued_workflows[0]["UpdatedAt"]) > 0
+    )
+    assert queued_workflows[0]["QueueName"] == test_queue1.name
+    assert queued_workflows[0]["ApplicationVersion"] == GlobalParams.app_version
+    assert queued_workflows[0]["ExecutorID"] == GlobalParams.executor_id
+
+    # Verify sort_desc inverts the order
+    filters: Dict[str, Any] = {
+        "sort_desc": True,
+    }
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
+    filtered_workflows = response.json()
+    assert len(filtered_workflows) == len(handles)
+    assert (
+        filtered_workflows[0]["WorkflowUUID"] == handles[2].workflow_id
+    ), "First workflow should be the last one enqueued"
+
+    # Test all filters
+    filters = {
+        "workflow_name": blocking_workflow.__qualname__,
+    }
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
+    filtered_workflows = response.json()
+    assert len(filtered_workflows) == len(handles)
+
+    filters = {
+        "end_time": (datetime.now(timezone.utc) - timedelta(minutes=10)).isoformat(),
+    }
     response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
     assert response.status_code == 200
+    filtered_workflows = response.json()
+    assert len(filtered_workflows) == 0
+
+    filters = {
+        "start_time": (datetime.now(timezone.utc) + timedelta(minutes=10)).isoformat(),
+    }
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
+    filtered_workflows = response.json()
+    assert len(filtered_workflows) == 0
 
+    filters = {
+        "status": ["PENDING", "ENQUEUED"],
+    }
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
+    filtered_workflows = response.json()
+    assert len(filtered_workflows) == len(handles)
+
+    filters = {
+        "queue_name": test_queue1.name,
+    }
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
+    filtered_workflows = response.json()
+    assert len(filtered_workflows) == 2
+
+    filters = {"queue_name": test_queue1.name, "limit": 1, "offset": 1}
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
     filtered_workflows = response.json()
     assert isinstance(filtered_workflows, list), "Response should be a list"
     assert (
         len(filtered_workflows) == 1
     ), f"Expected 1 workflow, got {len(filtered_workflows)}"
+    assert filtered_workflows[0]["WorkflowUUID"] == handles[1].workflow_id
 
     # Test with non-existent queue name
     filters = {"queue_name": "non-existent-queue"}

{dbos-1.6.0a5 → dbos-1.7.0a3}/tests/test_queue.py

@@ -215,7 +215,7 @@ def test_limiter(dbos: DBOS) -> None:
         return time.time()
 
     limit = 5
-    period = 2
+    period = 1.8
     queue = Queue("test_queue", limiter={"limit": limit, "period": period})
 
     handles: list[WorkflowHandle[float]] = []
@@ -235,12 +235,12 @@ def test_limiter(dbos: DBOS) -> None:
     # Verify that each "wave" of tasks started at the ~same time.
     for wave in range(num_waves):
         for i in range(wave * limit, (wave + 1) * limit - 1):
-            assert times[i + 1] - times[i] < 0.3
+            assert times[i + 1] - times[i] < 0.5
 
     # Verify that the gap between "waves" is ~equal to the period
     for wave in range(num_waves - 1):
-        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.3
-        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.3
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.5
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.5
 
     # Verify all workflows get the SUCCESS status eventually
     for h in handles:
@@ -280,7 +280,7 @@ def test_multiple_queues(dbos: DBOS) -> None:
         return time.time()
 
     limit = 5
-    period = 2
+    period = 1.8
     limiter_queue = Queue(
         "test_limit_queue", limiter={"limit": limit, "period": period}
     )
@@ -302,12 +302,12 @@ def test_multiple_queues(dbos: DBOS) -> None:
     # Verify that each "wave" of tasks started at the ~same time.
     for wave in range(num_waves):
         for i in range(wave * limit, (wave + 1) * limit - 1):
-            assert times[i + 1] - times[i] < 0.3
+            assert times[i + 1] - times[i] < 0.5
 
     # Verify that the gap between "waves" is ~equal to the period
     for wave in range(num_waves - 1):
-        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.3
-        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.3
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.5
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.5
 
     # Verify all workflows get the SUCCESS status eventually
     for h in handles:

{dbos-1.6.0a5 → dbos-1.7.0a3}/tests/test_scheduler.py

@@ -101,8 +101,8 @@ def test_scheduled_workflow(dbos: DBOS) -> None:
         nonlocal wf_counter
         wf_counter += 1
 
-    time.sleep(4)
-    assert wf_counter > 2 and wf_counter <= 4
+    time.sleep(5)
+    assert wf_counter > 2 and wf_counter <= 5
 
 
 def test_appdb_downtime(dbos: DBOS) -> None:
@@ -152,8 +152,8 @@ def test_scheduled_transaction(dbos: DBOS) -> None:
         nonlocal txn_counter
         txn_counter += 1
 
-    time.sleep(4)
-    assert txn_counter > 2 and txn_counter <= 4
+    time.sleep(5)
+    assert txn_counter > 2 and txn_counter <= 5
 
 
 def test_scheduled_step(dbos: DBOS) -> None:
@@ -165,8 +165,8 @@ def test_scheduled_step(dbos: DBOS) -> None:
         nonlocal step_counter
         step_counter += 1
 
-    time.sleep(4)
-    assert step_counter > 2 and step_counter <= 4
+    time.sleep(5)
+    assert step_counter > 2 and step_counter <= 5
 
 
 def test_scheduled_workflow_exception(dbos: DBOS) -> None:
@@ -179,8 +179,8 @@ def test_scheduled_workflow_exception(dbos: DBOS) -> None:
         wf_counter += 1
         raise Exception("error")
 
-    time.sleep(3)
-    assert wf_counter >= 1 and wf_counter <= 3
+    time.sleep(4)
+    assert wf_counter >= 1 and wf_counter <= 4
 
 
 def test_scheduler_oaoo(dbos: DBOS) -> None: