dbos 1.7.0a2__tar.gz → 1.7.0a3__tar.gz

{dbos-1.7.0a2 → dbos-1.7.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 1.7.0a2
+Version: 1.7.0a3
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-1.7.0a2 → dbos-1.7.0a3}/dbos/_queue.py

@@ -1,3 +1,4 @@
+import random
 import threading
 from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, TypedDict
 
@@ -94,8 +95,12 @@ class Queue:
 
 
 def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
+    polling_interval = 1.0
+    min_polling_interval = 1.0
+    max_polling_interval = 120.0
     while not stop_event.is_set():
-        if stop_event.wait(timeout=1):
+        # Wait for the polling interval with jitter
+        if stop_event.wait(timeout=polling_interval * random.uniform(0.95, 1.05)):
             return
         queues = dict(dbos._registry.queue_info_map)
         for _, queue in queues.items():
@@ -106,12 +111,22 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
                 for id in wf_ids:
                     execute_workflow_by_id(dbos, id)
             except OperationalError as e:
-                # Ignore serialization error
-                if not isinstance(
+                if isinstance(
                     e.orig, (errors.SerializationFailure, errors.LockNotAvailable)
                 ):
+                    # If a serialization error is encountered, increase the polling interval
+                    polling_interval = min(
+                        max_polling_interval,
+                        polling_interval * 2.0,
+                    )
+                    dbos.logger.warning(
+                        f"Contention detected in queue thread for {queue.name}. Increasing polling interval to {polling_interval:.2f}."
+                    )
+                else:
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
             except Exception as e:
                 if not stop_event.is_set():
                     # Only print the error if the thread is not stopping
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
+        # Attempt to scale back the polling interval on each iteration
+        polling_interval = max(min_polling_interval, polling_interval * 0.9)

{dbos-1.7.0a2 → dbos-1.7.0a3}/dbos/_sys_db.py

@@ -1650,7 +1650,7 @@ class SystemDatabase:
                     return []
 
             # Compute max_tasks, the number of workflows that can be dequeued given local and global concurrency limits,
-            max_tasks = float("inf")
+            max_tasks = 100  # To minimize contention with large queues, never dequeue more than 100 tasks
             if queue.worker_concurrency is not None or queue.concurrency is not None:
                 # Count how many workflows on this queue are currently PENDING both locally and globally.
                 pending_tasks_query = (
@@ -1694,6 +1694,7 @@ class SystemDatabase:
 
             # Retrieve the first max_tasks workflows in the queue.
             # Only retrieve workflows of the local version (or without version set)
+            skip_locks = queue.concurrency is None
             query = (
                 sa.select(
                     SystemSchema.workflow_status.c.workflow_uuid,
@@ -1711,7 +1712,10 @@ class SystemDatabase:
                         SystemSchema.workflow_status.c.application_version.is_(None),
                     )
                 )
-                .with_for_update(nowait=True)  # Error out early
+                # Unless global concurrency is set, use skip_locked to only select
+                # rows that can be locked. If global concurrency is set, use no_wait
+                # to ensure all processes have a consistent view of the table.
+                .with_for_update(skip_locked=skip_locks, nowait=(not skip_locks))
             )
             if queue.priority_enabled:
                 query = query.order_by(
@@ -1720,9 +1724,7 @@ class SystemDatabase:
                 )
             else:
                 query = query.order_by(SystemSchema.workflow_status.c.created_at.asc())
-            # Apply limit only if max_tasks is finite
-            if max_tasks != float("inf"):
-                query = query.limit(int(max_tasks))
+            query = query.limit(int(max_tasks))
 
             rows = c.execute(query).fetchall()
 

{dbos-1.7.0a2 → dbos-1.7.0a3}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "1.7.0a2"
+version = "1.7.0a3"
 
 [project.license]
 text = "MIT"

{dbos-1.7.0a2 → dbos-1.7.0a3}/tests/test_queue.py

@@ -215,7 +215,7 @@ def test_limiter(dbos: DBOS) -> None:
         return time.time()
 
     limit = 5
-    period = 2
+    period = 1.8
     queue = Queue("test_queue", limiter={"limit": limit, "period": period})
 
     handles: list[WorkflowHandle[float]] = []
@@ -235,12 +235,12 @@ def test_limiter(dbos: DBOS) -> None:
     # Verify that each "wave" of tasks started at the ~same time.
     for wave in range(num_waves):
         for i in range(wave * limit, (wave + 1) * limit - 1):
-            assert times[i + 1] - times[i] < 0.3
+            assert times[i + 1] - times[i] < 0.5
 
     # Verify that the gap between "waves" is ~equal to the period
     for wave in range(num_waves - 1):
-        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.3
-        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.3
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.5
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.5
 
     # Verify all workflows get the SUCCESS status eventually
     for h in handles:
@@ -280,7 +280,7 @@ def test_multiple_queues(dbos: DBOS) -> None:
         return time.time()
 
     limit = 5
-    period = 2
+    period = 1.8
     limiter_queue = Queue(
         "test_limit_queue", limiter={"limit": limit, "period": period}
     )
@@ -302,12 +302,12 @@ def test_multiple_queues(dbos: DBOS) -> None:
     # Verify that each "wave" of tasks started at the ~same time.
     for wave in range(num_waves):
         for i in range(wave * limit, (wave + 1) * limit - 1):
-            assert times[i + 1] - times[i] < 0.3
+            assert times[i + 1] - times[i] < 0.5
 
     # Verify that the gap between "waves" is ~equal to the period
     for wave in range(num_waves - 1):
-        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.3
-        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.3
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.5
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.5
 
     # Verify all workflows get the SUCCESS status eventually
     for h in handles:

{dbos-1.7.0a2 → dbos-1.7.0a3}/tests/test_scheduler.py

@@ -101,8 +101,8 @@ def test_scheduled_workflow(dbos: DBOS) -> None:
         nonlocal wf_counter
         wf_counter += 1
 
-    time.sleep(4)
-    assert wf_counter > 2 and wf_counter <= 4
+    time.sleep(5)
+    assert wf_counter > 2 and wf_counter <= 5
 
 
 def test_appdb_downtime(dbos: DBOS) -> None:
@@ -152,8 +152,8 @@ def test_scheduled_transaction(dbos: DBOS) -> None:
         nonlocal txn_counter
         txn_counter += 1
 
-    time.sleep(4)
-    assert txn_counter > 2 and txn_counter <= 4
+    time.sleep(5)
+    assert txn_counter > 2 and txn_counter <= 5
 
 
 def test_scheduled_step(dbos: DBOS) -> None:
@@ -165,8 +165,8 @@ def test_scheduled_step(dbos: DBOS) -> None:
         nonlocal step_counter
         step_counter += 1
 
-    time.sleep(4)
-    assert step_counter > 2 and step_counter <= 4
+    time.sleep(5)
+    assert step_counter > 2 and step_counter <= 5
 
 
 def test_scheduled_workflow_exception(dbos: DBOS) -> None:
@@ -179,8 +179,8 @@ def test_scheduled_workflow_exception(dbos: DBOS) -> None:
         wf_counter += 1
         raise Exception("error")
 
-    time.sleep(3)
-    assert wf_counter >= 1 and wf_counter <= 3
+    time.sleep(4)
+    assert wf_counter >= 1 and wf_counter <= 4
 
 
 def test_scheduler_oaoo(dbos: DBOS) -> None: