dbos 1.7.0a2__tar.gz → 1.7.0a4__tar.gz

{dbos-1.7.0a2 → dbos-1.7.0a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 1.7.0a2
+Version: 1.7.0a4
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-1.7.0a2 → dbos-1.7.0a4}/dbos/_admin_server.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import json
 import re
 import threading
-from dataclasses import asdict
 from functools import partial
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict
@@ -120,7 +119,12 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
                     self.send_response(404)
                     self._end_headers()
                     return
-                response_body = json.dumps(workflows[0].__dict__).encode("utf-8")
+                workflow_output = (
+                    conductor_protocol.WorkflowsOutput.from_workflow_information(
+                        workflows[0]
+                    )
+                )
+                response_body = json.dumps(workflow_output.__dict__).encode("utf-8")
                 self.send_response(200)
                 self.send_header("Content-Type", "application/json")
                 self.send_header("Content-Length", str(len(response_body)))

{dbos-1.7.0a2 → dbos-1.7.0a4}/dbos/_queue.py

@@ -1,3 +1,4 @@
+import random
 import threading
 from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, TypedDict
 
@@ -94,8 +95,12 @@ class Queue:
 
 
 def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
+    polling_interval = 1.0
+    min_polling_interval = 1.0
+    max_polling_interval = 120.0
     while not stop_event.is_set():
-        if stop_event.wait(timeout=1):
+        # Wait for the polling interval with jitter
+        if stop_event.wait(timeout=polling_interval * random.uniform(0.95, 1.05)):
             return
         queues = dict(dbos._registry.queue_info_map)
         for _, queue in queues.items():
@@ -106,12 +111,22 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
                 for id in wf_ids:
                     execute_workflow_by_id(dbos, id)
             except OperationalError as e:
-                # Ignore serialization error
-                if not isinstance(
+                if isinstance(
                     e.orig, (errors.SerializationFailure, errors.LockNotAvailable)
                 ):
+                    # If a serialization error is encountered, increase the polling interval
+                    polling_interval = min(
+                        max_polling_interval,
+                        polling_interval * 2.0,
+                    )
+                    dbos.logger.warning(
+                        f"Contention detected in queue thread for {queue.name}. Increasing polling interval to {polling_interval:.2f}."
+                    )
+                else:
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
             except Exception as e:
                 if not stop_event.is_set():
                     # Only print the error if the thread is not stopping
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
+        # Attempt to scale back the polling interval on each iteration
+        polling_interval = max(min_polling_interval, polling_interval * 0.9)

{dbos-1.7.0a2 → dbos-1.7.0a4}/dbos/_sys_db.py

@@ -1650,7 +1650,7 @@ class SystemDatabase:
                     return []
 
             # Compute max_tasks, the number of workflows that can be dequeued given local and global concurrency limits,
-            max_tasks = float("inf")
+            max_tasks = 100  # To minimize contention with large queues, never dequeue more than 100 tasks
             if queue.worker_concurrency is not None or queue.concurrency is not None:
                 # Count how many workflows on this queue are currently PENDING both locally and globally.
                 pending_tasks_query = (
@@ -1694,6 +1694,7 @@ class SystemDatabase:
 
             # Retrieve the first max_tasks workflows in the queue.
             # Only retrieve workflows of the local version (or without version set)
+            skip_locks = queue.concurrency is None
             query = (
                 sa.select(
                     SystemSchema.workflow_status.c.workflow_uuid,
@@ -1711,7 +1712,10 @@ class SystemDatabase:
                         SystemSchema.workflow_status.c.application_version.is_(None),
                     )
                 )
-                .with_for_update(nowait=True)  # Error out early
+                # Unless global concurrency is set, use skip_locked to only select
+                # rows that can be locked. If global concurrency is set, use no_wait
+                # to ensure all processes have a consistent view of the table.
+                .with_for_update(skip_locked=skip_locks, nowait=(not skip_locks))
             )
             if queue.priority_enabled:
                 query = query.order_by(
@@ -1720,9 +1724,7 @@ class SystemDatabase:
                 )
             else:
                 query = query.order_by(SystemSchema.workflow_status.c.created_at.asc())
-            # Apply limit only if max_tasks is finite
-            if max_tasks != float("inf"):
-                query = query.limit(int(max_tasks))
+            query = query.limit(int(max_tasks))
 
             rows = c.execute(query).fetchall()
 

{dbos-1.7.0a2 → dbos-1.7.0a4}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "1.7.0a2"
+version = "1.7.0a4"
 
 [project.license]
 text = "MIT"

{dbos-1.7.0a2 → dbos-1.7.0a4}/tests/test_admin_server.py

@@ -639,16 +639,16 @@ def test_get_workflow_by_id(dbos: DBOS) -> None:
         pass
 
     @DBOS.workflow()
-    def test_workflow_2() -> None:
-        pass
+    def test_workflow_2(my_time: datetime) -> str:
+        return DBOS.workflow_id + " completed at " + my_time.isoformat()
 
     # Start workflows
     handle_1 = DBOS.start_workflow(test_workflow_1)
-    handle_2 = DBOS.start_workflow(test_workflow_2)
+    handle_2 = DBOS.start_workflow(test_workflow_2, datetime.now())
 
     # Wait for workflows to complete
     handle_1.get_result()
-    handle_2.get_result()
+    assert handle_2.get_result() is not None
 
     # Get the workflow ID of the second workflow
     workflow_id = handle_2.workflow_id
@@ -660,10 +660,28 @@ def test_get_workflow_by_id(dbos: DBOS) -> None:
     ), f"Expected status code 200, but got {response.status_code}"
 
     workflow_data = response.json()
-    assert workflow_data["workflow_id"] == workflow_id, "Workflow ID mismatch"
+    assert workflow_data["WorkflowUUID"] == workflow_id, "Workflow ID mismatch"
     assert (
-        workflow_data["status"] == "SUCCESS"
+        workflow_data["Status"] == "SUCCESS"
     ), "Expected workflow status to be SUCCESS"
+    assert workflow_data["WorkflowName"] == test_workflow_2.__qualname__
+    assert workflow_data["WorkflowClassName"] is None
+    assert workflow_data["WorkflowConfigName"] is None
+    assert workflow_data["AuthenticatedUser"] is None
+    assert workflow_data["AssumedRole"] is None
+    assert workflow_data["AuthenticatedRoles"] is None
+    assert workflow_data["Input"] is not None and len(workflow_data["Input"]) > 0
+    assert workflow_data["Output"] is not None and len(workflow_data["Output"]) > 0
+    assert workflow_data["Error"] is None
+    assert (
+        workflow_data["CreatedAt"] is not None and len(workflow_data["CreatedAt"]) > 0
+    )
+    assert (
+        workflow_data["UpdatedAt"] is not None and len(workflow_data["UpdatedAt"]) > 0
+    )
+    assert workflow_data["QueueName"] is None
+    assert workflow_data["ApplicationVersion"] == GlobalParams.app_version
+    assert workflow_data["ExecutorID"] == GlobalParams.executor_id
 
     # Test GET /workflows/:workflow_id for a non-existing workflow
     non_existing_workflow_id = "non-existing-id"

{dbos-1.7.0a2 → dbos-1.7.0a4}/tests/test_queue.py

@@ -215,7 +215,7 @@ def test_limiter(dbos: DBOS) -> None:
         return time.time()
 
     limit = 5
-    period = 2
+    period = 1.8
     queue = Queue("test_queue", limiter={"limit": limit, "period": period})
 
     handles: list[WorkflowHandle[float]] = []
@@ -235,12 +235,12 @@ def test_limiter(dbos: DBOS) -> None:
     # Verify that each "wave" of tasks started at the ~same time.
     for wave in range(num_waves):
         for i in range(wave * limit, (wave + 1) * limit - 1):
-            assert times[i + 1] - times[i] < 0.3
+            assert times[i + 1] - times[i] < 0.5
 
     # Verify that the gap between "waves" is ~equal to the period
     for wave in range(num_waves - 1):
-        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.3
-        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.3
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.5
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.5
 
     # Verify all workflows get the SUCCESS status eventually
     for h in handles:
@@ -280,7 +280,7 @@ def test_multiple_queues(dbos: DBOS) -> None:
         return time.time()
 
     limit = 5
-    period = 2
+    period = 1.8
     limiter_queue = Queue(
         "test_limit_queue", limiter={"limit": limit, "period": period}
     )
@@ -302,12 +302,12 @@ def test_multiple_queues(dbos: DBOS) -> None:
     # Verify that each "wave" of tasks started at the ~same time.
     for wave in range(num_waves):
         for i in range(wave * limit, (wave + 1) * limit - 1):
-            assert times[i + 1] - times[i] < 0.3
+            assert times[i + 1] - times[i] < 0.5
 
     # Verify that the gap between "waves" is ~equal to the period
     for wave in range(num_waves - 1):
-        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.3
-        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.3
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.5
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.5
 
     # Verify all workflows get the SUCCESS status eventually
     for h in handles:

{dbos-1.7.0a2 → dbos-1.7.0a4}/tests/test_scheduler.py

@@ -101,8 +101,8 @@ def test_scheduled_workflow(dbos: DBOS) -> None:
         nonlocal wf_counter
         wf_counter += 1
 
-    time.sleep(4)
-    assert wf_counter > 2 and wf_counter <= 4
+    time.sleep(5)
+    assert wf_counter > 2 and wf_counter <= 5
 
 
 def test_appdb_downtime(dbos: DBOS) -> None:
@@ -152,8 +152,8 @@ def test_scheduled_transaction(dbos: DBOS) -> None:
         nonlocal txn_counter
         txn_counter += 1
 
-    time.sleep(4)
-    assert txn_counter > 2 and txn_counter <= 4
+    time.sleep(5)
+    assert txn_counter > 2 and txn_counter <= 5
 
 
 def test_scheduled_step(dbos: DBOS) -> None:
@@ -165,8 +165,8 @@ def test_scheduled_step(dbos: DBOS) -> None:
         nonlocal step_counter
         step_counter += 1
 
-    time.sleep(4)
-    assert step_counter > 2 and step_counter <= 4
+    time.sleep(5)
+    assert step_counter > 2 and step_counter <= 5
 
 
 def test_scheduled_workflow_exception(dbos: DBOS) -> None:
@@ -179,8 +179,8 @@ def test_scheduled_workflow_exception(dbos: DBOS) -> None:
         wf_counter += 1
         raise Exception("error")
 
-    time.sleep(3)
-    assert wf_counter >= 1 and wf_counter <= 3
+    time.sleep(4)
+    assert wf_counter >= 1 and wf_counter <= 4
 
 
 def test_scheduler_oaoo(dbos: DBOS) -> None: