dbos 1.6.0__tar.gz → 1.8.0a1__tar.gz

{dbos-1.6.0 → dbos-1.8.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 1.6.0
+Version: 1.8.0a1
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_admin_server.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict
 
 from dbos._workflow_commands import garbage_collect, global_timeout
 
+from ._conductor import protocol as conductor_protocol
 from ._context import SetWorkflowID
 from ._error import DBOSException
 from ._logger import dbos_logger
@@ -118,7 +119,12 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
                     self.send_response(404)
                     self._end_headers()
                     return
-                response_body = json.dumps(workflows[0].__dict__).encode("utf-8")
+                workflow_output = (
+                    conductor_protocol.WorkflowsOutput.from_workflow_information(
+                        workflows[0]
+                    )
+                )
+                response_body = json.dumps(workflow_output.__dict__).encode("utf-8")
                 self.send_response(200)
                 self.send_header("Content-Type", "application/json")
                 self.send_header("Content-Length", str(len(response_body)))
@@ -326,20 +332,26 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
 
     def _handle_workflows(self, filters: Dict[str, Any]) -> None:
         workflows = self.dbos.list_workflows(
-            workflow_ids=filters.get("workflow_ids"),
-            name=filters.get("name"),
+            workflow_ids=filters.get("workflow_uuids"),
+            user=filters.get("authenticated_user"),
             start_time=filters.get("start_time"),
             end_time=filters.get("end_time"),
             status=filters.get("status"),
             app_version=filters.get("application_version"),
+            name=filters.get("workflow_name"),
             limit=filters.get("limit"),
             offset=filters.get("offset"),
             sort_desc=filters.get("sort_desc", False),
             workflow_id_prefix=filters.get("workflow_id_prefix"),
+            load_input=filters.get("load_input", False),
+            load_output=filters.get("load_output", False),
         )
-
+        workflows_output = [
+            conductor_protocol.WorkflowsOutput.from_workflow_information(i)
+            for i in workflows
+        ]
         response_body = json.dumps(
-            [workflow.__dict__ for workflow in workflows]
+            [workflow.__dict__ for workflow in workflows_output]
         ).encode("utf-8")
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
@@ -349,18 +361,22 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
 
     def _handle_queued_workflows(self, filters: Dict[str, Any]) -> None:
         workflows = self.dbos.list_queued_workflows(
-            queue_name=filters.get("queue_name"),
-            name=filters.get("name"),
             start_time=filters.get("start_time"),
             end_time=filters.get("end_time"),
             status=filters.get("status"),
+            name=filters.get("workflow_name"),
             limit=filters.get("limit"),
             offset=filters.get("offset"),
+            queue_name=filters.get("queue_name"),
             sort_desc=filters.get("sort_desc", False),
+            load_input=filters.get("load_input", False),
         )
-
+        workflows_output = [
+            conductor_protocol.WorkflowsOutput.from_workflow_information(i)
+            for i in workflows
+        ]
         response_body = json.dumps(
-            [workflow.__dict__ for workflow in workflows]
+            [workflow.__dict__ for workflow in workflows_output]
         ).encode("utf-8")
         self.send_response(200)
         self.send_header("Content-Type", "application/json")

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_client.py

@@ -294,6 +294,8 @@ class DBOSClient:
         offset: Optional[int] = None,
         sort_desc: bool = False,
         workflow_id_prefix: Optional[str] = None,
+        load_input: bool = True,
+        load_output: bool = True,
     ) -> List[WorkflowStatus]:
         return list_workflows(
             self._sys_db,
@@ -308,6 +310,8 @@ class DBOSClient:
             offset=offset,
             sort_desc=sort_desc,
             workflow_id_prefix=workflow_id_prefix,
+            load_input=load_input,
+            load_output=load_output,
         )
 
     async def list_workflows_async(
@@ -324,6 +328,8 @@ class DBOSClient:
         offset: Optional[int] = None,
         sort_desc: bool = False,
         workflow_id_prefix: Optional[str] = None,
+        load_input: bool = True,
+        load_output: bool = True,
     ) -> List[WorkflowStatus]:
         return await asyncio.to_thread(
             self.list_workflows,
@@ -338,6 +344,8 @@ class DBOSClient:
             offset=offset,
             sort_desc=sort_desc,
             workflow_id_prefix=workflow_id_prefix,
+            load_input=load_input,
+            load_output=load_output,
         )
 
     def list_queued_workflows(
@@ -351,6 +359,7 @@ class DBOSClient:
         limit: Optional[int] = None,
         offset: Optional[int] = None,
         sort_desc: bool = False,
+        load_input: bool = True,
     ) -> List[WorkflowStatus]:
         return list_queued_workflows(
             self._sys_db,
@@ -362,6 +371,7 @@ class DBOSClient:
             limit=limit,
             offset=offset,
             sort_desc=sort_desc,
+            load_input=load_input,
         )
 
     async def list_queued_workflows_async(
@@ -375,6 +385,7 @@ class DBOSClient:
         limit: Optional[int] = None,
         offset: Optional[int] = None,
         sort_desc: bool = False,
+        load_input: bool = True,
     ) -> List[WorkflowStatus]:
         return await asyncio.to_thread(
             self.list_queued_workflows,
@@ -386,6 +397,7 @@ class DBOSClient:
             limit=limit,
             offset=offset,
             sort_desc=sort_desc,
+            load_input=load_input,
         )
 
     def list_workflow_steps(self, workflow_id: str) -> List[StepInfo]:

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_conductor/conductor.py

@@ -223,6 +223,8 @@ class ConductorWebsocket(threading.Thread):
                             body = list_workflows_message.body
                             infos = []
                             try:
+                                load_input = body.get("load_input", False)
+                                load_output = body.get("load_output", False)
                                 infos = list_workflows(
                                     self.dbos._sys_db,
                                     workflow_ids=body["workflow_uuids"],
@@ -235,6 +237,8 @@ class ConductorWebsocket(threading.Thread):
                                     limit=body["limit"],
                                     offset=body["offset"],
                                     sort_desc=body["sort_desc"],
+                                    load_input=load_input,
+                                    load_output=load_output,
                                 )
                             except Exception as e:
                                 error_message = f"Exception encountered when listing workflows: {traceback.format_exc()}"
@@ -257,6 +261,7 @@ class ConductorWebsocket(threading.Thread):
                             q_body = list_queued_workflows_message.body
                             infos = []
                             try:
+                                q_load_input = q_body.get("load_input", False)
                                 infos = list_queued_workflows(
                                     self.dbos._sys_db,
                                     start_time=q_body["start_time"],
@@ -267,6 +272,7 @@ class ConductorWebsocket(threading.Thread):
                                     offset=q_body["offset"],
                                     queue_name=q_body["queue_name"],
                                     sort_desc=q_body["sort_desc"],
+                                    load_input=q_load_input,
                                 )
                             except Exception as e:
                                 error_message = f"Exception encountered when listing queued workflows: {traceback.format_exc()}"

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_conductor/protocol.py

@@ -110,7 +110,7 @@ class RestartResponse(BaseMessage):
     error_message: Optional[str] = None
 
 
-class ListWorkflowsBody(TypedDict):
+class ListWorkflowsBody(TypedDict, total=False):
     workflow_uuids: List[str]
     workflow_name: Optional[str]
     authenticated_user: Optional[str]
@@ -121,6 +121,8 @@ class ListWorkflowsBody(TypedDict):
     limit: Optional[int]
     offset: Optional[int]
     sort_desc: bool
+    load_input: bool
+    load_output: bool
 
 
 @dataclass
@@ -209,7 +211,7 @@ class ListWorkflowsResponse(BaseMessage):
     error_message: Optional[str] = None
 
 
-class ListQueuedWorkflowsBody(TypedDict):
+class ListQueuedWorkflowsBody(TypedDict, total=False):
     workflow_name: Optional[str]
     start_time: Optional[str]
     end_time: Optional[str]
@@ -218,6 +220,7 @@ class ListQueuedWorkflowsBody(TypedDict):
     limit: Optional[int]
     offset: Optional[int]
     sort_desc: bool
+    load_input: bool
 
 
 @dataclass

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_core.py

@@ -1157,13 +1157,16 @@ def decorate_step(
         def wrapper(*args: Any, **kwargs: Any) -> Any:
             rr: Optional[str] = check_required_roles(func, fi)
             # Entering step is allowed:
+            #  No DBOS, just call the original function directly
             #  In a step already, just call the original function directly.
             #  In a workflow (that is not in a step already)
             #  Not in a workflow (we will start the single op workflow)
+            if not dbosreg.dbos or not dbosreg.dbos._launched:
+                # Call the original function directly
+                return func(*args, **kwargs)
             ctx = get_local_dbos_context()
             if ctx and ctx.is_step():
                 # Call the original function directly
-
                 return func(*args, **kwargs)
             if ctx and ctx.is_within_workflow():
                 assert ctx.is_workflow(), "Steps must be called from within workflows"

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_dbos.py

@@ -7,7 +7,6 @@ import inspect
 import os
 import sys
 import threading
-import traceback
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from logging import Logger
@@ -28,6 +27,7 @@ from typing import (
 )
 
 from opentelemetry.trace import Span
+from rich import print
 
 from dbos._conductor.conductor import ConductorWebsocket
 from dbos._sys_db import WorkflowStatus
@@ -517,6 +517,16 @@ class DBOS:
 
             dbos_logger.info("DBOS launched!")
 
+            if self.conductor_key is None and os.environ.get("DBOS__CLOUD") != "true":
+                # Hint the user to open the URL to register and set up Conductor
+                app_name = self._config["name"]
+                conductor_registration_url = (
+                    f"https://console.dbos.dev/self-host?appname={app_name}"
+                )
+                print(
+                    f"[bold]To view and manage workflows, connect to DBOS Conductor at:[/bold] [bold blue]{conductor_registration_url}[/bold blue]"
+                )
+
             # Flush handlers and add OTLP to all loggers if enabled
             # to enable their export in DBOS Cloud
             for handler in dbos_logger.handlers:
@@ -1022,6 +1032,8 @@ class DBOS:
         offset: Optional[int] = None,
         sort_desc: bool = False,
         workflow_id_prefix: Optional[str] = None,
+        load_input: bool = True,
+        load_output: bool = True,
     ) -> List[WorkflowStatus]:
         def fn() -> List[WorkflowStatus]:
             return list_workflows(
@@ -1037,6 +1049,8 @@ class DBOS:
                 offset=offset,
                 sort_desc=sort_desc,
                 workflow_id_prefix=workflow_id_prefix,
+                load_input=load_input,
+                load_output=load_output,
             )
 
         return _get_dbos_instance()._sys_db.call_function_as_step(
@@ -1055,6 +1069,7 @@ class DBOS:
         limit: Optional[int] = None,
         offset: Optional[int] = None,
         sort_desc: bool = False,
+        load_input: bool = True,
     ) -> List[WorkflowStatus]:
         def fn() -> List[WorkflowStatus]:
             return list_queued_workflows(
@@ -1067,6 +1082,7 @@ class DBOS:
                 limit=limit,
                 offset=offset,
                 sort_desc=sort_desc,
+                load_input=load_input,
             )
 
         return _get_dbos_instance()._sys_db.call_function_as_step(

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_queue.py

@@ -1,3 +1,4 @@
+import random
 import threading
 from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, TypedDict
 
@@ -94,8 +95,12 @@ class Queue:
 
 
 def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
+    polling_interval = 1.0
+    min_polling_interval = 1.0
+    max_polling_interval = 120.0
     while not stop_event.is_set():
-        if stop_event.wait(timeout=1):
+        # Wait for the polling interval with jitter
+        if stop_event.wait(timeout=polling_interval * random.uniform(0.95, 1.05)):
             return
         queues = dict(dbos._registry.queue_info_map)
         for _, queue in queues.items():
@@ -106,12 +111,22 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
                 for id in wf_ids:
                     execute_workflow_by_id(dbos, id)
             except OperationalError as e:
-                # Ignore serialization error
-                if not isinstance(
+                if isinstance(
                     e.orig, (errors.SerializationFailure, errors.LockNotAvailable)
                 ):
+                    # If a serialization error is encountered, increase the polling interval
+                    polling_interval = min(
+                        max_polling_interval,
+                        polling_interval * 2.0,
+                    )
+                    dbos.logger.warning(
+                        f"Contention detected in queue thread for {queue.name}. Increasing polling interval to {polling_interval:.2f}."
+                    )
+                else:
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
             except Exception as e:
                 if not stop_event.is_set():
                     # Only print the error if the thread is not stopping
                     dbos.logger.warning(f"Exception encountered in queue thread: {e}")
+        # Attempt to scale back the polling interval on each iteration
+        polling_interval = max(min_polling_interval, polling_interval * 0.9)

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_sys_db.py

@@ -788,11 +788,17 @@ class SystemDatabase:
                     pass  # CB: I guess we're assuming the WF will show up eventually.
             time.sleep(1)
 
-    def get_workflows(self, input: GetWorkflowsInput) -> List[WorkflowStatus]:
+    def get_workflows(
+        self,
+        input: GetWorkflowsInput,
+        *,
+        load_input: bool = True,
+        load_output: bool = True,
+    ) -> List[WorkflowStatus]:
         """
         Retrieve a list of workflows result and inputs based on the input criteria. The result is a list of external-facing workflow status objects.
         """
-        query = sa.select(
+        load_columns = [
             SystemSchema.workflow_status.c.workflow_uuid,
             SystemSchema.workflow_status.c.status,
             SystemSchema.workflow_status.c.name,
@@ -808,12 +814,16 @@ class SystemDatabase:
             SystemSchema.workflow_status.c.updated_at,
             SystemSchema.workflow_status.c.application_version,
             SystemSchema.workflow_status.c.application_id,
-            SystemSchema.workflow_status.c.inputs,
-            SystemSchema.workflow_status.c.output,
-            SystemSchema.workflow_status.c.error,
             SystemSchema.workflow_status.c.workflow_deadline_epoch_ms,
             SystemSchema.workflow_status.c.workflow_timeout_ms,
-        )
+        ]
+        if load_input:
+            load_columns.append(SystemSchema.workflow_status.c.inputs)
+        if load_output:
+            load_columns.append(SystemSchema.workflow_status.c.output)
+            load_columns.append(SystemSchema.workflow_status.c.error)
+
+        query = sa.select(*load_columns)
         if input.sort_desc:
             query = query.order_by(SystemSchema.workflow_status.c.created_at.desc())
         else:
@@ -880,29 +890,35 @@ class SystemDatabase:
             info.updated_at = row[12]
             info.app_version = row[13]
             info.app_id = row[14]
+            info.workflow_deadline_epoch_ms = row[15]
+            info.workflow_timeout_ms = row[16]
 
+            raw_input = row[17] if load_input else None
+            raw_output = row[18] if load_output else None
+            raw_error = row[19] if load_output else None
             inputs, output, exception = _serialization.safe_deserialize(
                 info.workflow_id,
-                serialized_input=row[15],
-                serialized_output=row[16],
-                serialized_exception=row[17],
+                serialized_input=raw_input,
+                serialized_output=raw_output,
+                serialized_exception=raw_error,
             )
             info.input = inputs
             info.output = output
             info.error = exception
-            info.workflow_deadline_epoch_ms = row[18]
-            info.workflow_timeout_ms = row[19]
 
             infos.append(info)
         return infos
 
     def get_queued_workflows(
-        self, input: GetQueuedWorkflowsInput
+        self,
+        input: GetQueuedWorkflowsInput,
+        *,
+        load_input: bool = True,
     ) -> List[WorkflowStatus]:
         """
         Retrieve a list of queued workflows result and inputs based on the input criteria. The result is a list of external-facing workflow status objects.
         """
-        query = sa.select(
+        load_columns = [
             SystemSchema.workflow_status.c.workflow_uuid,
             SystemSchema.workflow_status.c.status,
             SystemSchema.workflow_status.c.name,
@@ -918,12 +934,13 @@ class SystemDatabase:
             SystemSchema.workflow_status.c.updated_at,
             SystemSchema.workflow_status.c.application_version,
             SystemSchema.workflow_status.c.application_id,
-            SystemSchema.workflow_status.c.inputs,
-            SystemSchema.workflow_status.c.output,
-            SystemSchema.workflow_status.c.error,
             SystemSchema.workflow_status.c.workflow_deadline_epoch_ms,
             SystemSchema.workflow_status.c.workflow_timeout_ms,
-        ).where(
+        ]
+        if load_input:
+            load_columns.append(SystemSchema.workflow_status.c.inputs)
+
+        query = sa.select(*load_columns).where(
             sa.and_(
                 SystemSchema.workflow_status.c.queue_name.isnot(None),
                 SystemSchema.workflow_status.c.status.in_(["ENQUEUED", "PENDING"]),
@@ -984,18 +1001,21 @@ class SystemDatabase:
             info.updated_at = row[12]
             info.app_version = row[13]
             info.app_id = row[14]
+            info.workflow_deadline_epoch_ms = row[15]
+            info.workflow_timeout_ms = row[16]
+
+            raw_input = row[17] if load_input else None
 
+            # Error and Output are not loaded because they should always be None for queued workflows.
             inputs, output, exception = _serialization.safe_deserialize(
                 info.workflow_id,
-                serialized_input=row[15],
-                serialized_output=row[16],
-                serialized_exception=row[17],
+                serialized_input=raw_input,
+                serialized_output=None,
+                serialized_exception=None,
             )
             info.input = inputs
             info.output = output
             info.error = exception
-            info.workflow_deadline_epoch_ms = row[18]
-            info.workflow_timeout_ms = row[19]
 
             infos.append(info)
 
@@ -1650,7 +1670,7 @@ class SystemDatabase:
                     return []
 
             # Compute max_tasks, the number of workflows that can be dequeued given local and global concurrency limits,
-            max_tasks = float("inf")
+            max_tasks = 100  # To minimize contention with large queues, never dequeue more than 100 tasks
             if queue.worker_concurrency is not None or queue.concurrency is not None:
                 # Count how many workflows on this queue are currently PENDING both locally and globally.
                 pending_tasks_query = (
@@ -1694,6 +1714,7 @@ class SystemDatabase:
 
             # Retrieve the first max_tasks workflows in the queue.
             # Only retrieve workflows of the local version (or without version set)
+            skip_locks = queue.concurrency is None
             query = (
                 sa.select(
                     SystemSchema.workflow_status.c.workflow_uuid,
@@ -1711,7 +1732,10 @@ class SystemDatabase:
                         SystemSchema.workflow_status.c.application_version.is_(None),
                     )
                 )
-                .with_for_update(nowait=True)  # Error out early
+                # Unless global concurrency is set, use skip_locked to only select
+                # rows that can be locked. If global concurrency is set, use no_wait
+                # to ensure all processes have a consistent view of the table.
+                .with_for_update(skip_locked=skip_locks, nowait=(not skip_locks))
             )
             if queue.priority_enabled:
                 query = query.order_by(
@@ -1720,9 +1744,7 @@ class SystemDatabase:
                 )
             else:
                 query = query.order_by(SystemSchema.workflow_status.c.created_at.asc())
-            # Apply limit only if max_tasks is finite
-            if max_tasks != float("inf"):
-                query = query.limit(int(max_tasks))
+            query = query.limit(int(max_tasks))
 
             rows = c.execute(query).fetchall()
 

{dbos-1.6.0 → dbos-1.8.0a1}/dbos/_workflow_commands.py

@@ -33,6 +33,8 @@ def list_workflows(
     offset: Optional[int] = None,
     sort_desc: bool = False,
     workflow_id_prefix: Optional[str] = None,
+    load_input: bool = True,
+    load_output: bool = True,
 ) -> List[WorkflowStatus]:
     input = GetWorkflowsInput()
     input.workflow_ids = workflow_ids
@@ -47,7 +49,9 @@ def list_workflows(
     input.sort_desc = sort_desc
     input.workflow_id_prefix = workflow_id_prefix
 
-    infos: List[WorkflowStatus] = sys_db.get_workflows(input)
+    infos: List[WorkflowStatus] = sys_db.get_workflows(
+        input, load_input=load_input, load_output=load_output
+    )
 
     return infos
 
@@ -63,6 +67,7 @@ def list_queued_workflows(
     limit: Optional[int] = None,
     offset: Optional[int] = None,
     sort_desc: bool = False,
+    load_input: bool = True,
 ) -> List[WorkflowStatus]:
     input: GetQueuedWorkflowsInput = {
         "queue_name": queue_name,
@@ -75,7 +80,9 @@ def list_queued_workflows(
         "sort_desc": sort_desc,
     }
 
-    infos: List[WorkflowStatus] = sys_db.get_queued_workflows(input)
+    infos: List[WorkflowStatus] = sys_db.get_queued_workflows(
+        input, load_input=load_input
+    )
     return infos
 
 

{dbos-1.6.0 → dbos-1.8.0a1}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "1.6.0"
+version = "1.8.0a1"
 
 [project.license]
 text = "MIT"