dbos 0.7.0a5__py3-none-any.whl → 0.7.0a8__py3-none-any.whl

dbos/__init__.py

@@ -3,6 +3,7 @@ from .context import DBOSContextEnsure, SetWorkflowID
 from .dbos import DBOS, DBOSConfiguredInstance, WorkflowHandle, WorkflowStatus
 from .dbos_config import ConfigFile, get_dbos_database_url, load_config
 from .kafka_message import KafkaMessage
+from .queue import Queue
 from .system_database import GetWorkflowsInput, WorkflowStatusString
 
 __all__ = [
@@ -19,4 +20,5 @@ __all__ = [
     "load_config",
     "get_dbos_database_url",
     "error",
+    "Queue",
 ]

dbos/context.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import json
 import os
 import uuid
+from contextlib import AbstractContextManager
 from contextvars import ContextVar
 from enum import Enum
 from types import TracebackType
@@ -344,7 +345,7 @@ class SetWorkflowRecovery:
         return False  # Did not handle
 
 
-class EnterDBOSWorkflow:
+class EnterDBOSWorkflow(AbstractContextManager[DBOSContext, Literal[False]]):
     def __init__(self, attributes: TracedAttributes) -> None:
         self.created_ctx = False
         self.attributes = attributes
@@ -377,7 +378,7 @@ class EnterDBOSWorkflow:
         return False  # Did not handle
 
 
-class EnterDBOSChildWorkflow:
+class EnterDBOSChildWorkflow(AbstractContextManager[DBOSContext, Literal[False]]):
     def __init__(self, attributes: TracedAttributes) -> None:
         self.parent_ctx: Optional[DBOSContext] = None
         self.child_ctx: Optional[DBOSContext] = None

dbos/core.py

@@ -63,6 +63,7 @@ from dbos.system_database import (
     OperationResultInternal,
     WorkflowInputs,
     WorkflowStatusInternal,
+    WorkflowStatusString,
 )
 
 if TYPE_CHECKING:
@@ -108,7 +109,7 @@ class _WorkflowHandlePolling(Generic[R]):
         return self.workflow_id
 
     def get_result(self) -> R:
-        res: R = self.dbos.sys_db.await_workflow_result(self.workflow_id)
+        res: R = self.dbos._sys_db.await_workflow_result(self.workflow_id)
         return res
 
     def get_status(self) -> "WorkflowStatus":
@@ -126,6 +127,7 @@ def _init_workflow(
     class_name: Optional[str],
     config_name: Optional[str],
     temp_wf_type: Optional[str],
+    queue: Optional[str] = None,
 ) -> WorkflowStatusInternal:
     wfid = (
         ctx.workflow_id
@@ -134,7 +136,11 @@ def _init_workflow(
     )
     status: WorkflowStatusInternal = {
         "workflow_uuid": wfid,
-        "status": "PENDING",
+        "status": (
+            WorkflowStatusString.PENDING.value
+            if queue is None
+            else WorkflowStatusString.ENQUEUED.value
+        ),
         "name": wf_name,
         "class_name": class_name,
         "config_name": config_name,
@@ -150,20 +156,25 @@ def _init_workflow(
             json.dumps(ctx.authenticated_roles) if ctx.authenticated_roles else None
         ),
         "assumed_role": ctx.assumed_role,
+        "queue_name": queue,
     }
 
     # If we have a class name, the first arg is the instance and do not serialize
     if class_name is not None:
         inputs = {"args": inputs["args"][1:], "kwargs": inputs["kwargs"]}
 
-    if temp_wf_type != "transaction":
+    if temp_wf_type != "transaction" or queue is not None:
         # Synchronously record the status and inputs for workflows and single-step workflows
         # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
-        dbos.sys_db.update_workflow_status(status, False, ctx.in_recovery)
-        dbos.sys_db.update_workflow_inputs(wfid, utils.serialize(inputs))
+        # TODO: Make this transactional (and with the queue step below)
+        dbos._sys_db.update_workflow_status(status, False, ctx.in_recovery)
+        dbos._sys_db.update_workflow_inputs(wfid, utils.serialize(inputs))
     else:
         # Buffer the inputs for single-transaction workflows, but don't buffer the status
-        dbos.sys_db.buffer_workflow_inputs(wfid, utils.serialize(inputs))
+        dbos._sys_db.buffer_workflow_inputs(wfid, utils.serialize(inputs))
+
+    if queue is not None:
+        dbos._sys_db.enqueue(wfid, queue)
 
     return status
 
@@ -179,7 +190,9 @@ def _execute_workflow(
         output = func(*args, **kwargs)
         status["status"] = "SUCCESS"
         status["output"] = utils.serialize(output)
-        dbos.sys_db.buffer_workflow_status(status)
+        if status["queue_name"] is not None:
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+        dbos._sys_db.buffer_workflow_status(status)
     except DBOSWorkflowConflictIDError:
         # Retrieve the workflow handle and wait for the result.
         # Must use existing_workflow=False because workflow status might not be set yet for single transaction workflows.
@@ -191,7 +204,9 @@ def _execute_workflow(
     except Exception as error:
         status["status"] = "ERROR"
         status["error"] = utils.serialize(error)
-        dbos.sys_db.update_workflow_status(status)
+        if status["queue_name"] is not None:
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+        dbos._sys_db.update_workflow_status(status)
         raise
 
     return output
@@ -221,10 +236,10 @@ def _execute_workflow_wthread(
 
 
 def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]":
-    status = dbos.sys_db.get_workflow_status(workflow_id)
+    status = dbos._sys_db.get_workflow_status(workflow_id)
     if not status:
         raise DBOSRecoveryError(workflow_id, "Workflow status not found")
-    inputs = dbos.sys_db.get_workflow_inputs(workflow_id)
+    inputs = dbos._sys_db.get_workflow_inputs(workflow_id)
     if not inputs:
         raise DBOSRecoveryError(workflow_id, "Workflow inputs not found")
     wf_func = dbos._registry.workflow_info_map.get(status["name"], None)
@@ -249,6 +264,8 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
                 return _start_workflow(
                     dbos,
                     wf_func,
+                    status["queue_name"],
+                    True,
                     dbos._registry.instance_info_map[iname],
                     *inputs["args"],
                     **inputs["kwargs"],
@@ -264,6 +281,8 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
                 return _start_workflow(
                     dbos,
                     wf_func,
+                    status["queue_name"],
+                    True,
                     dbos._registry.class_info_map[class_name],
                     *inputs["args"],
                     **inputs["kwargs"],
@@ -271,7 +290,12 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
         else:
             with SetWorkflowID(workflow_id):
                 return _start_workflow(
-                    dbos, wf_func, *inputs["args"], **inputs["kwargs"]
+                    dbos,
+                    wf_func,
+                    status["queue_name"],
+                    True,
+                    *inputs["args"],
+                    **inputs["kwargs"],
                 )
 
 
@@ -298,34 +322,22 @@ def _workflow_wrapper(dbosreg: "_DBOSRegistry", func: F) -> F:
             "kwargs": kwargs,
         }
         ctx = get_local_dbos_context()
-        if ctx and ctx.is_workflow():
-            with EnterDBOSChildWorkflow(attributes), DBOSAssumeRole(rr):
-                ctx = assert_current_dbos_context()  # Now the child ctx
-                status = _init_workflow(
-                    dbos,
-                    ctx,
-                    inputs=inputs,
-                    wf_name=get_dbos_func_name(func),
-                    class_name=get_dbos_class_name(fi, func, args),
-                    config_name=get_config_name(fi, func, args),
-                    temp_wf_type=get_temp_workflow_type(func),
-                )
-
-                return _execute_workflow(dbos, status, func, *args, **kwargs)
-        else:
-            with EnterDBOSWorkflow(attributes), DBOSAssumeRole(rr):
-                ctx = assert_current_dbos_context()
-                status = _init_workflow(
-                    dbos,
-                    ctx,
-                    inputs=inputs,
-                    wf_name=get_dbos_func_name(func),
-                    class_name=get_dbos_class_name(fi, func, args),
-                    config_name=get_config_name(fi, func, args),
-                    temp_wf_type=get_temp_workflow_type(func),
-                )
+        enterWorkflowCtxMgr = (
+            EnterDBOSChildWorkflow if ctx and ctx.is_workflow() else EnterDBOSWorkflow
+        )
+        with enterWorkflowCtxMgr(attributes), DBOSAssumeRole(rr):
+            ctx = assert_current_dbos_context()  # Now the child ctx
+            status = _init_workflow(
+                dbos,
+                ctx,
+                inputs=inputs,
+                wf_name=get_dbos_func_name(func),
+                class_name=get_dbos_class_name(fi, func, args),
+                config_name=get_config_name(fi, func, args),
+                temp_wf_type=get_temp_workflow_type(func),
+            )
 
-                return _execute_workflow(dbos, status, func, *args, **kwargs)
+            return _execute_workflow(dbos, status, func, *args, **kwargs)
 
     wrapped_func = cast(F, wrapper)
     return wrapped_func
@@ -343,6 +355,8 @@ def _workflow(reg: "_DBOSRegistry") -> Callable[[F], F]:
 def _start_workflow(
     dbos: "DBOS",
     func: "Workflow[P, R]",
+    queue_name: Optional[str],
+    execute_workflow: bool,
     *args: P.args,
     **kwargs: P.kwargs,
 ) -> "WorkflowHandle[R]":
@@ -396,10 +410,14 @@ def _start_workflow(
         class_name=get_dbos_class_name(fi, func, gin_args),
         config_name=get_config_name(fi, func, gin_args),
         temp_wf_type=get_temp_workflow_type(func),
+        queue=queue_name,
     )
 
+    if not execute_workflow:
+        return _WorkflowHandlePolling(new_wf_id, dbos)
+
     if fself is not None:
-        future = dbos.executor.submit(
+        future = dbos._executor.submit(
             cast(Callable[..., R], _execute_workflow_wthread),
             dbos,
             status,
@@ -410,7 +428,7 @@ def _start_workflow(
             **kwargs,
         )
     else:
-        future = dbos.executor.submit(
+        future = dbos._executor.submit(
             cast(Callable[..., R], _execute_workflow_wthread),
             dbos,
             status,
@@ -432,7 +450,7 @@ def _transaction(
                     f"Function {func.__name__} invoked before DBOS initialized"
                 )
             dbos = dbosreg.dbos
-            with dbos.app_db.sessionmaker() as session:
+            with dbos._app_db.sessionmaker() as session:
                 attributes: TracedAttributes = {
                     "name": func.__name__,
                     "operationType": OperationType.TRANSACTION.value,
@@ -510,7 +528,7 @@ def _transaction(
                             # Don't record the error if it was already recorded
                             if not has_recorded_error:
                                 txn_output["error"] = utils.serialize(error)
-                                dbos.app_db.record_transaction_error(txn_output)
+                                dbos._app_db.record_transaction_error(txn_output)
                             raise
             return output
 
@@ -541,6 +559,7 @@ def _transaction(
         set_dbos_func_name(temp_wf, "<temp>." + func.__qualname__)
         set_temp_workflow_type(temp_wf, "transaction")
         dbosreg.register_wf_function(get_dbos_func_name(temp_wf), wrapped_wf)
+        wrapper.__orig_func = temp_wf  # type: ignore
 
         return cast(F, wrapper)
 
@@ -575,7 +594,7 @@ def _step(
                     "output": None,
                     "error": None,
                 }
-                recorded_output = dbos.sys_db.check_operation_execution(
+                recorded_output = dbos._sys_db.check_operation_execution(
                     ctx.workflow_id, ctx.function_id
                 )
                 if recorded_output:
@@ -622,7 +641,7 @@ def _step(
                 step_output["error"] = (
                     utils.serialize(error) if error is not None else None
                 )
-                dbos.sys_db.record_operation_result(step_output)
+                dbos._sys_db.record_operation_result(step_output)
 
                 if error is not None:
                     raise error
@@ -657,6 +676,7 @@ def _step(
         set_dbos_func_name(temp_wf, "<temp>." + func.__qualname__)
         set_temp_workflow_type(temp_wf, "step")
         dbosreg.register_wf_function(get_dbos_func_name(temp_wf), wrapped_wf)
+        wrapper.__orig_func = temp_wf  # type: ignore
 
         return cast(F, wrapper)
 
@@ -671,7 +691,7 @@ def _send(
             "name": "send",
         }
         with EnterDBOSStep(attributes) as ctx:
-            dbos.sys_db.send(
+            dbos._sys_db.send(
                 ctx.workflow_id,
                 ctx.curr_step_function_id,
                 destination_id,
@@ -702,7 +722,7 @@ def _recv(
         with EnterDBOSStep(attributes) as ctx:
             ctx.function_id += 1  # Reserve for the sleep
             timeout_function_id = ctx.function_id
-            return dbos.sys_db.recv(
+            return dbos._sys_db.recv(
                 ctx.workflow_id,
                 ctx.curr_step_function_id,
                 timeout_function_id,
@@ -725,7 +745,7 @@ def _set_event(dbos: "DBOS", key: str, value: Any) -> None:
             "name": "set_event",
         }
         with EnterDBOSStep(attributes) as ctx:
-            dbos.sys_db.set_event(
+            dbos._sys_db.set_event(
                 ctx.workflow_id, ctx.curr_step_function_id, key, value
             )
     else:
@@ -753,7 +773,7 @@ def _get_event(
                 "function_id": ctx.curr_step_function_id,
                 "timeout_function_id": timeout_function_id,
             }
-            return dbos.sys_db.get_event(workflow_id, key, timeout_seconds, caller_ctx)
+            return dbos._sys_db.get_event(workflow_id, key, timeout_seconds, caller_ctx)
     else:
         # Directly call it outside of a workflow
-        return dbos.sys_db.get_event(workflow_id, key, timeout_seconds)
+        return dbos._sys_db.get_event(workflow_id, key, timeout_seconds)

dbos/dbos.py

@@ -20,7 +20,6 @@ from typing import (
     Tuple,
     Type,
     TypeVar,
-    cast,
 )
 
 from opentelemetry.trace import Span
@@ -40,6 +39,7 @@ from dbos.core import (
     _WorkflowHandlePolling,
 )
 from dbos.decorators import classproperty
+from dbos.queue import Queue, queue_thread
 from dbos.recovery import _recover_pending_workflows, _startup_recovery_thread
 from dbos.registrations import (
     DBOSClassInfo,
@@ -138,6 +138,7 @@ class _DBOSRegistry:
         self.workflow_info_map: dict[str, Workflow[..., Any]] = {}
         self.class_info_map: dict[str, type] = {}
         self.instance_info_map: dict[str, object] = {}
+        self.queue_info_map: dict[str, Queue] = {}
         self.pollers: list[_RegisteredJob] = []
         self.dbos: Optional[DBOS] = None
         self.config: Optional[ConfigFile] = None
@@ -163,7 +164,7 @@ class _DBOSRegistry:
     ) -> None:
         if self.dbos and self.dbos._launched:
             self.dbos.stop_events.append(evt)
-            self.dbos.executor.submit(func, *args, **kwargs)
+            self.dbos._executor.submit(func, *args, **kwargs)
         else:
             self.pollers.append((evt, func, args, kwargs))
 
@@ -265,15 +266,15 @@ class DBOS:
         dbos_logger.info("Initializing DBOS")
         self.config: ConfigFile = config
         self._launched: bool = False
-        self._sys_db: Optional[SystemDatabase] = None
-        self._app_db: Optional[ApplicationDatabase] = None
+        self._sys_db_field: Optional[SystemDatabase] = None
+        self._app_db_field: Optional[ApplicationDatabase] = None
         self._registry: _DBOSRegistry = _get_or_create_dbos_registry()
         self._registry.dbos = self
-        self._admin_server: Optional[AdminServer] = None
+        self._admin_server_field: Optional[AdminServer] = None
         self.stop_events: List[threading.Event] = []
         self.fastapi: Optional["FastAPI"] = fastapi
         self.flask: Optional["Flask"] = flask
-        self._executor: Optional[ThreadPoolExecutor] = None
+        self._executor_field: Optional[ThreadPoolExecutor] = None
 
         # If using FastAPI, set up middleware and lifecycle events
         if self.fastapi is not None:
@@ -302,33 +303,33 @@ class DBOS:
             handler.flush()
 
     @property
-    def executor(self) -> ThreadPoolExecutor:
-        if self._executor is None:
+    def _executor(self) -> ThreadPoolExecutor:
+        if self._executor_field is None:
             raise DBOSException("Executor accessed before DBOS was launched")
-        rv: ThreadPoolExecutor = self._executor
+        rv: ThreadPoolExecutor = self._executor_field
         return rv
 
     @property
-    def sys_db(self) -> SystemDatabase:
-        if self._sys_db is None:
+    def _sys_db(self) -> SystemDatabase:
+        if self._sys_db_field is None:
             raise DBOSException("System database accessed before DBOS was launched")
-        rv: SystemDatabase = self._sys_db
+        rv: SystemDatabase = self._sys_db_field
         return rv
 
     @property
-    def app_db(self) -> ApplicationDatabase:
-        if self._app_db is None:
+    def _app_db(self) -> ApplicationDatabase:
+        if self._app_db_field is None:
             raise DBOSException(
                 "Application database accessed before DBOS was launched"
             )
-        rv: ApplicationDatabase = self._app_db
+        rv: ApplicationDatabase = self._app_db_field
         return rv
 
     @property
-    def admin_server(self) -> AdminServer:
-        if self._admin_server is None:
+    def _admin_server(self) -> AdminServer:
+        if self._admin_server_field is None:
             raise DBOSException("Admin server accessed before DBOS was launched")
-        rv: AdminServer = self._admin_server
+        rv: AdminServer = self._admin_server_field
         return rv
 
     @classmethod
@@ -341,25 +342,30 @@ class DBOS:
             dbos_logger.warning(f"DBOS was already launched")
             return
         self._launched = True
-        self._executor = ThreadPoolExecutor(max_workers=64)
-        self._sys_db = SystemDatabase(self.config)
-        self._app_db = ApplicationDatabase(self.config)
-        self._admin_server = AdminServer(dbos=self)
+        self._executor_field = ThreadPoolExecutor(max_workers=64)
+        self._sys_db_field = SystemDatabase(self.config)
+        self._app_db_field = ApplicationDatabase(self.config)
+        self._admin_server_field = AdminServer(dbos=self)
 
         if not os.environ.get("DBOS__VMID"):
-            workflow_ids = self.sys_db.get_pending_workflows("local")
-            self.executor.submit(_startup_recovery_thread, self, workflow_ids)
+            workflow_ids = self._sys_db.get_pending_workflows("local")
+            self._executor.submit(_startup_recovery_thread, self, workflow_ids)
 
         # Listen to notifications
-        self.executor.submit(self.sys_db._notification_listener)
+        self._executor.submit(self._sys_db._notification_listener)
 
         # Start flush workflow buffers thread
-        self.executor.submit(self.sys_db.flush_workflow_buffers)
+        self._executor.submit(self._sys_db.flush_workflow_buffers)
+
+        # Start the queue thread
+        evt = threading.Event()
+        self.stop_events.append(evt)
+        self._executor.submit(queue_thread, evt, self)
 
         # Grab any pollers that were deferred and start them
         for evt, func, args, kwargs in self._registry.pollers:
             self.stop_events.append(evt)
-            self.executor.submit(func, *args, **kwargs)
+            self._executor.submit(func, *args, **kwargs)
         self._registry.pollers = []
 
         dbos_logger.info("DBOS launched")
@@ -374,20 +380,20 @@ class DBOS:
         self._initialized = False
         for event in self.stop_events:
             event.set()
-        if self._sys_db is not None:
-            self._sys_db.destroy()
-            self._sys_db = None
-        if self._app_db is not None:
-            self._app_db.destroy()
-            self._app_db = None
-        if self._admin_server is not None:
-            self._admin_server.stop()
-            self._admin_server = None
+        if self._sys_db_field is not None:
+            self._sys_db_field.destroy()
+            self._sys_db_field = None
+        if self._app_db_field is not None:
+            self._app_db_field.destroy()
+            self._app_db_field = None
+        if self._admin_server_field is not None:
+            self._admin_server_field.stop()
+            self._admin_server_field = None
         # CB - This needs work, some things ought to stop before DBs are tossed out,
         #  on the other hand it hangs to move it
-        if self._executor is not None:
-            self._executor.shutdown(cancel_futures=True)
-            self._executor = None
+        if self._executor_field is not None:
+            self._executor_field.shutdown(cancel_futures=True)
+            self._executor_field = None
 
     @classmethod
     def register_instance(cls, inst: object) -> None:
@@ -488,13 +494,18 @@ class DBOS:
 
     @classmethod
     def kafka_consumer(
-        cls, config: dict[str, Any], topics: list[str]
+        cls,
+        config: dict[str, Any],
+        topics: list[str],
+        in_order: bool = False,
     ) -> Callable[[KafkaConsumerWorkflow], KafkaConsumerWorkflow]:
         """Decorate a function to be used as a Kafka consumer."""
         try:
             from dbos.kafka import kafka_consumer
 
-            return kafka_consumer(_get_or_create_dbos_registry(), config, topics)
+            return kafka_consumer(
+                _get_or_create_dbos_registry(), config, topics, in_order
+            )
         except ModuleNotFoundError as e:
             raise DBOSException(
                 f"{e.name} dependency not found. Please install {e.name} via your package manager."
@@ -508,7 +519,7 @@ class DBOS:
         **kwargs: P.kwargs,
     ) -> WorkflowHandle[R]:
         """Invoke a workflow function in the background, returning a handle to the ongoing execution."""
-        return _start_workflow(_get_dbos_instance(), func, *args, **kwargs)
+        return _start_workflow(_get_dbos_instance(), func, None, True, *args, **kwargs)
 
     @classmethod
     def get_workflow_status(cls, workflow_id: str) -> Optional[WorkflowStatus]:
@@ -516,11 +527,11 @@ class DBOS:
         ctx = get_local_dbos_context()
         if ctx and ctx.is_within_workflow():
             ctx.function_id += 1
-            stat = _get_dbos_instance().sys_db.get_workflow_status_within_wf(
+            stat = _get_dbos_instance()._sys_db.get_workflow_status_within_wf(
                 workflow_id, ctx.workflow_id, ctx.function_id
             )
         else:
-            stat = _get_dbos_instance().sys_db.get_workflow_status(workflow_id)
+            stat = _get_dbos_instance()._sys_db.get_workflow_status(workflow_id)
         if stat is None:
             return None
 
@@ -584,7 +595,7 @@ class DBOS:
         if seconds <= 0:
             return
         with EnterDBOSStep(attributes) as ctx:
-            _get_dbos_instance().sys_db.sleep(
+            _get_dbos_instance()._sys_db.sleep(
                 ctx.workflow_id, ctx.curr_step_function_id, seconds
             )
 

dbos/kafka.py

@@ -1,26 +1,30 @@
 import threading
-import traceback
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Callable, Generator, NoReturn, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, NoReturn
 
 from confluent_kafka import Consumer, KafkaError, KafkaException
-from confluent_kafka import Message as CTypeMessage
+
+from dbos.queue import Queue
 
 if TYPE_CHECKING:
     from dbos.dbos import _DBOSRegistry
 
 from .context import SetWorkflowID
+from .error import DBOSInitializationError
 from .kafka_message import KafkaMessage
 from .logger import dbos_logger
 
 KafkaConsumerWorkflow = Callable[[KafkaMessage], None]
 
+kafka_queue: Queue
+in_order_kafka_queues: dict[str, Queue] = {}
+
 
 def _kafka_consumer_loop(
     func: KafkaConsumerWorkflow,
     config: dict[str, Any],
     topics: list[str],
     stop_event: threading.Event,
+    in_order: bool,
 ) -> None:
 
     def on_error(err: KafkaError) -> NoReturn:
@@ -70,24 +74,35 @@ def _kafka_consumer_loop(
                 with SetWorkflowID(
                     f"kafka-unique-id-{msg.topic}-{msg.partition}-{msg.offset}"
                 ):
-                    try:
-                        func(msg)
-                    except Exception as e:
-                        dbos_logger.error(
-                            f"Exception encountered in Kafka consumer: {traceback.format_exc()}"
-                        )
+                    if in_order:
+                        assert msg.topic is not None
+                        queue = in_order_kafka_queues[msg.topic]
+                        queue.enqueue(func, msg)
+                    else:
+                        kafka_queue.enqueue(func, msg)
 
     finally:
         consumer.close()
 
 
 def kafka_consumer(
-    dbosreg: "_DBOSRegistry", config: dict[str, Any], topics: list[str]
+    dbosreg: "_DBOSRegistry", config: dict[str, Any], topics: list[str], in_order: bool
 ) -> Callable[[KafkaConsumerWorkflow], KafkaConsumerWorkflow]:
     def decorator(func: KafkaConsumerWorkflow) -> KafkaConsumerWorkflow:
+        if in_order:
+            for topic in topics:
+                if topic.startswith("^"):
+                    raise DBOSInitializationError(
+                        f"Error: in-order processing is not supported for regular expression topic selectors ({topic})"
+                    )
+                queue = Queue(f"_dbos_kafka_queue_topic_{topic}", concurrency=1)
+                in_order_kafka_queues[topic] = queue
+        else:
+            global kafka_queue
+            kafka_queue = Queue("_dbos_internal_queue")
         stop_event = threading.Event()
         dbosreg.register_poller(
-            stop_event, _kafka_consumer_loop, func, config, topics, stop_event
+            stop_event, _kafka_consumer_loop, func, config, topics, stop_event, in_order
         )
         return func
 

dbos/migrations/versions/eab0cc1d9a14_job_queue.py

@@ -0,0 +1,55 @@
+"""job_queue
+
+Revision ID: eab0cc1d9a14
+Revises: a3b18ad34abe
+Create Date: 2024-09-13 14:50:00.531294
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "eab0cc1d9a14"
+down_revision: Union[str, None] = "a3b18ad34abe"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "job_queue",
+        sa.Column("workflow_uuid", sa.Text(), nullable=False),
+        sa.Column("queue_name", sa.Text(), nullable=False),
+        sa.Column(
+            "created_at_epoch_ms",
+            sa.BigInteger(),
+            server_default=sa.text(
+                "(EXTRACT(epoch FROM now()) * 1000::numeric)::bigint"
+            ),
+            nullable=False,
+            primary_key=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["workflow_uuid"],
+            ["dbos.workflow_status.workflow_uuid"],
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        schema="dbos",
+    )
+    op.add_column(
+        "workflow_status",
+        sa.Column(
+            "queue_name",
+            sa.Text(),
+        ),
+        schema="dbos",
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("job_queue", schema="dbos")
+    op.drop_column("workflow_status", "queue_name", schema="dbos")

dbos/queue.py

@@ -0,0 +1,36 @@
+import threading
+import time
+from typing import TYPE_CHECKING, Optional
+
+from dbos.core import P, R, _execute_workflow_id, _start_workflow
+from dbos.error import DBOSInitializationError
+
+if TYPE_CHECKING:
+    from dbos.dbos import DBOS, Workflow, WorkflowHandle
+
+
+class Queue:
+    def __init__(self, name: str, concurrency: Optional[int] = None) -> None:
+        self.name = name
+        self.concurrency = concurrency
+        from dbos.dbos import _get_or_create_dbos_registry
+
+        registry = _get_or_create_dbos_registry()
+        registry.queue_info_map[self.name] = self
+
+    def enqueue(
+        self, func: "Workflow[P, R]", *args: P.args, **kwargs: P.kwargs
+    ) -> "WorkflowHandle[R]":
+        from dbos.dbos import _get_dbos_instance
+
+        dbos = _get_dbos_instance()
+        return _start_workflow(dbos, func, self.name, False, *args, **kwargs)
+
+
+def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
+    while not stop_event.is_set():
+        time.sleep(1)
+        for queue_name, queue in dbos._registry.queue_info_map.items():
+            wf_ids = dbos._sys_db.start_queued_workflows(queue_name, queue.concurrency)
+            for id in wf_ids:
+                _execute_workflow_id(dbos, id)

dbos/recovery.py

@@ -41,7 +41,7 @@ def _recover_pending_workflows(
                 f"Skip local recovery because it's running in a VM: {os.environ.get('DBOS__VMID')}"
             )
         dbos.logger.debug(f"Recovering pending workflows for executor: {executor_id}")
-        workflow_ids = dbos.sys_db.get_pending_workflows(executor_id)
+        workflow_ids = dbos._sys_db.get_pending_workflows(executor_id)
         dbos.logger.debug(f"Pending workflows: {workflow_ids}")
 
         for workflowID in workflow_ids:

dbos/scheduler/scheduler.py

@@ -1,17 +1,19 @@
 import threading
-import traceback
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Callable
 
+from dbos.queue import Queue
+
 if TYPE_CHECKING:
     from dbos.dbos import _DBOSRegistry
 
 from ..context import SetWorkflowID
-from ..logger import dbos_logger
 from .croniter import croniter  # type: ignore
 
 ScheduledWorkflow = Callable[[datetime, datetime], None]
 
+scheduler_queue: Queue
+
 
 def scheduler_loop(
     func: ScheduledWorkflow, cron: str, stop_event: threading.Event
@@ -23,19 +25,15 @@ def scheduler_loop(
         if stop_event.wait(timeout=sleepTime.total_seconds()):
             return
         with SetWorkflowID(f"sched-{func.__qualname__}-{nextExecTime.isoformat()}"):
-            try:
-                func(nextExecTime, datetime.now(timezone.utc))
-            except Exception as e:
-                dbos_logger.error(
-                    f"Exception encountered in scheduled workflow: {traceback.format_exc()}"
-                )
-                pass  # Let the thread keep running
+            scheduler_queue.enqueue(func, nextExecTime, datetime.now(timezone.utc))
 
 
 def scheduled(
     dbosreg: "_DBOSRegistry", cron: str
 ) -> Callable[[ScheduledWorkflow], ScheduledWorkflow]:
     def decorator(func: ScheduledWorkflow) -> ScheduledWorkflow:
+        global scheduler_queue
+        scheduler_queue = Queue("_dbos_internal_queue")
         stop_event = threading.Event()
         dbosreg.register_poller(stop_event, scheduler_loop, func, cron, stop_event)
         return func

dbos/schemas/system_database.py

@@ -1,5 +1,6 @@
 from sqlalchemy import (
     BigInteger,
+    Boolean,
     Column,
     ForeignKey,
     Index,
@@ -53,6 +54,7 @@ class SystemSchema:
             nullable=True,
             server_default=text("'0'::bigint"),
         ),
+        Column("queue_name", Text),
         Index("workflow_status_created_at_index", "created_at"),
         Index("workflow_status_executor_id_index", "executor_id"),
     )
@@ -139,3 +141,24 @@ class SystemSchema:
         Column("workflow_fn_name", Text, primary_key=True, nullable=False),
         Column("last_run_time", BigInteger, nullable=False),
     )
+
+    job_queue = Table(
+        "job_queue",
+        metadata_obj,
+        Column(
+            "workflow_uuid",
+            Text,
+            ForeignKey(
+                "workflow_status.workflow_uuid", onupdate="CASCADE", ondelete="CASCADE"
+            ),
+            nullable=False,
+            primary_key=True,
+        ),
+        Column("queue_name", Text, nullable=False),
+        Column(
+            "created_at_epoch_ms",
+            BigInteger,
+            nullable=False,
+            server_default=text("(EXTRACT(epoch FROM now()) * 1000::numeric)::bigint"),
+        ),
+    )

dbos/system_database.py

@@ -33,10 +33,11 @@ class WorkflowStatusString(Enum):
     ERROR = "ERROR"
     RETRIES_EXCEEDED = "RETRIES_EXCEEDED"
     CANCELLED = "CANCELLED"
+    ENQUEUED = "ENQUEUED"
 
 
 WorkflowStatuses = Literal[
-    "PENDING", "SUCCESS", "ERROR", "RETRIES_EXCEEDED", "CANCELLED"
+    "PENDING", "SUCCESS", "ERROR", "RETRIES_EXCEEDED", "CANCELLED", "ENQUEUED"
 ]
 
 
@@ -61,6 +62,7 @@ class WorkflowStatusInternal(TypedDict):
     authenticated_user: Optional[str]
     assumed_role: Optional[str]
     authenticated_roles: Optional[str]  # JSON list of roles.
+    queue_name: Optional[str]
 
 
 class RecordedResult(TypedDict):
@@ -247,6 +249,7 @@ class SystemDatabase:
             authenticated_user=status["authenticated_user"],
             authenticated_roles=status["authenticated_roles"],
             assumed_role=status["assumed_role"],
+            queue_name=status["queue_name"],
         )
         if replace:
             cmd = cmd.on_conflict_do_update(
@@ -320,6 +323,7 @@ class SystemDatabase:
                     SystemSchema.workflow_status.c.authenticated_user,
                     SystemSchema.workflow_status.c.authenticated_roles,
                     SystemSchema.workflow_status.c.assumed_role,
+                    SystemSchema.workflow_status.c.queue_name,
                 ).where(SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid)
             ).fetchone()
             if row is None:
@@ -340,6 +344,7 @@ class SystemDatabase:
                 "authenticated_user": row[6],
                 "authenticated_roles": row[7],
                 "assumed_role": row[8],
+                "queue_name": row[9],
             }
             return status
 
@@ -379,6 +384,7 @@ class SystemDatabase:
                     SystemSchema.workflow_status.c.authenticated_user,
                     SystemSchema.workflow_status.c.authenticated_roles,
                     SystemSchema.workflow_status.c.assumed_role,
+                    SystemSchema.workflow_status.c.queue_name,
                 ).where(SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid)
             ).fetchone()
             if row is None:
@@ -399,6 +405,7 @@ class SystemDatabase:
                 "authenticated_user": row[7],
                 "authenticated_roles": row[8],
                 "assumed_role": row[9],
+                "queue_name": row[10],
             }
             return status
 
@@ -1010,3 +1017,50 @@ class SystemDatabase:
             len(self._workflow_status_buffer) == 0
             and len(self._workflow_inputs_buffer) == 0
         )
+
+    def enqueue(self, workflow_id: str, queue_name: str) -> None:
+        with self.engine.begin() as c:
+            c.execute(
+                pg.insert(SystemSchema.job_queue)
+                .values(
+                    workflow_uuid=workflow_id,
+                    queue_name=queue_name,
+                )
+                .on_conflict_do_nothing()
+            )
+
+    def start_queued_workflows(
+        self, queue_name: str, concurrency: Optional[int]
+    ) -> List[str]:
+        with self.engine.begin() as c:
+            query = sa.select(SystemSchema.job_queue.c.workflow_uuid).where(
+                SystemSchema.job_queue.c.queue_name == queue_name
+            )
+            if concurrency is not None:
+                query = query.order_by(
+                    SystemSchema.job_queue.c.created_at_epoch_ms.asc()
+                ).limit(concurrency)
+            rows = c.execute(query).fetchall()
+            dequeued_ids: List[str] = [row[0] for row in rows]
+            ret_ids = []
+            for id in dequeued_ids:
+                result = c.execute(
+                    SystemSchema.workflow_status.update()
+                    .where(SystemSchema.workflow_status.c.workflow_uuid == id)
+                    .where(
+                        SystemSchema.workflow_status.c.status
+                        == WorkflowStatusString.ENQUEUED.value
+                    )
+                    .values(status=WorkflowStatusString.PENDING.value)
+                )
+                if result.rowcount > 0:
+                    ret_ids.append(id)
+            return ret_ids
+
+    def remove_from_queue(self, workflow_id: str) -> None:
+        with self.engine.begin() as c:
+            c.execute(
+                sa.delete(SystemSchema.job_queue).where(
+                    SystemSchema.job_queue.c.workflow_uuid == workflow_id
+                )
+            )

{dbos-0.7.0a5.dist-info → dbos-0.7.0a8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.7.0a5
+Version: 0.7.0a8
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.7.0a5.dist-info → dbos-0.7.0a8.dist-info}/RECORD

@@ -1,38 +1,40 @@
-dbos-0.7.0a5.dist-info/METADATA,sha256=GpkOCHTB3NQUEMNI2bezOCYqBYMnn1FUzSemILcJBF0,5010
-dbos-0.7.0a5.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
-dbos-0.7.0a5.dist-info/entry_points.txt,sha256=3PmOPbM4FYxEmggRRdJw0oAsiBzKR8U0yx7bmwUmMOM,39
-dbos-0.7.0a5.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
-dbos/__init__.py,sha256=heuB3bqRXlVdfea9sKHBIVKSqFP6UuwhQecQfV4hyas,642
+dbos-0.7.0a8.dist-info/METADATA,sha256=WCNkAofLXq_TaFlZzWzLRCBhYqN1JeaOE7zydAtR5TU,5010
+dbos-0.7.0a8.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
+dbos-0.7.0a8.dist-info/entry_points.txt,sha256=3PmOPbM4FYxEmggRRdJw0oAsiBzKR8U0yx7bmwUmMOM,39
+dbos-0.7.0a8.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
+dbos/__init__.py,sha256=jjlBFzSAzO2e-LD5IKJw7bRqZjrxpF5Sn-_JUJJptHU,680
 dbos/admin_sever.py,sha256=Qg5T3YRrbPW05PR_99yAaxgo1ugQrAp_uTeTqSfjm_k,3397
 dbos/application_database.py,sha256=knFK8We8y6WrIpnFCKvFq5hvSuFQqUuJqOqDpSVMCPI,5521
 dbos/cli.py,sha256=z5dXbbnGWzSC3E1rfS8Lp1_OIImzcDKM7jP-iu_Q4aI,8602
-dbos/context.py,sha256=NVMGyvAa2RIiBVspvDz-8MBk_BQyGyYdPdorgO-GSng,16407
-dbos/core.py,sha256=av2O5BsUH5UqQ6ICuH63RgAJFCjBkSHPKchbX_KtKcc,28560
+dbos/context.py,sha256=1Xp0i-ZP72Vj6eMdHuyfXi9RHnoT2w4MZ2Kx1CrKnQ8,16559
+dbos/core.py,sha256=LnQRHLjfpdlzlRkElduay8nObNMyRNC8SJSZEzqei7o,29161
 dbos/dbos-config.schema.json,sha256=azpfmoDZg7WfSy3kvIsk9iEiKB_-VZt03VEOoXJAkqE,5331
-dbos/dbos.py,sha256=SnY5CSHLnrjx128wGaaB-4zC4VU2basCMLZq9hXrAIg,28768
+dbos/dbos.py,sha256=RtDcvKe4sm1TlnCGU4cyex-UI7hMMlhgzmOl1NuRLo4,29294
 dbos/dbos_config.py,sha256=NJVze2GkKgYUmcPP31Unb-QpsA0TzImEeQGJgVq6W6k,5352
 dbos/decorators.py,sha256=lbPefsLK6Cya4cb7TrOcLglOpGT3pc6qjZdsQKlfZLg,629
 dbos/error.py,sha256=DDhB0VHmoZE_CP51ICdFMZSL2gmVS3Dm0aPNWncci94,3876
 dbos/fastapi.py,sha256=gx9hlpxYOiwbuhSlbY9bn5C-F_FsCbrJvkX9ZAvDG6U,3418
 dbos/flask.py,sha256=azr4geMEGuuTBCyxIZmgDmmP-6s_pTIF-lGyp9Q4IB8,2430
-dbos/kafka.py,sha256=FtngQHBu2TKfyDF7GFsKJAawFQJiOFxgKEUlNNxrdrw,3055
+dbos/kafka.py,sha256=LH3hbNapnkjLcuXNUtdGU0398JafWb-t0GwUl3LOzkc,3645
 dbos/kafka_message.py,sha256=NYvOXNG3Qn7bghn1pv3fg4Pbs86ILZGcK4IB-MLUNu0,409
 dbos/logger.py,sha256=D-aFSZUCHBP34J1IZ5YNkTrJW-rDiH3py_v9jLU4Yrk,3565
 dbos/migrations/env.py,sha256=38SIGVbmn_VV2x2u1aHLcPOoWgZ84eCymf3g_NljmbU,1626
 dbos/migrations/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvDgl93dMj8,635
 dbos/migrations/versions/5c361fc04708_added_system_tables.py,sha256=QMgFMb0aLgC25YicsvPSr6AHRCA6Zd66hyaRUhwKzrQ,6404
 dbos/migrations/versions/a3b18ad34abe_added_triggers.py,sha256=Rv0ZsZYZ_WdgGEULYsPfnp4YzaO5L198gDTgYY39AVA,2022
+dbos/migrations/versions/eab0cc1d9a14_job_queue.py,sha256=_9-FCW-zOpCQfblTS_yRLtFiUaWlC1tM4BoKBTDeH9k,1395
 dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
-dbos/recovery.py,sha256=xfwQFWNuD8DXg5HD5_-3tG7Neo9j-x1lrqiwtn5FSh8,2015
+dbos/queue.py,sha256=5unMPX1ThoVWfXOR2IUSeIjgATgcX_YmZS3Fz1Uoc7o,1228
+dbos/recovery.py,sha256=zqtO_ExGoIErLMVnbneU3VeHLVWvhV4jnfqssAVlQQk,2016
 dbos/registrations.py,sha256=gMI-u05tv5bpvyddQGtoUgCsqARx51aOY7p0JXPafQo,6539
 dbos/request.py,sha256=-FIwtknayvRl6OjvqO4V2GySVzSdP1Ft3cc9ZBS-PLY,928
 dbos/roles.py,sha256=7Lh7uwUq1dpa6TXCOHre4mPTd5qmXzK_QPkvYR52DXg,2285
 dbos/scheduler/croniter.py,sha256=hbhgfsHBqclUS8VeLnJ9PSE9Z54z6mi4nnrr1aUXn0k,47561
-dbos/scheduler/scheduler.py,sha256=ks5zdOtt524TIuquh4u1ZwmDNWzguOpNLeFZ8bWTeUA,1498
+dbos/scheduler/scheduler.py,sha256=Sz4EIpAtur7so2YajTic64GrTpa4qPw8QxXn0M34v80,1360
 dbos/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/schemas/application_database.py,sha256=q_Wr2XbiZNBYFkOtu7uKavo1T_cSOBblxKGHThYGGsY,962
-dbos/schemas/system_database.py,sha256=5V3vqnEzry0Hn7ZbVS9Gs_dJKia8uX8p7mGC82Ru8rk,4303
-dbos/system_database.py,sha256=Jf2O1O7mM0_9xZSSBqRKpi4uPtjsNkq9FlilzR5vQis,39796
+dbos/schemas/system_database.py,sha256=ed4c1UntsD-cqXD0ekM4jvcYYEViavDh_G6c0pVDe7k,4938
+dbos/system_database.py,sha256=y59uFRWsNlp_tBSyZpq4geXDAC1Vii6074iFQZkgltY,41974
 dbos/templates/hello/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
 dbos/templates/hello/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/templates/hello/__package/main.py,sha256=eI0SS9Nwj-fldtiuSzIlIG6dC91GXXwdRsoHxv6S_WI,2719
@@ -46,4 +48,4 @@ dbos/templates/hello/start_postgres_docker.py,sha256=lQVLlYO5YkhGPEgPqwGc7Y8uDKs
 dbos/tracer.py,sha256=GaXDhdKKF_IQp5SAMipGXiDVwteRKjNbrXyYCH1mor0,2520
 dbos/utils.py,sha256=hWj9iWDrby2cVEhb0pG-IdnrxLqP64NhkaWUXiLc8bA,402
 version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
-dbos-0.7.0a5.dist-info/RECORD,,
+dbos-0.7.0a8.dist-info/RECORD,,