dbos 0.7.0a5__tar.gz → 0.7.0a8__tar.gz

{dbos-0.7.0a5 → dbos-0.7.0a8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.7.0a5
+Version: 0.7.0a8
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.7.0a5 → dbos-0.7.0a8}/dbos/__init__.py

@@ -3,6 +3,7 @@ from .context import DBOSContextEnsure, SetWorkflowID
 from .dbos import DBOS, DBOSConfiguredInstance, WorkflowHandle, WorkflowStatus
 from .dbos_config import ConfigFile, get_dbos_database_url, load_config
 from .kafka_message import KafkaMessage
+from .queue import Queue
 from .system_database import GetWorkflowsInput, WorkflowStatusString
 
 __all__ = [
@@ -19,4 +20,5 @@ __all__ = [
     "load_config",
     "get_dbos_database_url",
     "error",
+    "Queue",
 ]

{dbos-0.7.0a5 → dbos-0.7.0a8}/dbos/context.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import json
 import os
 import uuid
+from contextlib import AbstractContextManager
 from contextvars import ContextVar
 from enum import Enum
 from types import TracebackType
@@ -344,7 +345,7 @@ class SetWorkflowRecovery:
         return False  # Did not handle
 
 
-class EnterDBOSWorkflow:
+class EnterDBOSWorkflow(AbstractContextManager[DBOSContext, Literal[False]]):
     def __init__(self, attributes: TracedAttributes) -> None:
         self.created_ctx = False
         self.attributes = attributes
@@ -377,7 +378,7 @@ class EnterDBOSWorkflow:
         return False  # Did not handle
 
 
-class EnterDBOSChildWorkflow:
+class EnterDBOSChildWorkflow(AbstractContextManager[DBOSContext, Literal[False]]):
     def __init__(self, attributes: TracedAttributes) -> None:
         self.parent_ctx: Optional[DBOSContext] = None
         self.child_ctx: Optional[DBOSContext] = None

{dbos-0.7.0a5 → dbos-0.7.0a8}/dbos/core.py

@@ -63,6 +63,7 @@ from dbos.system_database import (
     OperationResultInternal,
     WorkflowInputs,
     WorkflowStatusInternal,
+    WorkflowStatusString,
 )
 
 if TYPE_CHECKING:
@@ -108,7 +109,7 @@ class _WorkflowHandlePolling(Generic[R]):
         return self.workflow_id
 
     def get_result(self) -> R:
-        res: R = self.dbos.sys_db.await_workflow_result(self.workflow_id)
+        res: R = self.dbos._sys_db.await_workflow_result(self.workflow_id)
         return res
 
     def get_status(self) -> "WorkflowStatus":
@@ -126,6 +127,7 @@ def _init_workflow(
     class_name: Optional[str],
     config_name: Optional[str],
     temp_wf_type: Optional[str],
+    queue: Optional[str] = None,
 ) -> WorkflowStatusInternal:
     wfid = (
         ctx.workflow_id
@@ -134,7 +136,11 @@ def _init_workflow(
     )
     status: WorkflowStatusInternal = {
         "workflow_uuid": wfid,
-        "status": "PENDING",
+        "status": (
+            WorkflowStatusString.PENDING.value
+            if queue is None
+            else WorkflowStatusString.ENQUEUED.value
+        ),
         "name": wf_name,
         "class_name": class_name,
         "config_name": config_name,
@@ -150,20 +156,25 @@ def _init_workflow(
             json.dumps(ctx.authenticated_roles) if ctx.authenticated_roles else None
         ),
         "assumed_role": ctx.assumed_role,
+        "queue_name": queue,
     }
 
     # If we have a class name, the first arg is the instance and do not serialize
     if class_name is not None:
         inputs = {"args": inputs["args"][1:], "kwargs": inputs["kwargs"]}
 
-    if temp_wf_type != "transaction":
+    if temp_wf_type != "transaction" or queue is not None:
         # Synchronously record the status and inputs for workflows and single-step workflows
         # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
-        dbos.sys_db.update_workflow_status(status, False, ctx.in_recovery)
-        dbos.sys_db.update_workflow_inputs(wfid, utils.serialize(inputs))
+        # TODO: Make this transactional (and with the queue step below)
+        dbos._sys_db.update_workflow_status(status, False, ctx.in_recovery)
+        dbos._sys_db.update_workflow_inputs(wfid, utils.serialize(inputs))
     else:
         # Buffer the inputs for single-transaction workflows, but don't buffer the status
-        dbos.sys_db.buffer_workflow_inputs(wfid, utils.serialize(inputs))
+        dbos._sys_db.buffer_workflow_inputs(wfid, utils.serialize(inputs))
+
+    if queue is not None:
+        dbos._sys_db.enqueue(wfid, queue)
 
     return status
 
@@ -179,7 +190,9 @@ def _execute_workflow(
         output = func(*args, **kwargs)
         status["status"] = "SUCCESS"
         status["output"] = utils.serialize(output)
-        dbos.sys_db.buffer_workflow_status(status)
+        if status["queue_name"] is not None:
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+        dbos._sys_db.buffer_workflow_status(status)
     except DBOSWorkflowConflictIDError:
         # Retrieve the workflow handle and wait for the result.
         # Must use existing_workflow=False because workflow status might not be set yet for single transaction workflows.
@@ -191,7 +204,9 @@ def _execute_workflow(
     except Exception as error:
         status["status"] = "ERROR"
         status["error"] = utils.serialize(error)
-        dbos.sys_db.update_workflow_status(status)
+        if status["queue_name"] is not None:
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+        dbos._sys_db.update_workflow_status(status)
         raise
 
     return output
@@ -221,10 +236,10 @@ def _execute_workflow_wthread(
 
 
 def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]":
-    status = dbos.sys_db.get_workflow_status(workflow_id)
+    status = dbos._sys_db.get_workflow_status(workflow_id)
     if not status:
         raise DBOSRecoveryError(workflow_id, "Workflow status not found")
-    inputs = dbos.sys_db.get_workflow_inputs(workflow_id)
+    inputs = dbos._sys_db.get_workflow_inputs(workflow_id)
     if not inputs:
         raise DBOSRecoveryError(workflow_id, "Workflow inputs not found")
     wf_func = dbos._registry.workflow_info_map.get(status["name"], None)
@@ -249,6 +264,8 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
                 return _start_workflow(
                     dbos,
                     wf_func,
+                    status["queue_name"],
+                    True,
                     dbos._registry.instance_info_map[iname],
                     *inputs["args"],
                     **inputs["kwargs"],
@@ -264,6 +281,8 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
                 return _start_workflow(
                     dbos,
                     wf_func,
+                    status["queue_name"],
+                    True,
                     dbos._registry.class_info_map[class_name],
                     *inputs["args"],
                     **inputs["kwargs"],
@@ -271,7 +290,12 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
         else:
             with SetWorkflowID(workflow_id):
                 return _start_workflow(
-                    dbos, wf_func, *inputs["args"], **inputs["kwargs"]
+                    dbos,
+                    wf_func,
+                    status["queue_name"],
+                    True,
+                    *inputs["args"],
+                    **inputs["kwargs"],
                 )
 
 
@@ -298,34 +322,22 @@ def _workflow_wrapper(dbosreg: "_DBOSRegistry", func: F) -> F:
             "kwargs": kwargs,
         }
         ctx = get_local_dbos_context()
-        if ctx and ctx.is_workflow():
-            with EnterDBOSChildWorkflow(attributes), DBOSAssumeRole(rr):
-                ctx = assert_current_dbos_context()  # Now the child ctx
-                status = _init_workflow(
-                    dbos,
-                    ctx,
-                    inputs=inputs,
-                    wf_name=get_dbos_func_name(func),
-                    class_name=get_dbos_class_name(fi, func, args),
-                    config_name=get_config_name(fi, func, args),
-                    temp_wf_type=get_temp_workflow_type(func),
-                )
-
-                return _execute_workflow(dbos, status, func, *args, **kwargs)
-        else:
-            with EnterDBOSWorkflow(attributes), DBOSAssumeRole(rr):
-                ctx = assert_current_dbos_context()
-                status = _init_workflow(
-                    dbos,
-                    ctx,
-                    inputs=inputs,
-                    wf_name=get_dbos_func_name(func),
-                    class_name=get_dbos_class_name(fi, func, args),
-                    config_name=get_config_name(fi, func, args),
-                    temp_wf_type=get_temp_workflow_type(func),
-                )
+        enterWorkflowCtxMgr = (
+            EnterDBOSChildWorkflow if ctx and ctx.is_workflow() else EnterDBOSWorkflow
+        )
+        with enterWorkflowCtxMgr(attributes), DBOSAssumeRole(rr):
+            ctx = assert_current_dbos_context()  # Now the child ctx
+            status = _init_workflow(
+                dbos,
+                ctx,
+                inputs=inputs,
+                wf_name=get_dbos_func_name(func),
+                class_name=get_dbos_class_name(fi, func, args),
+                config_name=get_config_name(fi, func, args),
+                temp_wf_type=get_temp_workflow_type(func),
+            )
 
-                return _execute_workflow(dbos, status, func, *args, **kwargs)
+            return _execute_workflow(dbos, status, func, *args, **kwargs)
 
     wrapped_func = cast(F, wrapper)
     return wrapped_func
@@ -343,6 +355,8 @@ def _workflow(reg: "_DBOSRegistry") -> Callable[[F], F]:
 def _start_workflow(
     dbos: "DBOS",
     func: "Workflow[P, R]",
+    queue_name: Optional[str],
+    execute_workflow: bool,
     *args: P.args,
     **kwargs: P.kwargs,
 ) -> "WorkflowHandle[R]":
@@ -396,10 +410,14 @@ def _start_workflow(
         class_name=get_dbos_class_name(fi, func, gin_args),
         config_name=get_config_name(fi, func, gin_args),
         temp_wf_type=get_temp_workflow_type(func),
+        queue=queue_name,
     )
 
+    if not execute_workflow:
+        return _WorkflowHandlePolling(new_wf_id, dbos)
+
     if fself is not None:
-        future = dbos.executor.submit(
+        future = dbos._executor.submit(
             cast(Callable[..., R], _execute_workflow_wthread),
             dbos,
             status,
@@ -410,7 +428,7 @@ def _start_workflow(
             **kwargs,
         )
     else:
-        future = dbos.executor.submit(
+        future = dbos._executor.submit(
             cast(Callable[..., R], _execute_workflow_wthread),
             dbos,
             status,
@@ -432,7 +450,7 @@ def _transaction(
                     f"Function {func.__name__} invoked before DBOS initialized"
                 )
             dbos = dbosreg.dbos
-            with dbos.app_db.sessionmaker() as session:
+            with dbos._app_db.sessionmaker() as session:
                 attributes: TracedAttributes = {
                     "name": func.__name__,
                     "operationType": OperationType.TRANSACTION.value,
@@ -510,7 +528,7 @@ def _transaction(
                             # Don't record the error if it was already recorded
                             if not has_recorded_error:
                                 txn_output["error"] = utils.serialize(error)
-                                dbos.app_db.record_transaction_error(txn_output)
+                                dbos._app_db.record_transaction_error(txn_output)
                             raise
             return output
 
@@ -541,6 +559,7 @@ def _transaction(
         set_dbos_func_name(temp_wf, "<temp>." + func.__qualname__)
         set_temp_workflow_type(temp_wf, "transaction")
         dbosreg.register_wf_function(get_dbos_func_name(temp_wf), wrapped_wf)
+        wrapper.__orig_func = temp_wf  # type: ignore
 
         return cast(F, wrapper)
 
@@ -575,7 +594,7 @@ def _step(
                     "output": None,
                     "error": None,
                 }
-                recorded_output = dbos.sys_db.check_operation_execution(
+                recorded_output = dbos._sys_db.check_operation_execution(
                     ctx.workflow_id, ctx.function_id
                 )
                 if recorded_output:
@@ -622,7 +641,7 @@ def _step(
                 step_output["error"] = (
                     utils.serialize(error) if error is not None else None
                 )
-                dbos.sys_db.record_operation_result(step_output)
+                dbos._sys_db.record_operation_result(step_output)
 
                 if error is not None:
                     raise error
@@ -657,6 +676,7 @@ def _step(
         set_dbos_func_name(temp_wf, "<temp>." + func.__qualname__)
         set_temp_workflow_type(temp_wf, "step")
         dbosreg.register_wf_function(get_dbos_func_name(temp_wf), wrapped_wf)
+        wrapper.__orig_func = temp_wf  # type: ignore
 
         return cast(F, wrapper)
 
@@ -671,7 +691,7 @@ def _send(
             "name": "send",
         }
         with EnterDBOSStep(attributes) as ctx:
-            dbos.sys_db.send(
+            dbos._sys_db.send(
                 ctx.workflow_id,
                 ctx.curr_step_function_id,
                 destination_id,
@@ -702,7 +722,7 @@ def _recv(
         with EnterDBOSStep(attributes) as ctx:
             ctx.function_id += 1  # Reserve for the sleep
             timeout_function_id = ctx.function_id
-            return dbos.sys_db.recv(
+            return dbos._sys_db.recv(
                 ctx.workflow_id,
                 ctx.curr_step_function_id,
                 timeout_function_id,
@@ -725,7 +745,7 @@ def _set_event(dbos: "DBOS", key: str, value: Any) -> None:
             "name": "set_event",
         }
         with EnterDBOSStep(attributes) as ctx:
-            dbos.sys_db.set_event(
+            dbos._sys_db.set_event(
                 ctx.workflow_id, ctx.curr_step_function_id, key, value
             )
     else:
@@ -753,7 +773,7 @@ def _get_event(
                 "function_id": ctx.curr_step_function_id,
                 "timeout_function_id": timeout_function_id,
             }
-            return dbos.sys_db.get_event(workflow_id, key, timeout_seconds, caller_ctx)
+            return dbos._sys_db.get_event(workflow_id, key, timeout_seconds, caller_ctx)
     else:
         # Directly call it outside of a workflow
-        return dbos.sys_db.get_event(workflow_id, key, timeout_seconds)
+        return dbos._sys_db.get_event(workflow_id, key, timeout_seconds)

{dbos-0.7.0a5 → dbos-0.7.0a8}/dbos/dbos.py

@@ -20,7 +20,6 @@ from typing import (
     Tuple,
     Type,
     TypeVar,
-    cast,
 )
 
 from opentelemetry.trace import Span
@@ -40,6 +39,7 @@ from dbos.core import (
     _WorkflowHandlePolling,
 )
 from dbos.decorators import classproperty
+from dbos.queue import Queue, queue_thread
 from dbos.recovery import _recover_pending_workflows, _startup_recovery_thread
 from dbos.registrations import (
     DBOSClassInfo,
@@ -138,6 +138,7 @@ class _DBOSRegistry:
         self.workflow_info_map: dict[str, Workflow[..., Any]] = {}
         self.class_info_map: dict[str, type] = {}
         self.instance_info_map: dict[str, object] = {}
+        self.queue_info_map: dict[str, Queue] = {}
         self.pollers: list[_RegisteredJob] = []
         self.dbos: Optional[DBOS] = None
         self.config: Optional[ConfigFile] = None
@@ -163,7 +164,7 @@ class _DBOSRegistry:
     ) -> None:
         if self.dbos and self.dbos._launched:
             self.dbos.stop_events.append(evt)
-            self.dbos.executor.submit(func, *args, **kwargs)
+            self.dbos._executor.submit(func, *args, **kwargs)
         else:
             self.pollers.append((evt, func, args, kwargs))
 
@@ -265,15 +266,15 @@ class DBOS:
         dbos_logger.info("Initializing DBOS")
         self.config: ConfigFile = config
         self._launched: bool = False
-        self._sys_db: Optional[SystemDatabase] = None
-        self._app_db: Optional[ApplicationDatabase] = None
+        self._sys_db_field: Optional[SystemDatabase] = None
+        self._app_db_field: Optional[ApplicationDatabase] = None
         self._registry: _DBOSRegistry = _get_or_create_dbos_registry()
         self._registry.dbos = self
-        self._admin_server: Optional[AdminServer] = None
+        self._admin_server_field: Optional[AdminServer] = None
         self.stop_events: List[threading.Event] = []
         self.fastapi: Optional["FastAPI"] = fastapi
         self.flask: Optional["Flask"] = flask
-        self._executor: Optional[ThreadPoolExecutor] = None
+        self._executor_field: Optional[ThreadPoolExecutor] = None
 
         # If using FastAPI, set up middleware and lifecycle events
         if self.fastapi is not None:
@@ -302,33 +303,33 @@ class DBOS:
             handler.flush()
 
     @property
-    def executor(self) -> ThreadPoolExecutor:
-        if self._executor is None:
+    def _executor(self) -> ThreadPoolExecutor:
+        if self._executor_field is None:
             raise DBOSException("Executor accessed before DBOS was launched")
-        rv: ThreadPoolExecutor = self._executor
+        rv: ThreadPoolExecutor = self._executor_field
         return rv
 
     @property
-    def sys_db(self) -> SystemDatabase:
-        if self._sys_db is None:
+    def _sys_db(self) -> SystemDatabase:
+        if self._sys_db_field is None:
             raise DBOSException("System database accessed before DBOS was launched")
-        rv: SystemDatabase = self._sys_db
+        rv: SystemDatabase = self._sys_db_field
         return rv
 
     @property
-    def app_db(self) -> ApplicationDatabase:
-        if self._app_db is None:
+    def _app_db(self) -> ApplicationDatabase:
+        if self._app_db_field is None:
             raise DBOSException(
                 "Application database accessed before DBOS was launched"
             )
-        rv: ApplicationDatabase = self._app_db
+        rv: ApplicationDatabase = self._app_db_field
         return rv
 
     @property
-    def admin_server(self) -> AdminServer:
-        if self._admin_server is None:
+    def _admin_server(self) -> AdminServer:
+        if self._admin_server_field is None:
             raise DBOSException("Admin server accessed before DBOS was launched")
-        rv: AdminServer = self._admin_server
+        rv: AdminServer = self._admin_server_field
         return rv
 
     @classmethod
@@ -341,25 +342,30 @@ class DBOS:
             dbos_logger.warning(f"DBOS was already launched")
             return
         self._launched = True
-        self._executor = ThreadPoolExecutor(max_workers=64)
-        self._sys_db = SystemDatabase(self.config)
-        self._app_db = ApplicationDatabase(self.config)
-        self._admin_server = AdminServer(dbos=self)
+        self._executor_field = ThreadPoolExecutor(max_workers=64)
+        self._sys_db_field = SystemDatabase(self.config)
+        self._app_db_field = ApplicationDatabase(self.config)
+        self._admin_server_field = AdminServer(dbos=self)
 
         if not os.environ.get("DBOS__VMID"):
-            workflow_ids = self.sys_db.get_pending_workflows("local")
-            self.executor.submit(_startup_recovery_thread, self, workflow_ids)
+            workflow_ids = self._sys_db.get_pending_workflows("local")
+            self._executor.submit(_startup_recovery_thread, self, workflow_ids)
 
         # Listen to notifications
-        self.executor.submit(self.sys_db._notification_listener)
+        self._executor.submit(self._sys_db._notification_listener)
 
         # Start flush workflow buffers thread
-        self.executor.submit(self.sys_db.flush_workflow_buffers)
+        self._executor.submit(self._sys_db.flush_workflow_buffers)
+
+        # Start the queue thread
+        evt = threading.Event()
+        self.stop_events.append(evt)
+        self._executor.submit(queue_thread, evt, self)
 
         # Grab any pollers that were deferred and start them
         for evt, func, args, kwargs in self._registry.pollers:
             self.stop_events.append(evt)
-            self.executor.submit(func, *args, **kwargs)
+            self._executor.submit(func, *args, **kwargs)
         self._registry.pollers = []
 
         dbos_logger.info("DBOS launched")
@@ -374,20 +380,20 @@ class DBOS:
         self._initialized = False
         for event in self.stop_events:
             event.set()
-        if self._sys_db is not None:
-            self._sys_db.destroy()
-            self._sys_db = None
-        if self._app_db is not None:
-            self._app_db.destroy()
-            self._app_db = None
-        if self._admin_server is not None:
-            self._admin_server.stop()
-            self._admin_server = None
+        if self._sys_db_field is not None:
+            self._sys_db_field.destroy()
+            self._sys_db_field = None
+        if self._app_db_field is not None:
+            self._app_db_field.destroy()
+            self._app_db_field = None
+        if self._admin_server_field is not None:
+            self._admin_server_field.stop()
+            self._admin_server_field = None
         # CB - This needs work, some things ought to stop before DBs are tossed out,
         #  on the other hand it hangs to move it
-        if self._executor is not None:
-            self._executor.shutdown(cancel_futures=True)
-            self._executor = None
+        if self._executor_field is not None:
+            self._executor_field.shutdown(cancel_futures=True)
+            self._executor_field = None
 
     @classmethod
     def register_instance(cls, inst: object) -> None:
@@ -488,13 +494,18 @@ class DBOS:
 
     @classmethod
     def kafka_consumer(
-        cls, config: dict[str, Any], topics: list[str]
+        cls,
+        config: dict[str, Any],
+        topics: list[str],
+        in_order: bool = False,
     ) -> Callable[[KafkaConsumerWorkflow], KafkaConsumerWorkflow]:
         """Decorate a function to be used as a Kafka consumer."""
         try:
             from dbos.kafka import kafka_consumer
 
-            return kafka_consumer(_get_or_create_dbos_registry(), config, topics)
+            return kafka_consumer(
+                _get_or_create_dbos_registry(), config, topics, in_order
+            )
         except ModuleNotFoundError as e:
             raise DBOSException(
                 f"{e.name} dependency not found. Please install {e.name} via your package manager."
@@ -508,7 +519,7 @@ class DBOS:
         **kwargs: P.kwargs,
     ) -> WorkflowHandle[R]:
         """Invoke a workflow function in the background, returning a handle to the ongoing execution."""
-        return _start_workflow(_get_dbos_instance(), func, *args, **kwargs)
+        return _start_workflow(_get_dbos_instance(), func, None, True, *args, **kwargs)
 
     @classmethod
     def get_workflow_status(cls, workflow_id: str) -> Optional[WorkflowStatus]:
@@ -516,11 +527,11 @@ class DBOS:
         ctx = get_local_dbos_context()
         if ctx and ctx.is_within_workflow():
             ctx.function_id += 1
-            stat = _get_dbos_instance().sys_db.get_workflow_status_within_wf(
+            stat = _get_dbos_instance()._sys_db.get_workflow_status_within_wf(
                 workflow_id, ctx.workflow_id, ctx.function_id
             )
         else:
-            stat = _get_dbos_instance().sys_db.get_workflow_status(workflow_id)
+            stat = _get_dbos_instance()._sys_db.get_workflow_status(workflow_id)
         if stat is None:
             return None
 
@@ -584,7 +595,7 @@ class DBOS:
         if seconds <= 0:
             return
         with EnterDBOSStep(attributes) as ctx:
-            _get_dbos_instance().sys_db.sleep(
+            _get_dbos_instance()._sys_db.sleep(
                 ctx.workflow_id, ctx.curr_step_function_id, seconds
             )
 

{dbos-0.7.0a5 → dbos-0.7.0a8}/dbos/kafka.py

@@ -1,26 +1,30 @@
 import threading
-import traceback
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Callable, Generator, NoReturn, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, NoReturn
 
 from confluent_kafka import Consumer, KafkaError, KafkaException
-from confluent_kafka import Message as CTypeMessage
+
+from dbos.queue import Queue
 
 if TYPE_CHECKING:
     from dbos.dbos import _DBOSRegistry
 
 from .context import SetWorkflowID
+from .error import DBOSInitializationError
 from .kafka_message import KafkaMessage
 from .logger import dbos_logger
 
 KafkaConsumerWorkflow = Callable[[KafkaMessage], None]
 
+kafka_queue: Queue
+in_order_kafka_queues: dict[str, Queue] = {}
+
 
 def _kafka_consumer_loop(
     func: KafkaConsumerWorkflow,
     config: dict[str, Any],
     topics: list[str],
     stop_event: threading.Event,
+    in_order: bool,
 ) -> None:
 
     def on_error(err: KafkaError) -> NoReturn:
@@ -70,24 +74,35 @@ def _kafka_consumer_loop(
                 with SetWorkflowID(
                     f"kafka-unique-id-{msg.topic}-{msg.partition}-{msg.offset}"
                 ):
-                    try:
-                        func(msg)
-                    except Exception as e:
-                        dbos_logger.error(
-                            f"Exception encountered in Kafka consumer: {traceback.format_exc()}"
-                        )
+                    if in_order:
+                        assert msg.topic is not None
+                        queue = in_order_kafka_queues[msg.topic]
+                        queue.enqueue(func, msg)
+                    else:
+                        kafka_queue.enqueue(func, msg)
 
     finally:
         consumer.close()
 
 
 def kafka_consumer(
-    dbosreg: "_DBOSRegistry", config: dict[str, Any], topics: list[str]
+    dbosreg: "_DBOSRegistry", config: dict[str, Any], topics: list[str], in_order: bool
 ) -> Callable[[KafkaConsumerWorkflow], KafkaConsumerWorkflow]:
     def decorator(func: KafkaConsumerWorkflow) -> KafkaConsumerWorkflow:
+        if in_order:
+            for topic in topics:
+                if topic.startswith("^"):
+                    raise DBOSInitializationError(
+                        f"Error: in-order processing is not supported for regular expression topic selectors ({topic})"
+                    )
+                queue = Queue(f"_dbos_kafka_queue_topic_{topic}", concurrency=1)
+                in_order_kafka_queues[topic] = queue
+        else:
+            global kafka_queue
+            kafka_queue = Queue("_dbos_internal_queue")
         stop_event = threading.Event()
         dbosreg.register_poller(
-            stop_event, _kafka_consumer_loop, func, config, topics, stop_event
+            stop_event, _kafka_consumer_loop, func, config, topics, stop_event, in_order
         )
         return func