dbos 0.19.0a4__py3-none-any.whl → 0.20.0__py3-none-any.whl

dbos/_admin_server.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import re
 import threading
 from functools import partial
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -15,6 +16,9 @@ if TYPE_CHECKING:
 _health_check_path = "/dbos-healthz"
 _workflow_recovery_path = "/dbos-workflow-recovery"
 _deactivate_path = "/deactivate"
+# /workflows/:workflow_id/cancel
+# /workflows/:workflow_id/resume
+# /workflows/:workflow_id/restart
 
 
 class AdminServer:
@@ -79,12 +83,51 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
             self._end_headers()
             self.wfile.write(json.dumps(workflow_ids).encode("utf-8"))
         else:
-            self.send_response(404)
-            self._end_headers()
+
+            restart_match = re.match(
+                r"^/workflows/(?P<workflow_id>[^/]+)/restart$", self.path
+            )
+            resume_match = re.match(
+                r"^/workflows/(?P<workflow_id>[^/]+)/resume$", self.path
+            )
+            cancel_match = re.match(
+                r"^/workflows/(?P<workflow_id>[^/]+)/cancel$", self.path
+            )
+
+            if restart_match:
+                workflow_id = restart_match.group("workflow_id")
+                self._handle_restart(workflow_id)
+            elif resume_match:
+                workflow_id = resume_match.group("workflow_id")
+                self._handle_resume(workflow_id)
+            elif cancel_match:
+                workflow_id = cancel_match.group("workflow_id")
+                self._handle_cancel(workflow_id)
+            else:
+                self.send_response(404)
+                self._end_headers()
 
     def log_message(self, format: str, *args: Any) -> None:
         return  # Disable admin server request logging
 
+    def _handle_restart(self, workflow_id: str) -> None:
+        self.dbos.restart_workflow(workflow_id)
+        print("Restarting workflow", workflow_id)
+        self.send_response(204)
+        self._end_headers()
+
+    def _handle_resume(self, workflow_id: str) -> None:
+        print("Resuming workflow", workflow_id)
+        self.dbos.resume_workflow(workflow_id)
+        self.send_response(204)
+        self._end_headers()
+
+    def _handle_cancel(self, workflow_id: str) -> None:
+        print("Cancelling workflow", workflow_id)
+        self.dbos.cancel_workflow(workflow_id)
+        self.send_response(204)
+        self._end_headers()
+
 
 # Be consistent with DBOS-TS response.
 class PerfUtilization(TypedDict):

dbos/_context.py

@@ -57,6 +57,7 @@ class DBOSContext:
         self.request: Optional["Request"] = None
 
         self.id_assigned_for_next_workflow: str = ""
+        self.is_within_set_workflow_id_block: bool = False
 
         self.parent_workflow_id: str = ""
         self.parent_workflow_fid: int = -1
@@ -78,6 +79,7 @@ class DBOSContext:
         rv.logger = self.logger
         rv.id_assigned_for_next_workflow = self.id_assigned_for_next_workflow
         self.id_assigned_for_next_workflow = ""
+        rv.is_within_set_workflow_id_block = self.is_within_set_workflow_id_block
         rv.parent_workflow_id = self.workflow_id
         rv.parent_workflow_fid = self.function_id
         rv.in_recovery = self.in_recovery
@@ -95,6 +97,10 @@ class DBOSContext:
         if len(self.id_assigned_for_next_workflow) > 0:
             wfid = self.id_assigned_for_next_workflow
         else:
+            if self.is_within_set_workflow_id_block:
+                self.logger.warning(
+                    f"Multiple workflows started in the same SetWorkflowID block. Only the first workflow is assigned the specified workflow ID; subsequent workflows will use a generated workflow ID."
+                )
             wfid = str(uuid.uuid4())
         return wfid
 
@@ -286,7 +292,7 @@ class DBOSContextSwap:
 
 class SetWorkflowID:
     """
-    Set the workflow ID to be used for the enclosed workflow invocation.
+    Set the workflow ID to be used for the enclosed workflow invocation. Note: Only the first workflow will be started with the specified workflow ID within a `with SetWorkflowID` block.
 
     Typical Usage
         ```
@@ -311,7 +317,9 @@ class SetWorkflowID:
         if ctx is None:
             self.created_ctx = True
             _set_local_dbos_context(DBOSContext())
-        assert_current_dbos_context().id_assigned_for_next_workflow = self.wfid
+        ctx = assert_current_dbos_context()
+        ctx.id_assigned_for_next_workflow = self.wfid
+        ctx.is_within_set_workflow_id_block = True
         return self
 
     def __exit__(
@@ -321,6 +329,7 @@ class SetWorkflowID:
         traceback: Optional[TracebackType],
     ) -> Literal[False]:
         # Code to clean up the basic context if we created it
+        assert_current_dbos_context().is_within_set_workflow_id_block = False
         if self.created_ctx:
             _clear_local_dbos_context()
         return False  # Did not handle

dbos/_core.py

@@ -188,6 +188,7 @@ def _init_workflow(
         wf_status = dbos._sys_db.update_workflow_status(
             status, False, ctx.in_recovery, max_recovery_attempts=max_recovery_attempts
         )
+        # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
         dbos._sys_db.update_workflow_inputs(wfid, _serialization.serialize_args(inputs))
     else:
         # Buffer the inputs for single-transaction workflows, but don't buffer the status
@@ -265,7 +266,9 @@ def _execute_workflow_wthread(
                 raise
 
 
-def execute_workflow_by_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]":
+def execute_workflow_by_id(
+    dbos: "DBOS", workflow_id: str, startNew: bool = False
+) -> "WorkflowHandle[Any]":
     status = dbos._sys_db.get_workflow_status(workflow_id)
     if not status:
         raise DBOSRecoveryError(workflow_id, "Workflow status not found")
@@ -292,7 +295,8 @@ def execute_workflow_by_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[An
                     workflow_id,
                     f"Cannot execute workflow because instance '{iname}' is not registered",
                 )
-            with SetWorkflowID(workflow_id):
+
+            if startNew:
                 return start_workflow(
                     dbos,
                     wf_func,
@@ -302,6 +306,17 @@ def execute_workflow_by_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[An
                     *inputs["args"],
                     **inputs["kwargs"],
                 )
+            else:
+                with SetWorkflowID(workflow_id):
+                    return start_workflow(
+                        dbos,
+                        wf_func,
+                        status["queue_name"],
+                        True,
+                        dbos._registry.instance_info_map[iname],
+                        *inputs["args"],
+                        **inputs["kwargs"],
+                    )
         elif status["class_name"] is not None:
             class_name = status["class_name"]
             if class_name not in dbos._registry.class_info_map:
@@ -309,7 +324,8 @@ def execute_workflow_by_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[An
                     workflow_id,
                     f"Cannot execute workflow because class '{class_name}' is not registered",
                 )
-            with SetWorkflowID(workflow_id):
+
+            if startNew:
                 return start_workflow(
                     dbos,
                     wf_func,
@@ -319,8 +335,19 @@ def execute_workflow_by_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[An
                     *inputs["args"],
                     **inputs["kwargs"],
                 )
+            else:
+                with SetWorkflowID(workflow_id):
+                    return start_workflow(
+                        dbos,
+                        wf_func,
+                        status["queue_name"],
+                        True,
+                        dbos._registry.class_info_map[class_name],
+                        *inputs["args"],
+                        **inputs["kwargs"],
+                    )
         else:
-            with SetWorkflowID(workflow_id):
+            if startNew:
                 return start_workflow(
                     dbos,
                     wf_func,
@@ -329,6 +356,16 @@ def execute_workflow_by_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[An
                     *inputs["args"],
                     **inputs["kwargs"],
                 )
+            else:
+                with SetWorkflowID(workflow_id):
+                    return start_workflow(
+                        dbos,
+                        wf_func,
+                        status["queue_name"],
+                        True,
+                        *inputs["args"],
+                        **inputs["kwargs"],
+                    )
 
 
 @overload
@@ -422,6 +459,9 @@ def start_workflow(
         or wf_status == WorkflowStatusString.ERROR.value
         or wf_status == WorkflowStatusString.SUCCESS.value
     ):
+        dbos.logger.debug(
+            f"Workflow {new_wf_id} already completed with status {wf_status}. Directly returning a workflow handle."
+        )
         return WorkflowHandlePolling(new_wf_id, dbos)
 
     if fself is not None:
@@ -448,6 +488,22 @@ def start_workflow(
     return WorkflowHandleFuture(new_wf_id, future, dbos)
 
 
+if sys.version_info < (3, 12):
+
+    def _mark_coroutine(func: Callable[P, R]) -> Callable[P, R]:
+        @wraps(func)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> R:
+            return await func(*args, **kwargs)  # type: ignore
+
+        return async_wrapper  # type: ignore
+
+else:
+
+    def _mark_coroutine(func: Callable[P, R]) -> Callable[P, R]:
+        inspect.markcoroutinefunction(func)
+        return func
+
+
 def workflow_wrapper(
     dbosreg: "DBOSRegistry",
     func: Callable[P, R],
@@ -494,7 +550,7 @@ def workflow_wrapper(
                 temp_wf_type=get_temp_workflow_type(func),
                 max_recovery_attempts=max_recovery_attempts,
             )
-
+            # TODO: maybe modify the parameters if they've been changed by `_init_workflow`
             dbos.logger.debug(
                 f"Running workflow, id: {ctx.workflow_id}, name: {get_dbos_func_name(func)}"
             )
@@ -508,7 +564,7 @@ def workflow_wrapper(
         )
         return outcome()  # type: ignore
 
-    return wrapper
+    return _mark_coroutine(wrapper) if inspect.iscoroutinefunction(func) else wrapper
 
 
 def decorate_workflow(
@@ -798,6 +854,10 @@ def decorate_step(
                 assert tempwf
                 return tempwf(*args, **kwargs)
 
+        wrapper = (
+            _mark_coroutine(wrapper) if inspect.iscoroutinefunction(func) else wrapper  # type: ignore
+        )
+
         def temp_wf_sync(*args: Any, **kwargs: Any) -> Any:
             return wrapper(*args, **kwargs)
 

dbos/_dbos.py

@@ -56,6 +56,7 @@ from ._registrations import (
 )
 from ._roles import default_required_roles, required_roles
 from ._scheduler import ScheduledWorkflow, scheduled
+from ._sys_db import WorkflowStatusString, reset_system_database
 from ._tracer import dbos_tracer
 
 if TYPE_CHECKING:
@@ -231,6 +232,7 @@ class DBOS:
                         f"DBOS configured multiple times with conflicting information"
                     )
                 config = _dbos_global_registry.config
+
             _dbos_global_instance = super().__new__(cls)
             _dbos_global_instance.__init__(fastapi=fastapi, config=config, flask=flask)  # type: ignore
         else:
@@ -243,7 +245,7 @@ class DBOS:
         return _dbos_global_instance
 
     @classmethod
-    def destroy(cls, *, destroy_registry: bool = True) -> None:
+    def destroy(cls, *, destroy_registry: bool = False) -> None:
         global _dbos_global_instance
         if _dbos_global_instance is not None:
             _dbos_global_instance._destroy()
@@ -407,6 +409,22 @@ class DBOS:
             dbos_logger.error(f"DBOS failed to launch: {traceback.format_exc()}")
             raise
 
+    @classmethod
+    def reset_system_database(cls) -> None:
+        """
+        Destroy the DBOS system database. Useful for resetting the state of DBOS between tests.
+        This is a destructive operation and should only be used in a test environment.
+        More information on testing DBOS apps: https://docs.dbos.dev/python/tutorials/testing
+        """
+        if _dbos_global_instance is not None:
+            _dbos_global_instance._reset_system_database()
+
+    def _reset_system_database(self) -> None:
+        assert (
+            not self._launched
+        ), "The system database cannot be reset after DBOS is launched. Resetting the system database is a destructive operation that should only be used in a test environment."
+        reset_system_database(self.config)
+
     def _destroy(self) -> None:
         self._initialized = False
         for event in self.stop_events:
@@ -767,6 +785,11 @@ class DBOS:
         """Execute a workflow by ID (for recovery)."""
         return execute_workflow_by_id(_get_dbos_instance(), workflow_id)
 
+    @classmethod
+    def restart_workflow(cls, workflow_id: str) -> None:
+        """Execute a workflow by ID (for recovery)."""
+        execute_workflow_by_id(_get_dbos_instance(), workflow_id, True)
+
     @classmethod
     def recover_pending_workflows(
         cls, executor_ids: List[str] = ["local"]
@@ -774,6 +797,18 @@ class DBOS:
         """Find all PENDING workflows and execute them."""
         return recover_pending_workflows(_get_dbos_instance(), executor_ids)
 
+    @classmethod
+    def cancel_workflow(cls, workflow_id: str) -> None:
+        """Cancel a workflow by ID."""
+        _get_dbos_instance()._sys_db.set_workflow_status(
+            workflow_id, WorkflowStatusString.CANCELLED, False
+        )
+
+    @classmethod
+    def resume_workflow(cls, workflow_id: str) -> None:
+        """Resume a workflow by ID."""
+        execute_workflow_by_id(_get_dbos_instance(), workflow_id, False)
+
     @classproperty
     def logger(cls) -> Logger:
         """Return the DBOS `Logger` for the current context."""

dbos/_error.py

@@ -35,6 +35,7 @@ class DBOSErrorCode(Enum):
     DeadLetterQueueError = 6
     MaxStepRetriesExceeded = 7
     NotAuthorized = 8
+    ConflictingWorkflowError = 9
 
 
 class DBOSWorkflowConflictIDError(DBOSException):
@@ -47,6 +48,16 @@ class DBOSWorkflowConflictIDError(DBOSException):
         )
 
 
+class DBOSConflictingWorkflowError(DBOSException):
+    """Exception raised different workflows started with the same workflow ID."""
+
+    def __init__(self, workflow_id: str, message: Optional[str] = None):
+        super().__init__(
+            f"Conflicting workflow invocation with the same ID ({workflow_id}): {message}",
+            dbos_error_code=DBOSErrorCode.ConflictingWorkflowError.value,
+        )
+
+
 class DBOSRecoveryError(DBOSException):
     """Exception raised when a workflow recovery fails."""
 

dbos/_fastapi.py

@@ -1,10 +1,10 @@
 import uuid
-from typing import Any, Callable, cast
+from typing import Any, Callable, MutableMapping, cast
 
 from fastapi import FastAPI
 from fastapi import Request as FastAPIRequest
 from fastapi.responses import JSONResponse
-from starlette.types import ASGIApp, Message, Receive, Scope, Send
+from starlette.types import ASGIApp, Receive, Scope, Send
 
 from . import DBOS
 from ._context import (
@@ -61,15 +61,16 @@ class LifespanMiddleware:
 
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
         if scope["type"] == "lifespan":
-            while True:
-                message = await receive()
-                if message["type"] == "lifespan.startup":
+
+            async def wrapped_send(message: MutableMapping[str, Any]) -> None:
+                if message["type"] == "lifespan.startup.complete":
                     self.dbos._launch()
-                    await send({"type": "lifespan.startup.complete"})
-                elif message["type"] == "lifespan.shutdown":
+                elif message["type"] == "lifespan.shutdown.complete":
                     self.dbos._destroy()
-                    await send({"type": "lifespan.shutdown.complete"})
-                    break
+                await send(message)
+
+            # Call the original app with our wrapped functions
+            await self.app(scope, receive, wrapped_send)
         else:
             await self.app(scope, receive, send)
 
@@ -94,7 +95,11 @@ def setup_fastapi_middleware(app: FastAPI, dbos: DBOS) -> None:
         with EnterDBOSHandler(attributes):
             ctx = assert_current_dbos_context()
             ctx.request = _make_request(request)
-            workflow_id = request.headers.get("dbos-idempotency-key", "")
-            with SetWorkflowID(workflow_id):
+            workflow_id = request.headers.get("dbos-idempotency-key")
+            if workflow_id is not None:
+                # Set the workflow ID for the handler
+                with SetWorkflowID(workflow_id):
+                    response = await call_next(request)
+            else:
                 response = await call_next(request)
         return response

dbos/_flask.py

@@ -34,8 +34,12 @@ class FlaskMiddleware:
         with EnterDBOSHandler(attributes):
             ctx = assert_current_dbos_context()
             ctx.request = _make_request(request)
-            workflow_id = request.headers.get("dbos-idempotency-key", "")
-            with SetWorkflowID(workflow_id):
+            workflow_id = request.headers.get("dbos-idempotency-key")
+            if workflow_id is not None:
+                # Set the workflow ID for the handler
+                with SetWorkflowID(workflow_id):
+                    response = self.app(environ, start_response)
+            else:
                 response = self.app(environ, start_response)
         return response
 

dbos/_kafka.py

@@ -1,3 +1,4 @@
+import re
 import threading
 from typing import TYPE_CHECKING, Any, Callable, NoReturn
 
@@ -19,6 +20,14 @@ _kafka_queue: Queue
 _in_order_kafka_queues: dict[str, Queue] = {}
 
 
+def safe_group_name(method_name: str, topics: list[str]) -> str:
+    safe_group_id = "-".join(
+        re.sub(r"[^a-zA-Z0-9\-]", "", str(r)) for r in [method_name, *topics]
+    )
+
+    return f"dbos-kafka-group-{safe_group_id}"[:255]
+
+
 def _kafka_consumer_loop(
     func: _KafkaConsumerWorkflow,
     config: dict[str, Any],
@@ -34,6 +43,12 @@ def _kafka_consumer_loop(
     if "auto.offset.reset" not in config:
         config["auto.offset.reset"] = "earliest"
 
+    if config.get("group.id") is None:
+        config["group.id"] = safe_group_name(func.__qualname__, topics)
+        dbos_logger.warning(
+            f"Consumer group ID not found. Using generated group.id {config['group.id']}"
+        )
+
     consumer = Consumer(config)
     try:
         consumer.subscribe(topics)
@@ -71,8 +86,9 @@ def _kafka_consumer_loop(
                     topic=cmsg.topic(),
                     value=cmsg.value(),
                 )
+                groupID = config.get("group.id")
                 with SetWorkflowID(
-                    f"kafka-unique-id-{msg.topic}-{msg.partition}-{msg.offset}"
+                    f"kafka-unique-id-{msg.topic}-{msg.partition}-{groupID}-{msg.offset}"
                 ):
                     if in_order:
                         assert msg.topic is not None

dbos/_queue.py

@@ -36,6 +36,7 @@ class Queue:
         name: str,
         concurrency: Optional[int] = None,
         limiter: Optional[QueueRateLimit] = None,
+        *,  # Disable positional arguments from here on
         worker_concurrency: Optional[int] = None,
     ) -> None:
         if (