dbos 0.25.1__py3-none-any.whl → 0.26.0__py3-none-any.whl

dbos/_dbos.py

@@ -4,7 +4,6 @@ import asyncio
 import atexit
 import hashlib
 import inspect
-import json
 import os
 import sys
 import threading
@@ -31,14 +30,10 @@ from typing import (
 
 from opentelemetry.trace import Span
 
-from dbos import _serialization
 from dbos._conductor.conductor import ConductorWebsocket
-from dbos._utils import GlobalParams
-from dbos._workflow_commands import (
-    WorkflowStatus,
-    list_queued_workflows,
-    list_workflows,
-)
+from dbos._sys_db import WorkflowStatus
+from dbos._utils import INTERNAL_QUEUE_NAME, GlobalParams
+from dbos._workflow_commands import fork_workflow, list_queued_workflows, list_workflows
 
 from ._classproperty import classproperty
 from ._core import (
@@ -62,13 +57,14 @@ from ._recovery import recover_pending_workflows, startup_recovery_thread
 from ._registrations import (
     DEFAULT_MAX_RECOVERY_ATTEMPTS,
     DBOSClassInfo,
+    _class_fqn,
     get_or_create_class_info,
     set_dbos_func_name,
     set_temp_workflow_type,
 )
 from ._roles import default_required_roles, required_roles
 from ._scheduler import ScheduledWorkflow, scheduled
-from ._sys_db import reset_system_database
+from ._sys_db import StepInfo, WorkflowStatus, reset_system_database
 from ._tracer import dbos_tracer
 
 if TYPE_CHECKING:
@@ -111,9 +107,10 @@ from ._error import (
     DBOSException,
     DBOSNonExistentWorkflowError,
 )
+from ._event_loop import BackgroundEventLoop
 from ._logger import add_otlp_to_all_loggers, config_logger, dbos_logger, init_logger
 from ._sys_db import SystemDatabase
-from ._workflow_commands import WorkflowStatus, get_workflow
+from ._workflow_commands import get_workflow, list_workflow_steps
 
 # Most DBOS functions are just any callable F, so decorators / wrappers work on F
 # There are cases where the parameters P and return value R should be separate
@@ -166,25 +163,32 @@ class DBOSRegistry:
         self.pollers: list[RegisteredJob] = []
         self.dbos: Optional[DBOS] = None
         self.config: Optional[ConfigFile] = None
-        self.workflow_cancelled_map: dict[str, bool] = {}
 
     def register_wf_function(self, name: str, wrapped_func: F, functype: str) -> None:
         if name in self.function_type_map:
             if self.function_type_map[name] != functype:
                 raise DBOSConflictingRegistrationError(name)
+            if name != TEMP_SEND_WF_NAME:
+                # Remove the `<temp>` prefix from the function name to avoid confusion
+                truncated_name = name.replace("<temp>.", "")
+                dbos_logger.warning(
+                    f"Duplicate registration of function '{truncated_name}'. A function named '{truncated_name}' has already been registered with DBOS. All functions registered with DBOS must have unique names."
+                )
         self.function_type_map[name] = functype
         self.workflow_info_map[name] = wrapped_func
 
     def register_class(self, cls: type, ci: DBOSClassInfo) -> None:
-        class_name = cls.__name__
+        class_name = _class_fqn(cls)
         if class_name in self.class_info_map:
             if self.class_info_map[class_name] is not cls:
                 raise Exception(f"Duplicate type registration for class '{class_name}'")
         else:
             self.class_info_map[class_name] = cls
 
-    def create_class_info(self, cls: Type[T]) -> Type[T]:
-        ci = get_or_create_class_info(cls)
+    def create_class_info(
+        self, cls: Type[T], class_name: Optional[str] = None
+    ) -> Type[T]:
+        ci = get_or_create_class_info(cls, class_name)
         self.register_class(cls, ci)
         return cls
 
@@ -199,7 +203,7 @@ class DBOSRegistry:
 
     def register_instance(self, inst: object) -> None:
         config_name = getattr(inst, "config_name")
-        class_name = inst.__class__.__name__
+        class_name = _class_fqn(inst.__class__)
         fn = f"{class_name}/{config_name}"
         if fn in self.instance_info_map:
             if self.instance_info_map[fn] is not inst:
@@ -209,15 +213,6 @@ class DBOSRegistry:
         else:
             self.instance_info_map[fn] = inst
 
-    def cancel_workflow(self, workflow_id: str) -> None:
-        self.workflow_cancelled_map[workflow_id] = True
-
-    def is_workflow_cancelled(self, workflow_id: str) -> bool:
-        return self.workflow_cancelled_map.get(workflow_id, False)
-
-    def clear_workflow_cancelled(self, workflow_id: str) -> None:
-        self.workflow_cancelled_map.pop(workflow_id, None)
-
     def compute_app_version(self) -> str:
         """
         An application's version is computed from a hash of the source of its workflows.
@@ -234,6 +229,13 @@ class DBOSRegistry:
             hasher.update(source.encode("utf-8"))
         return hasher.hexdigest()
 
+    def get_internal_queue(self) -> Queue:
+        """
+        Get or create the internal queue used for the DBOS scheduler, for Kafka, and for
+        programmatic resuming and restarting of workflows.
+        """
+        return Queue(INTERNAL_QUEUE_NAME)
+
 
 class DBOS:
     """
@@ -335,6 +337,7 @@ class DBOS:
         self.conductor_url: Optional[str] = conductor_url
         self.conductor_key: Optional[str] = conductor_key
         self.conductor_websocket: Optional[ConductorWebsocket] = None
+        self._background_event_loop: BackgroundEventLoop = BackgroundEventLoop()
 
         init_logger()
 
@@ -344,6 +347,9 @@ class DBOS:
             # If no config is provided, load it from dbos-config.yaml
             unvalidated_config = load_config(run_process_config=False)
         elif is_dbos_configfile(config):
+            dbos_logger.warning(
+                "ConfigFile config strutcture detected. This will be deprecated in favor of DBOSConfig in an upcoming release."
+            )
             unvalidated_config = cast(ConfigFile, config)
             if os.environ.get("DBOS__CLOUD") == "true":
                 unvalidated_config = overwrite_config(unvalidated_config)
@@ -357,13 +363,13 @@ class DBOS:
             check_config_consistency(name=unvalidated_config["name"])
 
         if unvalidated_config is not None:
-            self.config: ConfigFile = process_config(data=unvalidated_config)
+            self._config: ConfigFile = process_config(data=unvalidated_config)
         else:
             raise ValueError("No valid configuration was loaded.")
 
-        set_env_vars(self.config)
-        config_logger(self.config)
-        dbos_tracer.config(self.config)
+        set_env_vars(self._config)
+        config_logger(self._config)
+        dbos_tracer.config(self._config)
         dbos_logger.info("Initializing DBOS")
 
         # If using FastAPI, set up middleware and lifecycle events
@@ -445,20 +451,21 @@ class DBOS:
             dbos_logger.info(f"Executor ID: {GlobalParams.executor_id}")
             dbos_logger.info(f"Application version: {GlobalParams.app_version}")
             self._executor_field = ThreadPoolExecutor(max_workers=64)
+            self._background_event_loop.start()
             self._sys_db_field = SystemDatabase(
-                self.config["database"], debug_mode=debug_mode
+                self._config["database"], debug_mode=debug_mode
             )
             self._app_db_field = ApplicationDatabase(
-                self.config["database"], debug_mode=debug_mode
+                self._config["database"], debug_mode=debug_mode
             )
 
             if debug_mode:
                 return
 
-            admin_port = self.config.get("runtimeConfig", {}).get("admin_port")
+            admin_port = self._config.get("runtimeConfig", {}).get("admin_port")
             if admin_port is None:
                 admin_port = 3001
-            run_admin_server = self.config.get("runtimeConfig", {}).get(
+            run_admin_server = self._config.get("runtimeConfig", {}).get(
                 "run_admin_server"
             )
             if run_admin_server:
@@ -489,6 +496,9 @@ class DBOS:
             notification_listener_thread.start()
             self._background_threads.append(notification_listener_thread)
 
+            # Create the internal queue if it has not yet been created
+            self._registry.get_internal_queue()
+
             # Start the queue thread
             evt = threading.Event()
             self.stop_events.append(evt)
@@ -553,12 +563,13 @@ class DBOS:
         assert (
             not self._launched
         ), "The system database cannot be reset after DBOS is launched. Resetting the system database is a destructive operation that should only be used in a test environment."
-        reset_system_database(self.config)
+        reset_system_database(self._config)
 
     def _destroy(self) -> None:
         self._initialized = False
         for event in self.stop_events:
             event.set()
+        self._background_event_loop.stop()
         if self._sys_db_field is not None:
             self._sys_db_field.destroy()
             self._sys_db_field = None
@@ -588,7 +599,7 @@ class DBOS:
     # Decorators for DBOS functionality
     @classmethod
     def workflow(
-        cls, *, max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS
+        cls, *, max_recovery_attempts: Optional[int] = DEFAULT_MAX_RECOVERY_ATTEMPTS
     ) -> Callable[[Callable[P, R]], Callable[P, R]]:
         """Decorate a function for use as a DBOS workflow."""
         return decorate_workflow(_get_or_create_dbos_registry(), max_recovery_attempts)
@@ -635,15 +646,22 @@ class DBOS:
         )
 
     @classmethod
-    def dbos_class(cls) -> Callable[[Type[T]], Type[T]]:
+    def dbos_class(
+        cls, class_name: Optional[str] = None
+    ) -> Callable[[Type[T]], Type[T]]:
         """
         Decorate a class that contains DBOS member functions.
 
         All DBOS classes must be decorated, as this associates the class with
-        its member functions.
+        its member functions. Class names must be globally unique. By default, the class name is class.__qualname__  but you can optionally provide a class name that is different from the default name.
         """
 
-        return _get_or_create_dbos_registry().create_class_info
+        def register_class(cls: Type[T]) -> Type[T]:
+            # Register the class with the DBOS registry
+            _get_or_create_dbos_registry().create_class_info(cls, class_name)
+            return cls
+
+        return register_class
 
     @classmethod
     def default_required_roles(cls, roles: List[str]) -> Callable[[Type[T]], Type[T]]:
@@ -724,32 +742,11 @@ class DBOS:
     @classmethod
     def get_workflow_status(cls, workflow_id: str) -> Optional[WorkflowStatus]:
         """Return the status of a workflow execution."""
-        sys_db = _get_dbos_instance()._sys_db
-        ctx = get_local_dbos_context()
-        if ctx and ctx.is_within_workflow():
-            ctx.function_id += 1
-            res = sys_db.check_operation_execution(ctx.workflow_id, ctx.function_id)
-            if res is not None:
-                if res["output"]:
-                    resstat: WorkflowStatus = _serialization.deserialize(res["output"])
-                    return resstat
-                else:
-                    raise DBOSException(
-                        "Workflow status record not found. This should not happen! \033[1m Hint: Check if your workflow is deterministic.\033[0m"
-                    )
-        stat = get_workflow(_get_dbos_instance()._sys_db, workflow_id, True)
-
-        if ctx and ctx.is_within_workflow():
-            sys_db.record_operation_result(
-                {
-                    "workflow_uuid": ctx.workflow_id,
-                    "function_id": ctx.function_id,
-                    "function_name": "DBOS.getStatus",
-                    "output": _serialization.serialize(stat),
-                    "error": None,
-                }
-            )
-        return stat
+
+        def fn() -> Optional[WorkflowStatus]:
+            return get_workflow(_get_dbos_instance()._sys_db, workflow_id, True)
+
+        return _get_dbos_instance()._sys_db.call_function_as_step(fn, "DBOS.getStatus")
 
     @classmethod
     async def get_workflow_status_async(
@@ -925,17 +922,12 @@ class DBOS:
         )
 
     @classmethod
-    def execute_workflow_id(cls, workflow_id: str) -> WorkflowHandle[Any]:
+    def _execute_workflow_id(cls, workflow_id: str) -> WorkflowHandle[Any]:
         """Execute a workflow by ID (for recovery)."""
         return execute_workflow_by_id(_get_dbos_instance(), workflow_id)
 
     @classmethod
-    def restart_workflow(cls, workflow_id: str) -> None:
-        """Execute a workflow by ID (for recovery)."""
-        execute_workflow_by_id(_get_dbos_instance(), workflow_id, True)
-
-    @classmethod
-    def recover_pending_workflows(
+    def _recover_pending_workflows(
         cls, executor_ids: List[str] = ["local"]
     ) -> List[WorkflowHandle[Any]]:
         """Find all PENDING workflows and execute them."""
@@ -944,17 +936,48 @@ class DBOS:
     @classmethod
     def cancel_workflow(cls, workflow_id: str) -> None:
         """Cancel a workflow by ID."""
-        dbos_logger.info(f"Cancelling workflow: {workflow_id}")
-        _get_dbos_instance()._sys_db.cancel_workflow(workflow_id)
-        _get_or_create_dbos_registry().cancel_workflow(workflow_id)
+
+        def fn() -> None:
+            dbos_logger.info(f"Cancelling workflow: {workflow_id}")
+            _get_dbos_instance()._sys_db.cancel_workflow(workflow_id)
+
+        return _get_dbos_instance()._sys_db.call_function_as_step(
+            fn, "DBOS.cancelWorkflow"
+        )
 
     @classmethod
     def resume_workflow(cls, workflow_id: str) -> WorkflowHandle[Any]:
         """Resume a workflow by ID."""
-        dbos_logger.info(f"Resuming workflow: {workflow_id}")
-        _get_dbos_instance()._sys_db.resume_workflow(workflow_id)
-        _get_or_create_dbos_registry().clear_workflow_cancelled(workflow_id)
-        return execute_workflow_by_id(_get_dbos_instance(), workflow_id, False)
+
+        def fn() -> None:
+            dbos_logger.info(f"Resuming workflow: {workflow_id}")
+            _get_dbos_instance()._sys_db.resume_workflow(workflow_id)
+
+        _get_dbos_instance()._sys_db.call_function_as_step(fn, "DBOS.resumeWorkflow")
+        return cls.retrieve_workflow(workflow_id)
+
+    @classmethod
+    def restart_workflow(cls, workflow_id: str) -> WorkflowHandle[Any]:
+        """Restart a workflow with a new workflow ID"""
+        return cls.fork_workflow(workflow_id, 1)
+
+    @classmethod
+    def fork_workflow(cls, workflow_id: str, start_step: int) -> WorkflowHandle[Any]:
+        """Restart a workflow with a new workflow ID from a specific step"""
+
+        def fn() -> str:
+            dbos_logger.info(f"Forking workflow: {workflow_id} from step {start_step}")
+            return fork_workflow(
+                _get_dbos_instance()._sys_db,
+                _get_dbos_instance()._app_db,
+                workflow_id,
+                start_step,
+            )
+
+        new_id = _get_dbos_instance()._sys_db.call_function_as_step(
+            fn, "DBOS.forkWorkflow"
+        )
+        return cls.retrieve_workflow(new_id)
 
     @classmethod
     def list_workflows(
@@ -970,19 +993,26 @@ class DBOS:
         limit: Optional[int] = None,
         offset: Optional[int] = None,
         sort_desc: bool = False,
+        workflow_id_prefix: Optional[str] = None,
     ) -> List[WorkflowStatus]:
-        return list_workflows(
-            _get_dbos_instance()._sys_db,
-            workflow_ids=workflow_ids,
-            status=status,
-            start_time=start_time,
-            end_time=end_time,
-            name=name,
-            app_version=app_version,
-            user=user,
-            limit=limit,
-            offset=offset,
-            sort_desc=sort_desc,
+        def fn() -> List[WorkflowStatus]:
+            return list_workflows(
+                _get_dbos_instance()._sys_db,
+                workflow_ids=workflow_ids,
+                status=status,
+                start_time=start_time,
+                end_time=end_time,
+                name=name,
+                app_version=app_version,
+                user=user,
+                limit=limit,
+                offset=offset,
+                sort_desc=sort_desc,
+                workflow_id_prefix=workflow_id_prefix,
+            )
+
+        return _get_dbos_instance()._sys_db.call_function_as_step(
+            fn, "DBOS.listWorkflows"
         )
 
     @classmethod
@@ -998,16 +1028,32 @@ class DBOS:
         offset: Optional[int] = None,
         sort_desc: bool = False,
     ) -> List[WorkflowStatus]:
-        return list_queued_workflows(
-            _get_dbos_instance()._sys_db,
-            queue_name=queue_name,
-            status=status,
-            start_time=start_time,
-            end_time=end_time,
-            name=name,
-            limit=limit,
-            offset=offset,
-            sort_desc=sort_desc,
+        def fn() -> List[WorkflowStatus]:
+            return list_queued_workflows(
+                _get_dbos_instance()._sys_db,
+                queue_name=queue_name,
+                status=status,
+                start_time=start_time,
+                end_time=end_time,
+                name=name,
+                limit=limit,
+                offset=offset,
+                sort_desc=sort_desc,
+            )
+
+        return _get_dbos_instance()._sys_db.call_function_as_step(
+            fn, "DBOS.listQueuedWorkflows"
+        )
+
+    @classmethod
+    def list_workflow_steps(cls, workflow_id: str) -> List[StepInfo]:
+        def fn() -> List[StepInfo]:
+            return list_workflow_steps(
+                _get_dbos_instance()._sys_db, _get_dbos_instance()._app_db, workflow_id
+            )
+
+        return _get_dbos_instance()._sys_db.call_function_as_step(
+            fn, "DBOS.listWorkflowSteps"
         )
 
     @classproperty
@@ -1020,15 +1066,15 @@ class DBOS:
         """Return the DBOS `ConfigFile` for the current context."""
         global _dbos_global_instance
         if _dbos_global_instance is not None:
-            return _dbos_global_instance.config
+            return _dbos_global_instance._config
         reg = _get_or_create_dbos_registry()
         if reg.config is not None:
             return reg.config
-        config = (
+        loaded_config = (
             load_config()
         )  # This will return the processed & validated config (with defaults)
-        reg.config = config
-        return config
+        reg.config = loaded_config
+        return loaded_config
 
     @classproperty
     def sql_session(cls) -> Session:
@@ -1050,11 +1096,11 @@ class DBOS:
 
     @classproperty
     def step_id(cls) -> int:
-        """Return the step ID for the current context. This is a unique identifier of the current step within the workflow."""
+        """Return the step ID for the currently executing step. This is a unique identifier of the current step within the workflow."""
         ctx = assert_current_dbos_context()
         assert (
-            ctx.is_within_workflow()
-        ), "step_id is only available within a DBOS workflow."
+            ctx.is_step() or ctx.is_transaction()
+        ), "step_id is only available within a DBOS step."
         return ctx.function_id
 
     @classproperty