dbos 0.23.0a8__tar.gz → 0.23.0a10__tar.gz

{dbos-0.23.0a8 → dbos-0.23.0a10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.23.0a8
+Version: 0.23.0a10
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

dbos-0.23.0a10/dbos/__main__.py

@@ -0,0 +1,26 @@
+import re
+import sys
+from typing import NoReturn, Optional, Union
+
+from dbos.cli.cli import app
+
+
+def main() -> NoReturn:
+    # Modify sys.argv[0] to remove script or executable extensions
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+
+    retval: Optional[Union[str, int]] = 1
+    try:
+        app()
+        retval = None
+    except SystemExit as e:
+        retval = e.code
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        retval = 1
+    finally:
+        sys.exit(retval)
+
+
+if __name__ == "__main__":
+    main()

{dbos-0.23.0a8 → dbos-0.23.0a10}/dbos/_app_db.py

@@ -27,29 +27,30 @@ class RecordedResult(TypedDict):
 
 class ApplicationDatabase:
 
-    def __init__(self, config: ConfigFile):
+    def __init__(self, config: ConfigFile, *, debug_mode: bool = False):
         self.config = config
 
         app_db_name = config["database"]["app_db_name"]
 
         # If the application database does not already exist, create it
-        postgres_db_url = sa.URL.create(
-            "postgresql+psycopg",
-            username=config["database"]["username"],
-            password=config["database"]["password"],
-            host=config["database"]["hostname"],
-            port=config["database"]["port"],
-            database="postgres",
-        )
-        postgres_db_engine = sa.create_engine(postgres_db_url)
-        with postgres_db_engine.connect() as conn:
-            conn.execution_options(isolation_level="AUTOCOMMIT")
-            if not conn.execute(
-                sa.text("SELECT 1 FROM pg_database WHERE datname=:db_name"),
-                parameters={"db_name": app_db_name},
-            ).scalar():
-                conn.execute(sa.text(f"CREATE DATABASE {app_db_name}"))
-        postgres_db_engine.dispose()
+        if not debug_mode:
+            postgres_db_url = sa.URL.create(
+                "postgresql+psycopg",
+                username=config["database"]["username"],
+                password=config["database"]["password"],
+                host=config["database"]["hostname"],
+                port=config["database"]["port"],
+                database="postgres",
+            )
+            postgres_db_engine = sa.create_engine(postgres_db_url)
+            with postgres_db_engine.connect() as conn:
+                conn.execution_options(isolation_level="AUTOCOMMIT")
+                if not conn.execute(
+                    sa.text("SELECT 1 FROM pg_database WHERE datname=:db_name"),
+                    parameters={"db_name": app_db_name},
+                ).scalar():
+                    conn.execute(sa.text(f"CREATE DATABASE {app_db_name}"))
+            postgres_db_engine.dispose()
 
         # Create a connection pool for the application database
         app_db_url = sa.URL.create(
@@ -64,14 +65,16 @@ class ApplicationDatabase:
             app_db_url, pool_size=20, max_overflow=5, pool_timeout=30
         )
         self.sessionmaker = sessionmaker(bind=self.engine)
+        self.debug_mode = debug_mode
 
         # Create the dbos schema and transaction_outputs table in the application database
-        with self.engine.begin() as conn:
-            schema_creation_query = sa.text(
-                f"CREATE SCHEMA IF NOT EXISTS {ApplicationSchema.schema}"
-            )
-            conn.execute(schema_creation_query)
-        ApplicationSchema.metadata_obj.create_all(self.engine)
+        if not debug_mode:
+            with self.engine.begin() as conn:
+                schema_creation_query = sa.text(
+                    f"CREATE SCHEMA IF NOT EXISTS {ApplicationSchema.schema}"
+                )
+                conn.execute(schema_creation_query)
+            ApplicationSchema.metadata_obj.create_all(self.engine)
 
     def destroy(self) -> None:
         self.engine.dispose()
@@ -100,6 +103,8 @@ class ApplicationDatabase:
             raise
 
     def record_transaction_error(self, output: TransactionResultInternal) -> None:
+        if self.debug_mode:
+            raise Exception("called record_transaction_error in debug mode")
         try:
             with self.engine.begin() as conn:
                 conn.execute(

{dbos-0.23.0a8 → dbos-0.23.0a10}/dbos/_core.py

@@ -186,21 +186,31 @@ def _init_workflow(
         inputs = {"args": inputs["args"][1:], "kwargs": inputs["kwargs"]}
 
     wf_status = status["status"]
-    if temp_wf_type != "transaction" or queue is not None:
-        # Synchronously record the status and inputs for workflows and single-step workflows
-        # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
-        # TODO: Make this transactional (and with the queue step below)
-        wf_status = dbos._sys_db.insert_workflow_status(
-            status, max_recovery_attempts=max_recovery_attempts
-        )
-        # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
-        dbos._sys_db.update_workflow_inputs(wfid, _serialization.serialize_args(inputs))
+    if dbos.debug_mode:
+        get_status_result = dbos._sys_db.get_workflow_status(wfid)
+        if get_status_result is None:
+            raise DBOSNonExistentWorkflowError(wfid)
+        wf_status = get_status_result["status"]
     else:
-        # Buffer the inputs for single-transaction workflows, but don't buffer the status
-        dbos._sys_db.buffer_workflow_inputs(wfid, _serialization.serialize_args(inputs))
+        if temp_wf_type != "transaction" or queue is not None:
+            # Synchronously record the status and inputs for workflows and single-step workflows
+            # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
+            # TODO: Make this transactional (and with the queue step below)
+            wf_status = dbos._sys_db.insert_workflow_status(
+                status, max_recovery_attempts=max_recovery_attempts
+            )
+            # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
+            dbos._sys_db.update_workflow_inputs(
+                wfid, _serialization.serialize_args(inputs)
+            )
+        else:
+            # Buffer the inputs for single-transaction workflows, but don't buffer the status
+            dbos._sys_db.buffer_workflow_inputs(
+                wfid, _serialization.serialize_args(inputs)
+            )
 
-    if queue is not None and wf_status == WorkflowStatusString.ENQUEUED.value:
-        dbos._sys_db.enqueue(wfid, queue)
+        if queue is not None and wf_status == WorkflowStatusString.ENQUEUED.value:
+            dbos._sys_db.enqueue(wfid, queue)
 
     status["status"] = wf_status
     return status
@@ -215,10 +225,11 @@ def _get_wf_invoke_func(
             output = func()
             status["status"] = "SUCCESS"
             status["output"] = _serialization.serialize(output)
-            if status["queue_name"] is not None:
-                queue = dbos._registry.queue_info_map[status["queue_name"]]
-                dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
-            dbos._sys_db.buffer_workflow_status(status)
+            if not dbos.debug_mode:
+                if status["queue_name"] is not None:
+                    queue = dbos._registry.queue_info_map[status["queue_name"]]
+                    dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
+                dbos._sys_db.buffer_workflow_status(status)
             return output
         except DBOSWorkflowConflictIDError:
             # Retrieve the workflow handle and wait for the result.
@@ -233,10 +244,11 @@ def _get_wf_invoke_func(
         except Exception as error:
             status["status"] = "ERROR"
             status["error"] = _serialization.serialize_exception(error)
-            if status["queue_name"] is not None:
-                queue = dbos._registry.queue_info_map[status["queue_name"]]
-                dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
-            dbos._sys_db.update_workflow_status(status)
+            if not dbos.debug_mode:
+                if status["queue_name"] is not None:
+                    queue = dbos._registry.queue_info_map[status["queue_name"]]
+                    dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
+                dbos._sys_db.update_workflow_status(status)
             raise
 
     return persist
@@ -422,10 +434,12 @@ def start_workflow(
 
     wf_status = status["status"]
 
-    if (
-        not execute_workflow
-        or wf_status == WorkflowStatusString.ERROR.value
-        or wf_status == WorkflowStatusString.SUCCESS.value
+    if not execute_workflow or (
+        not dbos.debug_mode
+        and (
+            wf_status == WorkflowStatusString.ERROR.value
+            or wf_status == WorkflowStatusString.SUCCESS.value
+        )
     ):
         dbos.logger.debug(
             f"Workflow {new_wf_id} already completed with status {wf_status}. Directly returning a workflow handle."
@@ -597,6 +611,10 @@ def decorate_transaction(
                                         ctx.function_id,
                                     )
                                 )
+                                if dbos.debug_mode and recorded_output is None:
+                                    raise DBOSException(
+                                        "Transaction output not found in debug mode"
+                                    )
                                 if recorded_output:
                                     dbos.logger.debug(
                                         f"Replaying transaction, id: {ctx.function_id}, name: {attributes['name']}"
@@ -780,6 +798,8 @@ def decorate_step(
                 recorded_output = dbos._sys_db.check_operation_execution(
                     ctx.workflow_id, ctx.function_id
                 )
+                if dbos.debug_mode and recorded_output is None:
+                    raise DBOSException("Step output not found in debug mode")
                 if recorded_output:
                     dbos.logger.debug(
                         f"Replaying step, id: {ctx.function_id}, name: {attributes['name']}"

{dbos-0.23.0a8 → dbos-0.23.0a10}/dbos/_dbos.py

@@ -313,6 +313,7 @@ class DBOS:
         dbos_logger.info("Initializing DBOS")
         self.config: ConfigFile = config
         self._launched: bool = False
+        self._debug_mode: bool = False
         self._sys_db_field: Optional[SystemDatabase] = None
         self._app_db_field: Optional[ApplicationDatabase] = None
         self._registry: DBOSRegistry = _get_or_create_dbos_registry()
@@ -380,23 +381,32 @@ class DBOS:
         rv: AdminServer = self._admin_server_field
         return rv
 
+    @property
+    def debug_mode(self) -> bool:
+        return self._debug_mode
+
     @classmethod
-    def launch(cls) -> None:
+    def launch(cls, *, debug_mode: bool = False) -> None:
         if _dbos_global_instance is not None:
-            _dbos_global_instance._launch()
+            _dbos_global_instance._launch(debug_mode=debug_mode)
 
-    def _launch(self) -> None:
+    def _launch(self, *, debug_mode: bool = False) -> None:
         try:
             if self._launched:
                 dbos_logger.warning(f"DBOS was already launched")
                 return
             self._launched = True
+            self._debug_mode = debug_mode
             if GlobalParams.app_version == "":
                 GlobalParams.app_version = self._registry.compute_app_version()
             dbos_logger.info(f"Application version: {GlobalParams.app_version}")
             self._executor_field = ThreadPoolExecutor(max_workers=64)
-            self._sys_db_field = SystemDatabase(self.config)
-            self._app_db_field = ApplicationDatabase(self.config)
+            self._sys_db_field = SystemDatabase(self.config, debug_mode=debug_mode)
+            self._app_db_field = ApplicationDatabase(self.config, debug_mode=debug_mode)
+
+            if debug_mode:
+                return
+
             admin_port = self.config["runtimeConfig"].get("admin_port")
             if admin_port is None:
                 admin_port = 3001

{dbos-0.23.0a8 → dbos-0.23.0a10}/dbos/_dbos_config.py

@@ -192,7 +192,11 @@ def load_config(
     data = cast(ConfigFile, data)
     db_connection = load_db_connection()
     if not silent:
-        if data["database"].get("hostname"):
+        if os.getenv("DBOS_DBHOST"):
+            print(
+                "[bold blue]Loading database connection parameters from debug environment variables[/bold blue]"
+            )
+        elif data["database"].get("hostname"):
             print(
                 "[bold blue]Loading database connection parameters from dbos-config.yaml[/bold blue]"
             )
@@ -205,32 +209,62 @@ def load_config(
                 "[bold blue]Using default database connection parameters (localhost)[/bold blue]"
             )
 
+    dbos_dbport: Optional[int] = None
+    dbport_env = os.getenv("DBOS_DBPORT")
+    if dbport_env:
+        try:
+            dbos_dbport = int(dbport_env)
+        except ValueError:
+            pass
+    dbos_dblocalsuffix: Optional[bool] = None
+    dblocalsuffix_env = os.getenv("DBOS_DBLOCALSUFFIX")
+    if dblocalsuffix_env:
+        try:
+            dbos_dblocalsuffix = dblocalsuffix_env.casefold() == "true".casefold()
+        except ValueError:
+            pass
+
     data["database"]["hostname"] = (
-        data["database"].get("hostname") or db_connection.get("hostname") or "localhost"
+        os.getenv("DBOS_DBHOST")
+        or data["database"].get("hostname")
+        or db_connection.get("hostname")
+        or "localhost"
     )
+
     data["database"]["port"] = (
-        data["database"].get("port") or db_connection.get("port") or 5432
+        dbos_dbport or data["database"].get("port") or db_connection.get("port") or 5432
     )
     data["database"]["username"] = (
-        data["database"].get("username") or db_connection.get("username") or "postgres"
+        os.getenv("DBOS_DBUSER")
+        or data["database"].get("username")
+        or db_connection.get("username")
+        or "postgres"
     )
     data["database"]["password"] = (
-        data["database"].get("password")
+        os.getenv("DBOS_DBPASSWORD")
+        or data["database"].get("password")
         or db_connection.get("password")
         or os.environ.get("PGPASSWORD")
         or "dbos"
     )
-    data["database"]["local_suffix"] = (
-        data["database"].get("local_suffix")
-        or db_connection.get("local_suffix")
-        or False
-    )
+
+    local_suffix = False
+    dbcon_local_suffix = db_connection.get("local_suffix")
+    if dbcon_local_suffix is not None:
+        local_suffix = dbcon_local_suffix
+    if data["database"].get("local_suffix") is not None:
+        local_suffix = data["database"].get("local_suffix")
+    if dbos_dblocalsuffix is not None:
+        local_suffix = dbos_dblocalsuffix
+    data["database"]["local_suffix"] = local_suffix
 
     # Configure the DBOS logger
     config_logger(data)
 
     # Check the connectivity to the database and make sure it's properly configured
-    if use_db_wizard:
+    # Note, never use db wizard if the DBOS is running in debug mode (i.e. DBOS_DEBUG_WORKFLOW_ID env var is set)
+    debugWorkflowId = os.getenv("DBOS_DEBUG_WORKFLOW_ID")
+    if use_db_wizard and debugWorkflowId is None:
         data = db_wizard(data, config_file_path)
 
     if "local_suffix" in data["database"] and data["database"]["local_suffix"]:

dbos-0.23.0a10/dbos/_debug.py

@@ -0,0 +1,45 @@
+import re
+import runpy
+import sys
+from typing import Union
+
+from dbos import DBOS
+
+
+class PythonModule:
+    def __init__(self, module_name: str):
+        self.module_name = module_name
+
+
+def debug_workflow(workflow_id: str, entrypoint: Union[str, PythonModule]) -> None:
+    # include the current directory (represented by empty string) in the search path
+    # if it not already included
+    if "" not in sys.path:
+        sys.path.insert(0, "")
+    if isinstance(entrypoint, str):
+        runpy.run_path(entrypoint)
+    elif isinstance(entrypoint, PythonModule):
+        runpy.run_module(entrypoint.module_name)
+    else:
+        raise ValueError("Invalid entrypoint type. Must be a string or PythonModule.")
+
+    DBOS.logger.info(f"Debugging workflow {workflow_id}...")
+    DBOS.launch(debug_mode=True)
+    handle = DBOS.execute_workflow_id(workflow_id)
+    handle.get_result()
+    DBOS.logger.info("Workflow Debugging complete. Exiting process.")
+
+
+def parse_start_command(command: str) -> Union[str, PythonModule]:
+    match = re.match(r"fastapi\s+run\s+(\.?[\w/]+\.py)", command)
+    if match:
+        return match.group(1)
+    match = re.match(r"python3?\s+(\.?[\w/]+\.py)", command)
+    if match:
+        return match.group(1)
+    match = re.match(r"python3?\s+-m\s+([\w\.]+)", command)
+    if match:
+        return PythonModule(match.group(1))
+    raise ValueError(
+        "Invalid command format. Must be 'fastapi run <script>' or 'python <script>' or 'python -m <module>'"
+    )

{dbos-0.23.0a8 → dbos-0.23.0a10}/dbos/_recovery.py

@@ -27,8 +27,9 @@ def startup_recovery_thread(
                     pending_workflow.queue_name
                     and pending_workflow.queue_name != "_dbos_internal_queue"
                 ):
-                    dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
-                    continue
+                    cleared = dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
+                    if cleared:
+                        continue
                 execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
                 pending_workflows.remove(pending_workflow)
         except DBOSWorkflowFunctionNotFoundError:
@@ -56,10 +57,15 @@ def recover_pending_workflows(
                 and pending_workflow.queue_name != "_dbos_internal_queue"
             ):
                 try:
-                    dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
-                    workflow_handles.append(
-                        dbos.retrieve_workflow(pending_workflow.workflow_uuid)
-                    )
+                    cleared = dbos._sys_db.clear_queue_assignment(pending_workflow.workflow_uuid)
+                    if cleared:
+                        workflow_handles.append(
+                            dbos.retrieve_workflow(pending_workflow.workflow_uuid)
+                        )
+                    else:
+                        workflow_handles.append(
+                            execute_workflow_by_id(dbos, pending_workflow.workflow_uuid)
+                        )
                 except Exception as e:
                     dbos.logger.error(e)
             else: