dbos 0.24.0a7__tar.gz → 0.24.0a9__tar.gz

{dbos-0.24.0a7 → dbos-0.24.0a9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.24.0a7
+Version: 0.24.0a9
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.24.0a7 → dbos-0.24.0a9}/dbos/_conductor/conductor.py

@@ -33,7 +33,7 @@ class ConductorWebsocket(threading.Thread):
     def run(self) -> None:
         while not self.evt.is_set():
             try:
-                with connect(self.url) as websocket:
+                with connect(self.url, open_timeout=5) as websocket:
                     self.websocket = websocket
                     while not self.evt.is_set():
                         message = websocket.recv()
@@ -44,6 +44,7 @@ class ConductorWebsocket(threading.Thread):
                             continue
                         base_message = p.BaseMessage.from_json(message)
                         msg_type = base_message.type
+                        error_message = None
                         if msg_type == p.MessageType.EXECUTOR_INFO:
                             info_response = p.ExecutorInfoResponse(
                                 type=p.MessageType.EXECUTOR_INFO,
@@ -61,14 +62,14 @@ class ConductorWebsocket(threading.Thread):
                                     recovery_message.executor_ids
                                 )
                             except Exception as e:
-                                self.dbos.logger.error(
-                                    f"Exception encountered when recovering workflows: {traceback.format_exc()}"
-                                )
+                                error_message = f"Exception encountered when recovering workflows: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
                                 success = False
                             recovery_response = p.RecoveryResponse(
                                 type=p.MessageType.RECOVERY,
                                 request_id=base_message.request_id,
                                 success=success,
+                                error_message=error_message,
                             )
                             websocket.send(recovery_response.to_json())
                         elif msg_type == p.MessageType.CANCEL:
@@ -77,14 +78,14 @@ class ConductorWebsocket(threading.Thread):
                             try:
                                 self.dbos.cancel_workflow(cancel_message.workflow_id)
                             except Exception as e:
-                                self.dbos.logger.error(
-                                    f"Exception encountered when cancelling workflow {cancel_message.workflow_id}: {traceback.format_exc()}"
-                                )
+                                error_message = f"Exception encountered when cancelling workflow {cancel_message.workflow_id}: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
                                 success = False
                             cancel_response = p.CancelResponse(
                                 type=p.MessageType.CANCEL,
                                 request_id=base_message.request_id,
                                 success=success,
+                                error_message=error_message,
                             )
                             websocket.send(cancel_response.to_json())
                         elif msg_type == p.MessageType.RESUME:
@@ -93,14 +94,14 @@ class ConductorWebsocket(threading.Thread):
                             try:
                                 self.dbos.resume_workflow(resume_message.workflow_id)
                             except Exception as e:
-                                self.dbos.logger.error(
-                                    f"Exception encountered when resuming workflow {resume_message.workflow_id}: {traceback.format_exc()}"
-                                )
+                                error_message = f"Exception encountered when resuming workflow {resume_message.workflow_id}: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
                                 success = False
                             resume_response = p.ResumeResponse(
                                 type=p.MessageType.RESUME,
                                 request_id=base_message.request_id,
                                 success=success,
+                                error_message=error_message,
                             )
                             websocket.send(resume_response.to_json())
                         elif msg_type == p.MessageType.RESTART:
@@ -109,14 +110,14 @@ class ConductorWebsocket(threading.Thread):
                             try:
                                 self.dbos.restart_workflow(restart_message.workflow_id)
                             except Exception as e:
-                                self.dbos.logger.error(
-                                    f"Exception encountered when restarting workflow {restart_message.workflow_id}: {traceback.format_exc()}"
-                                )
+                                error_message = f"Exception encountered when restarting workflow {restart_message.workflow_id}: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
                                 success = False
                             restart_response = p.RestartResponse(
                                 type=p.MessageType.RESTART,
                                 request_id=base_message.request_id,
                                 success=success,
+                                error_message=error_message,
                             )
                             websocket.send(restart_response.to_json())
                         elif msg_type == p.MessageType.LIST_WORKFLOWS:
@@ -124,20 +125,26 @@ class ConductorWebsocket(threading.Thread):
                                 message
                             )
                             body = list_workflows_message.body
-                            infos = list_workflows(
-                                self.dbos._sys_db,
-                                workflow_ids=body["workflow_uuids"],
-                                user=body["authenticated_user"],
-                                start_time=body["start_time"],
-                                end_time=body["end_time"],
-                                status=body["status"],
-                                request=False,
-                                app_version=body["application_version"],
-                                name=body["workflow_name"],
-                                limit=body["limit"],
-                                offset=body["offset"],
-                                sort_desc=body["sort_desc"],
-                            )
+                            infos = []
+                            try:
+                                infos = list_workflows(
+                                    self.dbos._sys_db,
+                                    workflow_ids=body["workflow_uuids"],
+                                    user=body["authenticated_user"],
+                                    start_time=body["start_time"],
+                                    end_time=body["end_time"],
+                                    status=body["status"],
+                                    request=False,
+                                    app_version=body["application_version"],
+                                    name=body["workflow_name"],
+                                    limit=body["limit"],
+                                    offset=body["offset"],
+                                    sort_desc=body["sort_desc"],
+                                )
+                            except Exception as e:
+                                error_message = f"Exception encountered when listing workflows: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
+
                             list_workflows_response = p.ListWorkflowsResponse(
                                 type=p.MessageType.LIST_WORKFLOWS,
                                 request_id=base_message.request_id,
@@ -145,6 +152,7 @@ class ConductorWebsocket(threading.Thread):
                                     p.WorkflowsOutput.from_workflow_information(i)
                                     for i in infos
                                 ],
+                                error_message=error_message,
                             )
                             websocket.send(list_workflows_response.to_json())
                         elif msg_type == p.MessageType.LIST_QUEUED_WORKFLOWS:
@@ -152,18 +160,24 @@ class ConductorWebsocket(threading.Thread):
                                 p.ListQueuedWorkflowsRequest.from_json(message)
                             )
                             q_body = list_queued_workflows_message.body
-                            infos = list_queued_workflows(
-                                self.dbos._sys_db,
-                                start_time=q_body["start_time"],
-                                end_time=q_body["end_time"],
-                                status=q_body["status"],
-                                request=False,
-                                name=q_body["workflow_name"],
-                                limit=q_body["limit"],
-                                offset=q_body["offset"],
-                                queue_name=q_body["queue_name"],
-                                sort_desc=q_body["sort_desc"],
-                            )
+                            infos = []
+                            try:
+                                infos = list_queued_workflows(
+                                    self.dbos._sys_db,
+                                    start_time=q_body["start_time"],
+                                    end_time=q_body["end_time"],
+                                    status=q_body["status"],
+                                    request=False,
+                                    name=q_body["workflow_name"],
+                                    limit=q_body["limit"],
+                                    offset=q_body["offset"],
+                                    queue_name=q_body["queue_name"],
+                                    sort_desc=q_body["sort_desc"],
+                                )
+                            except Exception as e:
+                                error_message = f"Exception encountered when listing queued workflows: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
+
                             list_queued_workflows_response = (
                                 p.ListQueuedWorkflowsResponse(
                                     type=p.MessageType.LIST_QUEUED_WORKFLOWS,
@@ -172,6 +186,7 @@ class ConductorWebsocket(threading.Thread):
                                         p.WorkflowsOutput.from_workflow_information(i)
                                         for i in infos
                                     ],
+                                    error_message=error_message,
                                 )
                             )
                             websocket.send(list_queued_workflows_response.to_json())
@@ -179,11 +194,17 @@ class ConductorWebsocket(threading.Thread):
                             get_workflow_message = p.GetWorkflowRequest.from_json(
                                 message
                             )
-                            info = get_workflow(
-                                self.dbos._sys_db,
-                                get_workflow_message.workflow_id,
-                                getRequest=False,
-                            )
+                            info = None
+                            try:
+                                info = get_workflow(
+                                    self.dbos._sys_db,
+                                    get_workflow_message.workflow_id,
+                                    getRequest=False,
+                                )
+                            except Exception as e:
+                                error_message = f"Exception encountered when getting workflow {get_workflow_message.workflow_id}: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
+
                             get_workflow_response = p.GetWorkflowResponse(
                                 type=p.MessageType.GET_WORKFLOW,
                                 request_id=base_message.request_id,
@@ -192,21 +213,29 @@ class ConductorWebsocket(threading.Thread):
                                     if info is not None
                                     else None
                                 ),
+                                error_message=error_message,
                             )
                             websocket.send(get_workflow_response.to_json())
                         elif msg_type == p.MessageType.EXIST_PENDING_WORKFLOWS:
                             exist_pending_workflows_message = (
                                 p.ExistPendingWorkflowsRequest.from_json(message)
                             )
-                            pending_wfs = self.dbos._sys_db.get_pending_workflows(
-                                exist_pending_workflows_message.executor_id,
-                                exist_pending_workflows_message.application_version,
-                            )
+                            pending_wfs = []
+                            try:
+                                pending_wfs = self.dbos._sys_db.get_pending_workflows(
+                                    exist_pending_workflows_message.executor_id,
+                                    exist_pending_workflows_message.application_version,
+                                )
+                            except Exception as e:
+                                error_message = f"Exception encountered when checking for pending workflows: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
+
                             exist_pending_workflows_response = (
                                 p.ExistPendingWorkflowsResponse(
                                     type=p.MessageType.EXIST_PENDING_WORKFLOWS,
                                     request_id=base_message.request_id,
                                     exist=len(pending_wfs) > 0,
+                                    error_message=error_message,
                                 )
                             )
                             websocket.send(exist_pending_workflows_response.to_json())
@@ -214,6 +243,13 @@ class ConductorWebsocket(threading.Thread):
                             self.dbos.logger.warning(
                                 f"Unexpected message type: {msg_type}"
                             )
+                            unknown_message = p.BaseResponse(
+                                request_id=base_message.request_id,
+                                type=msg_type,
+                                error_message="Unknown message type",
+                            )
+                            # Still need to send a response to the conductor
+                            websocket.send(unknown_message.to_json())
             except ConnectionClosedOK:
                 self.dbos.logger.info("Conductor connection terminated")
                 break

{dbos-0.24.0a7 → dbos-0.24.0a9}/dbos/_conductor/protocol.py

@@ -45,6 +45,11 @@ class BaseMessage:
         return json.dumps(dict_data)
 
 
+@dataclass
+class BaseResponse(BaseMessage):
+    error_message: Optional[str] = None
+
+
 @dataclass
 class ExecutorInfoRequest(BaseMessage):
     pass
@@ -54,6 +59,7 @@ class ExecutorInfoRequest(BaseMessage):
 class ExecutorInfoResponse(BaseMessage):
     executor_id: str
     application_version: str
+    error_message: Optional[str] = None
 
 
 @dataclass
@@ -64,6 +70,7 @@ class RecoveryRequest(BaseMessage):
 @dataclass
 class RecoveryResponse(BaseMessage):
     success: bool
+    error_message: Optional[str] = None
 
 
 @dataclass
@@ -74,6 +81,7 @@ class CancelRequest(BaseMessage):
 @dataclass
 class CancelResponse(BaseMessage):
     success: bool
+    error_message: Optional[str] = None
 
 
 @dataclass
@@ -84,6 +92,7 @@ class ResumeRequest(BaseMessage):
 @dataclass
 class ResumeResponse(BaseMessage):
     success: bool
+    error_message: Optional[str] = None
 
 
 @dataclass
@@ -94,6 +103,7 @@ class RestartRequest(BaseMessage):
 @dataclass
 class RestartResponse(BaseMessage):
     success: bool
+    error_message: Optional[str] = None
 
 
 class ListWorkflowsBody(TypedDict):
@@ -165,6 +175,7 @@ class ListWorkflowsRequest(BaseMessage):
 @dataclass
 class ListWorkflowsResponse(BaseMessage):
     output: List[WorkflowsOutput]
+    error_message: Optional[str] = None
 
 
 class ListQueuedWorkflowsBody(TypedDict):
@@ -186,6 +197,7 @@ class ListQueuedWorkflowsRequest(BaseMessage):
 @dataclass
 class ListQueuedWorkflowsResponse(BaseMessage):
     output: List[WorkflowsOutput]
+    error_message: Optional[str] = None
 
 
 @dataclass
@@ -196,6 +208,7 @@ class GetWorkflowRequest(BaseMessage):
 @dataclass
 class GetWorkflowResponse(BaseMessage):
     output: Optional[WorkflowsOutput]
+    error_message: Optional[str] = None
 
 
 @dataclass
@@ -207,3 +220,4 @@ class ExistPendingWorkflowsRequest(BaseMessage):
 @dataclass
 class ExistPendingWorkflowsResponse(BaseMessage):
     exist: bool
+    error_message: Optional[str] = None

{dbos-0.24.0a7 → dbos-0.24.0a9}/dbos/_dbos_config.py

@@ -202,7 +202,7 @@ def translate_dbos_config_to_config_file(config: DBOSConfig) -> ConfigFile:
     return translated_config
 
 
-def _substitute_env_vars(content: str) -> str:
+def _substitute_env_vars(content: str, silent: bool = False) -> str:
     regex = r"\$\{([^}]+)\}"  # Regex to match ${VAR_NAME} style placeholders
 
     def replace_func(match: re.Match[str]) -> str:
@@ -210,7 +210,7 @@ def _substitute_env_vars(content: str) -> str:
         value = os.environ.get(
             var_name, ""
         )  # If the env variable is not set, return an empty string
-        if value == "":
+        if value == "" and not silent:
             dbos_logger.warning(
                 f"Variable {var_name} would be substituted from the process environment into dbos-config.yaml, but is not defined"
             )
@@ -233,7 +233,7 @@ def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
         str: Database URL for the application database
 
     """
-    dbos_config = process_config(data=load_config(config_file_path))
+    dbos_config = load_config(config_file_path, run_process_config=True)
     db_url = URL.create(
         "postgresql+psycopg",
         username=dbos_config["database"]["username"],
@@ -267,7 +267,7 @@ def load_config(
 
     with open(config_file_path, "r") as file:
         content = file.read()
-        substituted_content = _substitute_env_vars(content)
+        substituted_content = _substitute_env_vars(content, silent=silent)
         data = yaml.safe_load(substituted_content)
 
     if not isinstance(data, dict):
@@ -299,7 +299,6 @@ def process_config(
     data: ConfigFile,
     silent: bool = False,
 ) -> ConfigFile:
-
     if "name" not in data:
         raise DBOSInitializationError(f"Configuration must specify an application name")
 
@@ -337,25 +336,9 @@ def process_config(
     if "app_db_name" not in data["database"] or not (data["database"]["app_db_name"]):
         data["database"]["app_db_name"] = _app_name_to_db_name(data["name"])
 
-    # Load the DB connection file. Use its values for missing fields from dbos-config.yaml. Use defaults otherwise.
+    # Load the DB connection file. Use its values for missing connection parameters. Use defaults otherwise.
     db_connection = load_db_connection()
-    if not silent:
-        if os.getenv("DBOS_DBHOST"):
-            print(
-                "[bold blue]Loading database connection parameters from debug environment variables[/bold blue]"
-            )
-        elif data["database"].get("hostname"):
-            print(
-                "[bold blue]Loading database connection parameters from dbos-config.yaml[/bold blue]"
-            )
-        elif db_connection.get("hostname"):
-            print(
-                "[bold blue]Loading database connection parameters from .dbos/db_connection[/bold blue]"
-            )
-        else:
-            print(
-                "[bold blue]Using default database connection parameters (localhost)[/bold blue]"
-            )
+    connection_passed_in = data["database"].get("hostname", None) is not None
 
     dbos_dbport: Optional[int] = None
     dbport_env = os.getenv("DBOS_DBPORT")
@@ -424,6 +407,28 @@ def process_config(
     # Check the connectivity to the database and make sure it's properly configured
     # Note, never use db wizard if the DBOS is running in debug mode (i.e. DBOS_DEBUG_WORKFLOW_ID env var is set)
     debugWorkflowId = os.getenv("DBOS_DEBUG_WORKFLOW_ID")
+
+    # Pretty-print where we've loaded database connection information from, respecting the log level
+    if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
+        d = data["database"]
+        conn_string = f"postgresql://{d['username']}:*****@{d['hostname']}:{d['port']}/{d['app_db_name']}"
+        if os.getenv("DBOS_DBHOST"):
+            print(
+                f"[bold blue]Loading database connection string from debug environment variables: {conn_string}[/bold blue]"
+            )
+        elif connection_passed_in:
+            print(
+                f"[bold blue]Using database connection string: {conn_string}[/bold blue]"
+            )
+        elif db_connection.get("hostname"):
+            print(
+                f"[bold blue]Loading database connection string from .dbos/db_connection: {conn_string}[/bold blue]"
+            )
+        else:
+            print(
+                f"[bold blue]Using default database connection string: {conn_string}[/bold blue]"
+            )
+
     if use_db_wizard and debugWorkflowId is None:
         data = db_wizard(data)
 
@@ -539,7 +544,7 @@ def check_config_consistency(
 ) -> None:
     # First load the config file and check whether it is present
     try:
-        config = load_config(config_file_path)
+        config = load_config(config_file_path, silent=True, run_process_config=False)
     except FileNotFoundError:
         dbos_logger.debug(
             f"No configuration file {config_file_path} found. Skipping consistency check with provided config."

{dbos-0.24.0a7 → dbos-0.24.0a9}/dbos/cli/cli.py

@@ -41,7 +41,7 @@ def _on_windows() -> bool:
     help="Start your DBOS application using the start commands in 'dbos-config.yaml'"
 )
 def start() -> None:
-    config = load_config()
+    config = load_config(run_process_config=False, silent=True)
     start_commands = config["runtimeConfig"]["start"]
     typer.echo("Executing start commands from 'dbos-config.yaml'")
     for command in start_commands:

{dbos-0.24.0a7 → dbos-0.24.0a9}/pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "0.24.0a7"
+version = "0.24.0a9"
 
 [project.license]
 text = "MIT"

{dbos-0.24.0a7 → dbos-0.24.0a9}/tests/test_config.py

@@ -2,13 +2,14 @@
 
 import os
 from unittest.mock import mock_open
+from urllib.parse import quote
 
 import pytest
 import pytest_mock
-from sqlalchemy import event
+from sqlalchemy import URL, event
 
 # Public API
-from dbos import DBOS, load_config
+from dbos import DBOS, get_dbos_database_url, load_config
 from dbos._dbos_config import (
     ConfigFile,
     DBOSConfig,
@@ -1361,3 +1362,53 @@ def test_configured_app_db_connect_timeout():
         pass
 
     dbos.destroy()
+
+
+def test_get_dbos_database_url(mocker):
+    mock_config = """
+        name: "some-app"
+        database:
+          hostname: 'localhost'
+          port: 5432
+          username: 'postgres'
+          password: ${PGPASSWORD}
+          app_db_name: 'some_db'
+    """
+    mocker.patch(
+        "builtins.open", side_effect=generate_mock_open(mock_filename, mock_config)
+    )
+
+    expected_url = URL.create(
+        "postgresql+psycopg",
+        username="postgres",
+        password=os.environ.get("PGPASSWORD"),
+        host="localhost",
+        port=5432,
+        database="some_db",
+    ).render_as_string(hide_password=False)
+    assert get_dbos_database_url() == expected_url
+
+
+def test_get_dbos_database_url_local_suffix(mocker):
+    mock_config = """
+        name: "some-app"
+        database:
+          hostname: 'localhost'
+          port: 5432
+          username: 'postgres'
+          password: ${PGPASSWORD}
+          app_db_name: 'some_db'
+          local_suffix: true
+    """
+    mocker.patch(
+        "builtins.open", side_effect=generate_mock_open(mock_filename, mock_config)
+    )
+    expected_url = URL.create(
+        "postgresql+psycopg",
+        username="postgres",
+        password=os.environ.get("PGPASSWORD"),
+        host="localhost",
+        port=5432,
+        database="some_db_local",
+    ).render_as_string(hide_password=False)
+    assert get_dbos_database_url() == expected_url