dbos 0.28.0a12__py3-none-any.whl → 0.28.0a15__py3-none-any.whl

dbos/_app_db.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, TypedDict
+from typing import Any, Dict, List, Optional, TypedDict
 
 import sqlalchemy as sa
 import sqlalchemy.dialects.postgresql as pg
@@ -7,7 +7,6 @@ from sqlalchemy.exc import DBAPIError
 from sqlalchemy.orm import Session, sessionmaker
 
 from . import _serialization
-from ._dbos_config import DatabaseConfig
 from ._error import DBOSUnexpectedStepError, DBOSWorkflowConflictIDError
 from ._schemas.application_database import ApplicationSchema
 from ._sys_db import StepInfo
@@ -31,67 +30,33 @@ class RecordedResult(TypedDict):
 
 class ApplicationDatabase:
 
-    def __init__(self, database: DatabaseConfig, *, debug_mode: bool = False):
-
-        app_db_name = database["app_db_name"]
+    def __init__(
+        self,
+        *,
+        database_url: str,
+        engine_kwargs: Dict[str, Any],
+        debug_mode: bool = False,
+    ):
+        app_db_url = sa.make_url(database_url).set(drivername="postgresql+psycopg")
 
         # If the application database does not already exist, create it
         if not debug_mode:
-            postgres_db_url = sa.URL.create(
-                "postgresql+psycopg",
-                username=database["username"],
-                password=database["password"],
-                host=database["hostname"],
-                port=database["port"],
-                database="postgres",
+            postgres_db_engine = sa.create_engine(
+                app_db_url.set(database="postgres"),
+                **engine_kwargs,
             )
-            postgres_db_engine = sa.create_engine(postgres_db_url)
             with postgres_db_engine.connect() as conn:
                 conn.execution_options(isolation_level="AUTOCOMMIT")
                 if not conn.execute(
                     sa.text("SELECT 1 FROM pg_database WHERE datname=:db_name"),
-                    parameters={"db_name": app_db_name},
+                    parameters={"db_name": app_db_url.database},
                 ).scalar():
-                    conn.execute(sa.text(f"CREATE DATABASE {app_db_name}"))
+                    conn.execute(sa.text(f"CREATE DATABASE {app_db_url.database}"))
             postgres_db_engine.dispose()
 
-        # Create a connection pool for the application database
-        app_db_url = sa.URL.create(
-            "postgresql+psycopg",
-            username=database["username"],
-            password=database["password"],
-            host=database["hostname"],
-            port=database["port"],
-            database=app_db_name,
-        )
-
-        connect_args = {}
-        if (
-            "connectionTimeoutMillis" in database
-            and database["connectionTimeoutMillis"]
-        ):
-            connect_args["connect_timeout"] = int(
-                database["connectionTimeoutMillis"] / 1000
-            )
-
-        pool_size = database.get("app_db_pool_size")
-        if pool_size is None:
-            pool_size = 20
-
-        engine_kwargs = database.get("db_engine_kwargs")
         if engine_kwargs is None:
             engine_kwargs = {}
 
-        # Respect user-provided values. Otherwise, set defaults.
-        if "pool_size" not in engine_kwargs:
-            engine_kwargs["pool_size"] = pool_size
-        if "max_overflow" not in engine_kwargs:
-            engine_kwargs["max_overflow"] = 0
-        if "pool_timeout" not in engine_kwargs:
-            engine_kwargs["pool_timeout"] = 30
-        if "connect_args" not in engine_kwargs:
-            engine_kwargs["connect_args"] = connect_args
-
         self.engine = sa.create_engine(
             app_db_url,
             **engine_kwargs,

dbos/_client.py

@@ -15,7 +15,6 @@ else:
 
 from dbos import _serialization
 from dbos._dbos import WorkflowHandle, WorkflowHandleAsync
-from dbos._dbos_config import parse_database_url_to_dbconfig
 from dbos._error import DBOSException, DBOSNonExistentWorkflowError
 from dbos._registrations import DEFAULT_MAX_RECOVERY_ATTEMPTS
 from dbos._serialization import WorkflowInputs
@@ -100,11 +99,23 @@ class WorkflowHandleClientAsyncPolling(Generic[R]):
 
 class DBOSClient:
     def __init__(self, database_url: str, *, system_database: Optional[str] = None):
-        db_config = parse_database_url_to_dbconfig(database_url)
-        if system_database is not None:
-            db_config["sys_db_name"] = system_database
-        self._sys_db = SystemDatabase(db_config)
-        self._app_db = ApplicationDatabase(db_config)
+        self._sys_db = SystemDatabase(
+            database_url=database_url,
+            engine_kwargs={
+                "pool_timeout": 30,
+                "max_overflow": 0,
+                "pool_size": 2,
+            },
+            sys_db_name=system_database,
+        )
+        self._app_db = ApplicationDatabase(
+            database_url=database_url,
+            engine_kwargs={
+                "pool_timeout": 30,
+                "max_overflow": 0,
+                "pool_size": 2,
+            },
+        )
         self._db_url = database_url
 
     def destroy(self) -> None:

dbos/_conductor/conductor.py

@@ -2,6 +2,7 @@ import socket
 import threading
 import time
 import traceback
+import uuid
 from importlib.metadata import version
 from typing import TYPE_CHECKING, Optional
 
@@ -9,6 +10,7 @@ from websockets import ConnectionClosed, ConnectionClosedOK, InvalidStatus
 from websockets.sync.client import connect
 from websockets.sync.connection import Connection
 
+from dbos._context import SetWorkflowID
 from dbos._utils import GlobalParams
 from dbos._workflow_commands import (
     get_workflow,
@@ -168,10 +170,11 @@ class ConductorWebsocket(threading.Thread):
                             )
                             websocket.send(resume_response.to_json())
                         elif msg_type == p.MessageType.RESTART:
+                            # TODO: deprecate this message type in favor of Fork
                             restart_message = p.RestartRequest.from_json(message)
                             success = True
                             try:
-                                self.dbos.restart_workflow(restart_message.workflow_id)
+                                self.dbos.fork_workflow(restart_message.workflow_id, 1)
                             except Exception as e:
                                 error_message = f"Exception encountered when restarting workflow {restart_message.workflow_id}: {traceback.format_exc()}"
                                 self.dbos.logger.error(error_message)
@@ -183,6 +186,34 @@ class ConductorWebsocket(threading.Thread):
                                 error_message=error_message,
                             )
                             websocket.send(restart_response.to_json())
+                        elif msg_type == p.MessageType.FORK_WORKFLOW:
+                            fork_message = p.ForkWorkflowRequest.from_json(message)
+                            new_workflow_id = fork_message.body["new_workflow_id"]
+                            if new_workflow_id is None:
+                                new_workflow_id = str(uuid.uuid4())
+                            workflow_id = fork_message.body["workflow_id"]
+                            start_step = fork_message.body["start_step"]
+                            app_version = fork_message.body["application_version"]
+                            try:
+                                with SetWorkflowID(new_workflow_id):
+                                    new_handle = self.dbos.fork_workflow(
+                                        workflow_id,
+                                        start_step,
+                                        application_version=app_version,
+                                    )
+                                new_workflow_id = new_handle.workflow_id
+                            except Exception as e:
+                                error_message = f"Exception encountered when forking workflow {workflow_id} to new workflow {new_workflow_id} on step {start_step}, app version {app_version}: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
+                                new_workflow_id = None
+
+                            fork_response = p.ForkWorkflowResponse(
+                                type=p.MessageType.FORK_WORKFLOW,
+                                request_id=base_message.request_id,
+                                new_workflow_id=new_workflow_id,
+                                error_message=error_message,
+                            )
+                            websocket.send(fork_response.to_json())
                         elif msg_type == p.MessageType.LIST_WORKFLOWS:
                             list_workflows_message = p.ListWorkflowsRequest.from_json(
                                 message

dbos/_conductor/protocol.py

@@ -17,6 +17,7 @@ class MessageType(str, Enum):
     GET_WORKFLOW = "get_workflow"
     EXIST_PENDING_WORKFLOWS = "exist_pending_workflows"
     LIST_STEPS = "list_steps"
+    FORK_WORKFLOW = "fork_workflow"
 
 
 T = TypeVar("T", bound="BaseMessage")
@@ -133,7 +134,6 @@ class WorkflowsOutput:
     AuthenticatedRoles: Optional[str]
     Input: Optional[str]
     Output: Optional[str]
-    Request: Optional[str]
     Error: Optional[str]
     CreatedAt: Optional[str]
     UpdatedAt: Optional[str]
@@ -167,7 +167,6 @@ class WorkflowsOutput:
             AuthenticatedRoles=roles_str,
             Input=inputs_str,
             Output=outputs_str,
-            Request=request_str,
             Error=error_str,
             CreatedAt=created_at_str,
             UpdatedAt=updated_at_str,
@@ -263,3 +262,21 @@ class ListStepsRequest(BaseMessage):
 class ListStepsResponse(BaseMessage):
     output: Optional[List[WorkflowSteps]]
     error_message: Optional[str] = None
+
+
+class ForkWorkflowBody(TypedDict):
+    workflow_id: str
+    start_step: int
+    application_version: Optional[str]
+    new_workflow_id: Optional[str]
+
+
+@dataclass
+class ForkWorkflowRequest(BaseMessage):
+    body: ForkWorkflowBody
+
+
+@dataclass
+class ForkWorkflowResponse(BaseMessage):
+    new_workflow_id: Optional[str]
+    error_message: Optional[str] = None

dbos/_dbos.py

@@ -24,8 +24,6 @@ from typing import (
     Tuple,
     Type,
     TypeVar,
-    Union,
-    cast,
 )
 
 from opentelemetry.trace import Span
@@ -64,7 +62,6 @@ from ._registrations import (
 )
 from ._roles import default_required_roles, required_roles
 from ._scheduler import ScheduledWorkflow, scheduled
-from ._schemas.system_database import SystemSchema
 from ._sys_db import StepInfo, SystemDatabase, WorkflowStatus, reset_system_database
 from ._tracer import DBOSTracer, dbos_tracer
 
@@ -73,7 +70,7 @@ if TYPE_CHECKING:
     from ._kafka import _KafkaConsumerWorkflow
     from flask import Flask
 
-from sqlalchemy import URL
+from sqlalchemy import make_url
 from sqlalchemy.orm import Session
 
 if sys.version_info < (3, 10):
@@ -418,11 +415,19 @@ class DBOS:
             dbos_logger.info(f"Application version: {GlobalParams.app_version}")
             self._executor_field = ThreadPoolExecutor(max_workers=64)
             self._background_event_loop.start()
+            assert self._config["database_url"] is not None
+            assert self._config["database"]["sys_db_engine_kwargs"] is not None
             self._sys_db_field = SystemDatabase(
-                self._config["database"], debug_mode=debug_mode
+                database_url=self._config["database_url"],
+                engine_kwargs=self._config["database"]["sys_db_engine_kwargs"],
+                sys_db_name=self._config["database"]["sys_db_name"],
+                debug_mode=debug_mode,
             )
+            assert self._config["database"]["db_engine_kwargs"] is not None
             self._app_db_field = ApplicationDatabase(
-                self._config["database"], debug_mode=debug_mode
+                database_url=self._config["database_url"],
+                engine_kwargs=self._config["database"]["db_engine_kwargs"],
+                debug_mode=debug_mode,
             )
 
             if debug_mode:
@@ -530,21 +535,13 @@ class DBOS:
             not self._launched
         ), "The system database cannot be reset after DBOS is launched. Resetting the system database is a destructive operation that should only be used in a test environment."
 
-        sysdb_name = (
-            self._config["database"]["sys_db_name"]
-            if "sys_db_name" in self._config["database"]
-            and self._config["database"]["sys_db_name"]
-            else self._config["database"]["app_db_name"] + SystemSchema.sysdb_suffix
-        )
-        postgres_db_url = URL.create(
-            "postgresql+psycopg",
-            username=self._config["database"]["username"],
-            password=self._config["database"]["password"],
-            host=self._config["database"]["hostname"],
-            port=self._config["database"]["port"],
-            database="postgres",
-        )
-        reset_system_database(postgres_db_url, sysdb_name)
+        sysdb_name = self._config["database"]["sys_db_name"]
+        assert sysdb_name is not None
+
+        assert self._config["database_url"] is not None
+        pg_db_url = make_url(self._config["database_url"]).set(database="postgres")
+
+        reset_system_database(pg_db_url, sysdb_name)
 
     def _destroy(self) -> None:
         self._initialized = False

dbos/_dbos_config.py

@@ -11,6 +11,7 @@ from sqlalchemy import make_url
 
 from ._error import DBOSInitializationError
 from ._logger import dbos_logger
+from ._schemas.system_database import SystemSchema
 
 DBOS_CONFIG_PATH = "dbos-config.yaml"
 
@@ -22,7 +23,6 @@ class DBOSConfig(TypedDict, total=False):
     Attributes:
         name (str): Application name
         database_url (str): Database connection string
-        app_db_pool_size (int): Application database pool size
         sys_db_name (str): System database name
         sys_db_pool_size (int): System database pool size
         db_engine_kwargs (Dict[str, Any]): SQLAlchemy engine kwargs (See https://docs.sqlalchemy.org/en/20/core/engines.html#sqlalchemy.create_engine)
@@ -35,7 +35,6 @@ class DBOSConfig(TypedDict, total=False):
 
     name: str
     database_url: Optional[str]
-    app_db_pool_size: Optional[int]
     sys_db_name: Optional[str]
     sys_db_pool_size: Optional[int]
     db_engine_kwargs: Optional[Dict[str, Any]]
@@ -57,49 +56,22 @@ class DatabaseConfig(TypedDict, total=False):
     """
     Internal data structure containing the DBOS database configuration.
     Attributes:
-        app_db_pool_size (int): Application database pool size
         sys_db_name (str): System database name
         sys_db_pool_size (int): System database pool size
         db_engine_kwargs (Dict[str, Any]): SQLAlchemy engine kwargs
         migrate (List[str]): Migration commands to run on startup
     """
 
-    hostname: str  # Will be removed in a future version
-    port: int  # Will be removed in a future version
-    username: str  # Will be removed in a future version
-    password: str  # Will be removed in a future version
-    connectionTimeoutMillis: Optional[int]  # Will be removed in a future version
-    app_db_name: str  # Will be removed in a future version
-    app_db_pool_size: Optional[int]
     sys_db_name: Optional[str]
-    sys_db_pool_size: Optional[int]
+    sys_db_pool_size: Optional[
+        int
+    ]  # For internal use, will be removed in a future version
     db_engine_kwargs: Optional[Dict[str, Any]]
-    ssl: Optional[bool]  # Will be removed in a future version
-    ssl_ca: Optional[str]  # Will be removed in a future version
+    sys_db_engine_kwargs: Optional[Dict[str, Any]]
     migrate: Optional[List[str]]
     rollback: Optional[List[str]]  # Will be removed in a future version
 
 
-def parse_database_url_to_dbconfig(database_url: str) -> DatabaseConfig:
-    db_url = make_url(database_url)
-    db_config = {
-        "hostname": db_url.host,
-        "port": db_url.port or 5432,
-        "username": db_url.username,
-        "password": db_url.password,
-        "app_db_name": db_url.database,
-    }
-    for key, value in db_url.query.items():
-        str_value = value[0] if isinstance(value, tuple) else value
-        if key == "connect_timeout":
-            db_config["connectionTimeoutMillis"] = int(str_value) * 1000
-        elif key == "sslmode":
-            db_config["ssl"] = str_value == "require"
-        elif key == "sslrootcert":
-            db_config["ssl_ca"] = str_value
-    return cast(DatabaseConfig, db_config)
-
-
 class OTLPExporterConfig(TypedDict, total=False):
     logsEndpoint: Optional[List[str]]
     tracesEndpoint: Optional[List[str]]
@@ -150,13 +122,8 @@ def translate_dbos_config_to_config_file(config: DBOSConfig) -> ConfigFile:
 
     # Database config
     db_config: DatabaseConfig = {}
-    database_url = config.get("database_url")
-    if database_url:
-        db_config = parse_database_url_to_dbconfig(database_url)
     if "sys_db_name" in config:
         db_config["sys_db_name"] = config.get("sys_db_name")
-    if "app_db_pool_size" in config:
-        db_config["app_db_pool_size"] = config.get("app_db_pool_size")
     if "sys_db_pool_size" in config:
         db_config["sys_db_pool_size"] = config.get("sys_db_pool_size")
     if "db_engine_kwargs" in config:
@@ -164,6 +131,9 @@ def translate_dbos_config_to_config_file(config: DBOSConfig) -> ConfigFile:
     if db_config:
         translated_config["database"] = db_config
 
+    if "database_url" in config:
+        translated_config["database_url"] = config.get("database_url")
+
     # Runtime config
     translated_config["runtimeConfig"] = {"run_admin_server": True}
     if "admin_port" in config:
@@ -306,6 +276,24 @@ def process_config(
     data: ConfigFile,
     silent: bool = False,
 ) -> ConfigFile:
+    """
+    If a database_url is provided, pass it as is in the config.
+
+    Else, build a database_url from defaults.
+
+    Also build SQL Alchemy "kwargs" base on user input + defaults.
+    Specifically, db_engine_kwargs takes precedence over app_db_pool_size
+
+    In debug mode, apply overrides from DBOS_DBHOST, DBOS_DBPORT, DBOS_DBUSER, and DBOS_DBPASSWORD.
+
+    Default configuration:
+        - Hostname: localhost
+        - Port: 5432
+        - Username: postgres
+        - Password: $PGPASSWORD
+        - Database name: transformed application name.
+    """
+
     if "name" not in data:
         raise DBOSInitializationError(f"Configuration must specify an application name")
 
@@ -323,55 +311,7 @@ def process_config(
     if logs.get("logLevel") is None:
         logs["logLevel"] = "INFO"
 
-    if "database" not in data:
-        data["database"] = {}
-
-    # database_url takes precedence over database config, but we need to preserve rollback and migrate if they exist
-    migrate = data["database"].get("migrate", False)
-    rollback = data["database"].get("rollback", False)
-    if data.get("database_url"):
-        dbconfig = parse_database_url_to_dbconfig(cast(str, data["database_url"]))
-        if migrate:
-            dbconfig["migrate"] = cast(List[str], migrate)
-        if rollback:
-            dbconfig["rollback"] = cast(List[str], rollback)
-        data["database"] = dbconfig
-
-    if "app_db_name" not in data["database"] or not (data["database"]["app_db_name"]):
-        data["database"]["app_db_name"] = _app_name_to_db_name(data["name"])
-
-    connection_passed_in = data["database"].get("hostname", None) is not None
-
-    dbos_dbport: Optional[int] = None
-    dbport_env = os.getenv("DBOS_DBPORT")
-    if dbport_env:
-        try:
-            dbos_dbport = int(dbport_env)
-        except ValueError:
-            pass
-
-    data["database"]["hostname"] = (
-        os.getenv("DBOS_DBHOST") or data["database"].get("hostname") or "localhost"
-    )
-
-    data["database"]["port"] = dbos_dbport or data["database"].get("port") or 5432
-    data["database"]["username"] = (
-        os.getenv("DBOS_DBUSER") or data["database"].get("username") or "postgres"
-    )
-    data["database"]["password"] = (
-        os.getenv("DBOS_DBPASSWORD")
-        or data["database"].get("password")
-        or os.environ.get("PGPASSWORD")
-        or "dbos"
-    )
-
-    if not data["database"].get("app_db_pool_size"):
-        data["database"]["app_db_pool_size"] = 20
-    if not data["database"].get("sys_db_pool_size"):
-        data["database"]["sys_db_pool_size"] = 20
-    if not data["database"].get("connectionTimeoutMillis"):
-        data["database"]["connectionTimeoutMillis"] = 10000
-
+    # Handle admin server config
     if not data.get("runtimeConfig"):
         data["runtimeConfig"] = {
             "run_admin_server": True,
@@ -379,27 +319,118 @@ def process_config(
     elif "run_admin_server" not in data["runtimeConfig"]:
         data["runtimeConfig"]["run_admin_server"] = True
 
+    isDebugMode = os.getenv("DBOS_DBHOST") is not None
+
+    # Ensure database dict exists
+    data.setdefault("database", {})
+
+    # Database URL resolution
+    connect_timeout = None
+    if data.get("database_url") is not None and data["database_url"] != "":
+        # Parse the db string and check required fields
+        assert data["database_url"] is not None
+        url = make_url(data["database_url"])
+        required_fields = [
+            ("username", "Username must be specified in the connection URL"),
+            ("password", "Password must be specified in the connection URL"),
+            ("host", "Host must be specified in the connection URL"),
+            ("database", "Database name must be specified in the connection URL"),
+        ]
+        for field_name, error_message in required_fields:
+            field_value = getattr(url, field_name, None)
+            if not field_value:
+                raise DBOSInitializationError(error_message)
+
+        if not data["database"].get("sys_db_name"):
+            assert url.database is not None
+            data["database"]["sys_db_name"] = url.database + SystemSchema.sysdb_suffix
+
+        # Gather connect_timeout from the URL if provided. It should be used in engine kwargs if not provided there (instead of our default)
+        connect_timeout_str = url.query.get("connect_timeout")
+        if connect_timeout_str is not None:
+            assert isinstance(
+                connect_timeout_str, str
+            ), "connect_timeout must be a string and defined once in the URL"
+            if connect_timeout_str.isdigit():
+                connect_timeout = int(connect_timeout_str)
+
+        # In debug mode perform env vars overrides
+        if isDebugMode:
+            # Override the username, password, host, and port
+            port_str = os.getenv("DBOS_DBPORT")
+            port = (
+                int(port_str)
+                if port_str is not None and port_str.isdigit()
+                else url.port
+            )
+            data["database_url"] = url.set(
+                username=os.getenv("DBOS_DBUSER", url.username),
+                password=os.getenv("DBOS_DBPASSWORD", url.password),
+                host=os.getenv("DBOS_DBHOST", url.host),
+                port=port,
+            ).render_as_string(hide_password=False)
+    else:
+        _app_db_name = _app_name_to_db_name(data["name"])
+        _password = os.environ.get("PGPASSWORD", "dbos")
+        data["database_url"] = (
+            f"postgres://postgres:{_password}@localhost:5432/{_app_db_name}?connect_timeout=10&sslmode=prefer"
+        )
+        if not data["database"].get("sys_db_name"):
+            data["database"]["sys_db_name"] = _app_db_name + SystemSchema.sysdb_suffix
+        assert data["database_url"] is not None
+
+    configure_db_engine_parameters(data["database"], connect_timeout=connect_timeout)
+
     # Pretty-print where we've loaded database connection information from, respecting the log level
     if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
-        d = data["database"]
-        conn_string = f"postgresql://{d['username']}:*****@{d['hostname']}:{d['port']}/{d['app_db_name']}"
-        if os.getenv("DBOS_DBHOST"):
-            print(
-                f"[bold blue]Loading database connection string from debug environment variables: {conn_string}[/bold blue]"
-            )
-        elif connection_passed_in:
-            print(
-                f"[bold blue]Using database connection string: {conn_string}[/bold blue]"
-            )
-        else:
-            print(
-                f"[bold blue]Using default database connection string: {conn_string}[/bold blue]"
-            )
+        log_url = make_url(data["database_url"]).render_as_string(hide_password=True)
+        print(f"[bold blue]Using database connection string: {log_url}[/bold blue]")
 
     # Return data as ConfigFile type
     return data
 
 
+def configure_db_engine_parameters(
+    data: DatabaseConfig, connect_timeout: Optional[int] = None
+) -> None:
+    """
+    Configure SQLAlchemy engine parameters for both user and system databases.
+
+    If provided, sys_db_pool_size will take precedence over user_kwargs for the system db engine.
+
+    Args:
+        data: Configuration dictionary containing database settings
+    """
+
+    # Configure user database engine parameters
+    app_engine_kwargs: dict[str, Any] = {
+        "pool_timeout": 30,
+        "max_overflow": 0,
+        "pool_size": 20,
+    }
+    # If user-provided kwargs are present, use them instead
+    user_kwargs = data.get("db_engine_kwargs")
+    if user_kwargs is not None:
+        app_engine_kwargs.update(user_kwargs)
+
+    # If user-provided kwargs do not contain connect_timeout, check if their URL did (this function connect_timeout parameter).
+    # Else default to 10
+    if "connect_args" not in app_engine_kwargs:
+        app_engine_kwargs["connect_args"] = {}
+    if "connect_timeout" not in app_engine_kwargs["connect_args"]:
+        app_engine_kwargs["connect_args"]["connect_timeout"] = (
+            connect_timeout if connect_timeout else 10
+        )
+
+    # Configure system database engine parameters. User-provided sys_db_pool_size takes precedence
+    system_engine_kwargs = app_engine_kwargs.copy()
+    if data.get("sys_db_pool_size") is not None:
+        system_engine_kwargs["pool_size"] = data["sys_db_pool_size"]
+
+    data["db_engine_kwargs"] = app_engine_kwargs
+    data["sys_db_engine_kwargs"] = system_engine_kwargs
+
+
 def _is_valid_app_name(name: str) -> bool:
     name_len = len(name)
     if name_len < 3 or name_len > 30:
@@ -409,7 +440,7 @@ def _is_valid_app_name(name: str) -> bool:
 
 
 def _app_name_to_db_name(app_name: str) -> str:
-    name = app_name.replace("-", "_")
+    name = app_name.replace("-", "_").replace(" ", "_").lower()
     return name if not name[0].isdigit() else f"_{name}"
 
 
@@ -421,7 +452,7 @@ def set_env_vars(config: ConfigFile) -> None:
 
 def overwrite_config(provided_config: ConfigFile) -> ConfigFile:
     # Load the DBOS configuration file and force the use of:
-    # 1. The database connection parameters (sub the file data to the provided config)
+    # 1. The database url provided by DBOS_DATABASE_URL
     # 2. OTLP traces endpoints (add the config data to the provided config)
     # 3. Use the application name from the file. This is a defensive measure to ensure the application name is whatever it was registered with in the cloud
     # 4. Remove admin_port is provided in code
@@ -436,22 +467,19 @@ def overwrite_config(provided_config: ConfigFile) -> ConfigFile:
     # Name
     provided_config["name"] = config_from_file["name"]
 
-    # Database config. Note we disregard a potential database_url in config_from_file because it is not expected from DBOS Cloud
+    # Database config. Expects DBOS_DATABASE_URL to be set
     if "database" not in provided_config:
         provided_config["database"] = {}
-    provided_config["database"]["hostname"] = config_from_file["database"]["hostname"]
-    provided_config["database"]["port"] = config_from_file["database"]["port"]
-    provided_config["database"]["username"] = config_from_file["database"]["username"]
-    provided_config["database"]["password"] = config_from_file["database"]["password"]
-    provided_config["database"]["app_db_name"] = config_from_file["database"][
-        "app_db_name"
-    ]
     provided_config["database"]["sys_db_name"] = config_from_file["database"][
         "sys_db_name"
     ]
-    provided_config["database"]["ssl"] = config_from_file["database"]["ssl"]
-    if "ssl_ca" in config_from_file["database"]:
-        provided_config["database"]["ssl_ca"] = config_from_file["database"]["ssl_ca"]
+
+    db_url = os.environ.get("DBOS_DATABASE_URL")
+    if db_url is None:
+        raise DBOSInitializationError(
+            "DBOS_DATABASE_URL environment variable is not set. This is required to connect to the database."
+        )
+    provided_config["database_url"] = db_url
 
     # Telemetry config
     if "telemetry" not in provided_config or provided_config["telemetry"] is None:

dbos/_sys_db.py

@@ -5,7 +5,6 @@ import os
 import re
 import threading
 import time
-import uuid
 from enum import Enum
 from typing import (
     TYPE_CHECKING,
@@ -28,11 +27,10 @@ from alembic.config import Config
 from sqlalchemy.exc import DBAPIError
 from sqlalchemy.sql import func
 
-from dbos._utils import INTERNAL_QUEUE_NAME, GlobalParams
+from dbos._utils import INTERNAL_QUEUE_NAME
 
 from . import _serialization
 from ._context import get_local_dbos_context
-from ._dbos_config import ConfigFile, DatabaseConfig
 from ._error import (
     DBOSConflictingWorkflowError,
     DBOSDeadLetterQueueError,
@@ -227,26 +225,28 @@ _dbos_null_topic = "__null__topic__"
 
 class SystemDatabase:
 
-    def __init__(self, database: DatabaseConfig, *, debug_mode: bool = False):
-        sysdb_name = (
-            database["sys_db_name"]
-            if "sys_db_name" in database and database["sys_db_name"]
-            else database["app_db_name"] + SystemSchema.sysdb_suffix
-        )
+    def __init__(
+        self,
+        *,
+        database_url: str,
+        engine_kwargs: Dict[str, Any],
+        sys_db_name: Optional[str] = None,
+        debug_mode: bool = False,
+    ):
+        # Set driver
+        system_db_url = sa.make_url(database_url).set(drivername="postgresql+psycopg")
+        # Resolve system database name
+        sysdb_name = sys_db_name
+        if not sysdb_name:
+            assert system_db_url.database is not None
+            sysdb_name = system_db_url.database + SystemSchema.sysdb_suffix
+        system_db_url = system_db_url.set(database=sysdb_name)
 
         if not debug_mode:
             # If the system database does not already exist, create it
-            postgres_db_url = sa.URL.create(
-                "postgresql+psycopg",
-                username=database["username"],
-                password=database["password"],
-                host=database["hostname"],
-                port=database["port"],
-                database="postgres",
-                # fills the "application_name" column in pg_stat_activity
-                query={"application_name": f"dbos_transact_{GlobalParams.executor_id}"},
+            engine = sa.create_engine(
+                system_db_url.set(database="postgres"), **engine_kwargs
             )
-            engine = sa.create_engine(postgres_db_url)
             with engine.connect() as conn:
                 conn.execution_options(isolation_level="AUTOCOMMIT")
                 if not conn.execute(
@@ -257,36 +257,6 @@ class SystemDatabase:
                     conn.execute(sa.text(f"CREATE DATABASE {sysdb_name}"))
             engine.dispose()
 
-        system_db_url = sa.URL.create(
-            "postgresql+psycopg",
-            username=database["username"],
-            password=database["password"],
-            host=database["hostname"],
-            port=database["port"],
-            database=sysdb_name,
-            # fills the "application_name" column in pg_stat_activity
-            query={"application_name": f"dbos_transact_{GlobalParams.executor_id}"},
-        )
-
-        # Create a connection pool for the system database
-        pool_size = database.get("sys_db_pool_size")
-        if pool_size is None:
-            pool_size = 20
-
-        engine_kwargs = database.get("db_engine_kwargs")
-        if engine_kwargs is None:
-            engine_kwargs = {}
-
-        # Respect user-provided values. Otherwise, set defaults.
-        if "pool_size" not in engine_kwargs:
-            engine_kwargs["pool_size"] = pool_size
-        if "max_overflow" not in engine_kwargs:
-            engine_kwargs["max_overflow"] = 0
-        if "pool_timeout" not in engine_kwargs:
-            engine_kwargs["pool_timeout"] = 30
-        if "connect_args" not in engine_kwargs:
-            engine_kwargs["connect_args"] = {"connect_timeout": 10}
-
         self.engine = sa.create_engine(
             system_db_url,
             **engine_kwargs,
@@ -1958,7 +1928,10 @@ class SystemDatabase:
 def reset_system_database(postgres_db_url: sa.URL, sysdb_name: str) -> None:
     try:
         # Connect to postgres default database
-        engine = sa.create_engine(postgres_db_url)
+        engine = sa.create_engine(
+            postgres_db_url.set(drivername="postgresql+psycopg"),
+            connect_args={"connect_timeout": 10},
+        )
 
         with engine.connect() as conn:
             # Set autocommit required for database dropping

dbos/cli/cli.py

@@ -27,7 +27,7 @@ from ..cli._github_init import create_template_from_github
 from ._template_init import copy_template, get_project_name, get_templates_directory
 
 
-def start_client(db_url: Optional[str] = None) -> DBOSClient:
+def _get_db_url(db_url: Optional[str]) -> str:
     database_url = db_url
     if database_url is None:
         database_url = os.getenv("DBOS_DATABASE_URL")
@@ -35,7 +35,11 @@ def start_client(db_url: Optional[str] = None) -> DBOSClient:
         raise ValueError(
             "Missing database URL: please set it using the --db-url flag or the DBOS_DATABASE_URL environment variable."
         )
+    return database_url
+
 
+def start_client(db_url: Optional[str] = None) -> DBOSClient:
+    database_url = _get_db_url(db_url)
     return DBOSClient(database_url=database_url)
 
 
@@ -206,14 +210,30 @@ def init(
 @app.command(
     help="Run your database schema migrations using the migration commands in 'dbos-config.yaml'"
 )
-def migrate() -> None:
-    config = load_config()
-    if not config["database"]["password"]:
-        typer.echo(
-            "DBOS configuration does not contain database password, please check your config file and retry!"
-        )
-        raise typer.Exit(code=1)
-    app_db_name = config["database"]["app_db_name"]
+def migrate(
+    db_url: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--db-url",
+            "-D",
+            help="Your DBOS application database URL",
+        ),
+    ] = None,
+    sys_db_name: Annotated[
+        typing.Optional[str],
+        typer.Option(
+            "--sys-db-name",
+            "-s",
+            help="Specify the name of the system database to reset",
+        ),
+    ] = None,
+) -> None:
+    config = load_config(run_process_config=False, silent=True)
+    connection_string = _get_db_url(db_url)
+    app_db_name = sa.make_url(connection_string).database
+    assert app_db_name is not None, "Database name is required in URL"
+    if sys_db_name is None:
+        sys_db_name = app_db_name + SystemSchema.sysdb_suffix
 
     typer.echo(f"Starting schema migration for database {app_db_name}")
 
@@ -221,8 +241,23 @@ def migrate() -> None:
     app_db = None
     sys_db = None
     try:
-        sys_db = SystemDatabase(config["database"])
-        app_db = ApplicationDatabase(config["database"])
+        sys_db = SystemDatabase(
+            database_url=connection_string,
+            engine_kwargs={
+                "pool_timeout": 30,
+                "max_overflow": 0,
+                "pool_size": 2,
+            },
+            sys_db_name=sys_db_name,
+        )
+        app_db = ApplicationDatabase(
+            database_url=connection_string,
+            engine_kwargs={
+                "pool_timeout": 30,
+                "max_overflow": 0,
+                "pool_size": 2,
+            },
+        )
     except Exception as e:
         typer.echo(f"DBOS system schema migration failed: {e}")
     finally:
@@ -234,6 +269,9 @@ def migrate() -> None:
     # Next, run any custom migration commands specified in the configuration
     typer.echo("Executing migration commands from 'dbos-config.yaml'")
     try:
+        # handle the case where the user has not specified migrations commands
+        if "database" not in config:
+            config["database"] = {}
         migrate_commands = (
             config["database"]["migrate"]
             if "migrate" in config["database"] and config["database"]["migrate"]
@@ -283,17 +321,21 @@ def reset(
             typer.echo("Operation cancelled.")
             raise typer.Exit()
     try:
-        client = start_client(db_url=db_url)
-        pg_db_url = sa.make_url(client._db_url).set(drivername="postgresql+psycopg")
+        # Make a SA url out of the user-provided URL and verify a database name is present
+        database_url = _get_db_url(db_url)
+        pg_db_url = sa.make_url(database_url)
         assert (
             pg_db_url.database is not None
         ), f"Database name is required in URL: {pg_db_url.render_as_string(hide_password=True)}"
+        # Resolve system database name
         sysdb_name = (
             sys_db_name
             if sys_db_name
             else (pg_db_url.database + SystemSchema.sysdb_suffix)
         )
-        reset_system_database(pg_db_url.set(database="postgres"), sysdb_name)
+        reset_system_database(
+            postgres_db_url=pg_db_url.set(database="postgres"), sysdb_name=sysdb_name
+        )
     except sa.exc.SQLAlchemyError as e:
         typer.echo(f"Error resetting system database: {str(e)}")
         return

dbos/dbos-config.schema.json

@@ -70,16 +70,6 @@
             "description": "If using SSL/TLS to securely connect to a database, path to an SSL root certificate file. DEPRECATED: Use database_url instead",
             "deprecated": true
           },
-          "app_db_client": {
-            "type": "string",
-            "description": "Specify the database client to use to connect to the application database",
-            "enum": [
-              "pg-node",
-              "prisma",
-              "typeorm",
-              "knex"
-            ]
-          },
           "migrate": {
             "type": "array",
             "description": "Specify a list of user DB migration commands to run"

{dbos-0.28.0a12.dist-info → dbos-0.28.0a15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.28.0a12
+Version: 0.28.0a15
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.28.0a12.dist-info → dbos-0.28.0a15.dist-info}/RECORD

@@ -1,20 +1,20 @@
-dbos-0.28.0a12.dist-info/METADATA,sha256=nxjIfoX9CdMPTCpI-L-iIAHRyTjDDbjJKNbwI6gtiqc,13269
-dbos-0.28.0a12.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
-dbos-0.28.0a12.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
-dbos-0.28.0a12.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
+dbos-0.28.0a15.dist-info/METADATA,sha256=zpjEzcFWmk2kxievyVaZBkvakcjpprw6H8842nJMuPE,13269
+dbos-0.28.0a15.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
+dbos-0.28.0a15.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
+dbos-0.28.0a15.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
 dbos/__init__.py,sha256=NssPCubaBxdiKarOWa-wViz1hdJSkmBGcpLX_gQ4NeA,891
 dbos/__main__.py,sha256=G7Exn-MhGrVJVDbgNlpzhfh8WMX_72t3_oJaFT9Lmt8,653
 dbos/_admin_server.py,sha256=A_28_nJ1nBBYDmCxtklJR9O2v14JRMtD1rAo_D4y8Kc,9764
-dbos/_app_db.py,sha256=3j8_5-MlSDY0otLRszFE-GfenU6JC20fcfSL-drSNYk,11800
+dbos/_app_db.py,sha256=56jqU0oxcIkMscOg6DxC8dZ0uEJwG7iMu3aXYs2u66k,10428
 dbos/_classproperty.py,sha256=f0X-_BySzn3yFDRKB2JpCbLYQ9tLwt1XftfshvY7CBs,626
-dbos/_client.py,sha256=PPktnvaLfl8Fj27YoOjy_kfgeQm_c3QQZztPyts5GZo,13924
-dbos/_conductor/conductor.py,sha256=qP3fTe74gdaIN9SU9nCn2B5GYMBnhQ6shEhGezEcSPg,19693
-dbos/_conductor/protocol.py,sha256=jwX8ZjmAIlXu1vw9R3b5PfHSNdwofeYOKj8rkfAFVg0,6630
+dbos/_client.py,sha256=aPOAVWsH7uovSIsgJRamksra9y9-BVW1Jw4Gg5WjuZA,14114
+dbos/_conductor/conductor.py,sha256=o0IaZjwnZ2TOyHeP2H4iSX6UnXLXQ4uODvWAKD9hHMs,21703
+dbos/_conductor/protocol.py,sha256=wgOFZxmS81bv0WCB9dAyg0s6QzldpzVKQDoSPeaX0Ws,6967
 dbos/_context.py,sha256=Ly1CXF1nWxICQgIpDZSaONGlz1yERBs63gqmR-yqCzM,24476
 dbos/_core.py,sha256=UDpSgRA9m_YuViNXR9tVgNFLC-zxKZPxjlkj2a-Kj00,48317
 dbos/_croniter.py,sha256=XHAyUyibs_59sJQfSNWkP7rqQY6_XrlfuuCxk4jYqek,47559
-dbos/_dbos.py,sha256=tPpf_X9oLtpPICryXDMc589XuFeJtZ4cIq7vmsNDcls,46220
-dbos/_dbos_config.py,sha256=pgzE5UP3LV_nhwIV4d1-W9q2LumQb1kla75xdx9mBcI,20136
+dbos/_dbos.py,sha256=7fQPKfaePD3HwxSjBhziJcVd2heLKefe92skgmuHr34,46275
+dbos/_dbos_config.py,sha256=IufNrIC-M2xSNTXyT_KXlEdfB3j03pPLv_nE0fEq4_U,20955
 dbos/_debug.py,sha256=MNlQVZ6TscGCRQeEEL0VE8Uignvr6dPeDDDefS3xgIE,1823
 dbos/_docker_pg_helper.py,sha256=tLJXWqZ4S-ExcaPnxg_i6cVxL6ZxrYlZjaGsklY-s2I,6115
 dbos/_error.py,sha256=EN4eVBjMT3k7O7hfqJl6mIf4sxWPsiAOM086yhcGH_g,8012
@@ -47,7 +47,7 @@ dbos/_schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_schemas/application_database.py,sha256=SypAS9l9EsaBHFn9FR8jmnqt01M74d9AF1AMa4m2hhI,1040
 dbos/_schemas/system_database.py,sha256=3Z0L72bOgHnusK1hBaETWU9RfiLBP0QnS-fdu41i0yY,5835
 dbos/_serialization.py,sha256=bWuwhXSQcGmiazvhJHA5gwhrRWxtmFmcCFQSDJnqqkU,3666
-dbos/_sys_db.py,sha256=lceVny4RzQP7DeEJ8CpQ0crg8cJmuTznUsiGmv1oM-E,82486
+dbos/_sys_db.py,sha256=VJ-wLjukSVMXx2EGyc3ZTL9GFDpHHdFU4iXRSFobrRU,81229
 dbos/_templates/dbos-db-starter/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
 dbos/_templates/dbos-db-starter/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_templates/dbos-db-starter/__package/main.py.dbos,sha256=aQnBPSSQpkB8ERfhf7gB7P9tsU6OPKhZscfeh0yiaD8,2702
@@ -63,8 +63,8 @@ dbos/_utils.py,sha256=UbpMYRBSyvJqdXeWAnfSw8xXM1R1mfnyl1oTunhEjJM,513
 dbos/_workflow_commands.py,sha256=2E8FRUv_nLYkpBTwfhh_ELhySYpMrm8qGB9J44g6DSE,3872
 dbos/cli/_github_init.py,sha256=Y_bDF9gfO2jB1id4FV5h1oIxEJRWyqVjhb7bNEa5nQ0,3224
 dbos/cli/_template_init.py,sha256=7JBcpMqP1r2mfCnvWatu33z8ctEGHJarlZYKgB83cXE,2972
-dbos/cli/cli.py,sha256=SFmaHlCHtOojETd-P8BpbENQvwQoyq2l2yXZhSVo-50,18892
-dbos/dbos-config.schema.json,sha256=8KcwJb_sQc4-6tQG2TLmjE_nratfrQa0qVLl9XPsvWE,6367
+dbos/cli/cli.py,sha256=YPXZyAD3GIh1cw_kBTAcJxUGO6OgBHWhjQLVe66AY8k,20143
+dbos/dbos-config.schema.json,sha256=CjaspeYmOkx6Ip_pcxtmfXJTn_YGdSx_0pcPBF7KZmo,6060
 dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
 version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
-dbos-0.28.0a12.dist-info/RECORD,,
+dbos-0.28.0a15.dist-info/RECORD,,