dbos 0.25.1__py3-none-any.whl → 0.26.0__py3-none-any.whl

dbos/_dbos_config.py

@@ -15,7 +15,6 @@ from jsonschema import ValidationError, validate
 from rich import print
 from sqlalchemy import URL, make_url
 
-from ._db_wizard import db_wizard, load_db_connection
 from ._error import DBOSInitializationError
 from ._logger import dbos_logger
 
@@ -59,20 +58,28 @@ class RuntimeConfig(TypedDict, total=False):
 
 
 class DatabaseConfig(TypedDict, total=False):
-    hostname: str
-    port: int
-    username: str
-    password: str
-    connectionTimeoutMillis: Optional[int]
-    app_db_name: str
+    """
+    Internal data structure containing the DBOS database configuration.
+    Attributes:
+        app_db_pool_size (int): Application database pool size
+        sys_db_name (str): System database name
+        sys_db_pool_size (int): System database pool size
+        migrate (List[str]): Migration commands to run on startup
+    """
+
+    hostname: str  # Will be removed in a future version
+    port: int  # Will be removed in a future version
+    username: str  # Will be removed in a future version
+    password: str  # Will be removed in a future version
+    connectionTimeoutMillis: Optional[int]  # Will be removed in a future version
+    app_db_name: str  # Will be removed in a future version
     app_db_pool_size: Optional[int]
     sys_db_name: Optional[str]
     sys_db_pool_size: Optional[int]
-    ssl: Optional[bool]
-    ssl_ca: Optional[str]
-    local_suffix: Optional[bool]
+    ssl: Optional[bool]  # Will be removed in a future version
+    ssl_ca: Optional[str]  # Will be removed in a future version
     migrate: Optional[List[str]]
-    rollback: Optional[List[str]]
+    rollback: Optional[List[str]]  # Will be removed in a future version
 
 
 def parse_database_url_to_dbconfig(database_url: str) -> DatabaseConfig:
@@ -119,7 +126,7 @@ class ConfigFile(TypedDict, total=False):
     Attributes:
         name (str): Application name
         runtimeConfig (RuntimeConfig): Configuration for request serving
-        database (DatabaseConfig): Configuration for the application and system databases
+        database (DatabaseConfig): Configure pool sizes, migrate commands
         database_url (str): Database connection string
         telemetry (TelemetryConfig): Configuration for tracing / logging
         env (Dict[str,str]): Environment varialbes
@@ -215,9 +222,13 @@ def translate_dbos_config_to_config_file(config: DBOSConfig) -> ConfigFile:
 
 
 def _substitute_env_vars(content: str, silent: bool = False) -> str:
-    regex = r"\$\{([^}]+)\}"  # Regex to match ${VAR_NAME} style placeholders
 
-    def replace_func(match: re.Match[str]) -> str:
+    # Regex to match ${DOCKER_SECRET:SECRET_NAME} style placeholders for Docker secrets
+    secret_regex = r"\$\{DOCKER_SECRET:([^}]+)\}"
+    # Regex to match ${VAR_NAME} style placeholders for environment variables
+    env_regex = r"\$\{(?!DOCKER_SECRET:)([^}]+)\}"
+
+    def replace_env_func(match: re.Match[str]) -> str:
         var_name = match.group(1)
         value = os.environ.get(
             var_name, ""
@@ -228,7 +239,30 @@ def _substitute_env_vars(content: str, silent: bool = False) -> str:
             )
         return value
 
-    return re.sub(regex, replace_func, content)
+    def replace_secret_func(match: re.Match[str]) -> str:
+        secret_name = match.group(1)
+        try:
+            # Docker secrets are stored in /run/secrets/
+            secret_path = f"/run/secrets/{secret_name}"
+            if os.path.exists(secret_path):
+                with open(secret_path, "r") as f:
+                    return f.read().strip()
+            elif not silent:
+                dbos_logger.warning(
+                    f"Docker secret {secret_name} would be substituted from /run/secrets/{secret_name}, but the file does not exist"
+                )
+            return ""
+        except Exception as e:
+            if not silent:
+                dbos_logger.warning(
+                    f"Error reading Docker secret {secret_name}: {str(e)}"
+                )
+            return ""
+
+    # First replace Docker secrets
+    content = re.sub(secret_regex, replace_secret_func, content)
+    # Then replace environment variables
+    return re.sub(env_regex, replace_env_func, content)
 
 
 def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
@@ -261,7 +295,6 @@ def load_config(
     config_file_path: str = DBOS_CONFIG_PATH,
     *,
     run_process_config: bool = True,
-    use_db_wizard: bool = True,
     silent: bool = False,
 ) -> ConfigFile:
     """
@@ -312,13 +345,12 @@ def load_config(
 
     data = cast(ConfigFile, data)
     if run_process_config:
-        data = process_config(data=data, use_db_wizard=use_db_wizard, silent=silent)
+        data = process_config(data=data, silent=silent)
     return data  # type: ignore
 
 
 def process_config(
     *,
-    use_db_wizard: bool = True,
     data: ConfigFile,
     silent: bool = False,
 ) -> ConfigFile:
@@ -345,22 +377,17 @@ def process_config(
     # database_url takes precedence over database config, but we need to preserve rollback and migrate if they exist
     migrate = data["database"].get("migrate", False)
     rollback = data["database"].get("rollback", False)
-    local_suffix = data["database"].get("local_suffix", False)
     if data.get("database_url"):
         dbconfig = parse_database_url_to_dbconfig(cast(str, data["database_url"]))
         if migrate:
             dbconfig["migrate"] = cast(List[str], migrate)
         if rollback:
             dbconfig["rollback"] = cast(List[str], rollback)
-        if local_suffix:
-            dbconfig["local_suffix"] = cast(bool, local_suffix)
         data["database"] = dbconfig
 
     if "app_db_name" not in data["database"] or not (data["database"]["app_db_name"]):
         data["database"]["app_db_name"] = _app_name_to_db_name(data["name"])
 
-    # Load the DB connection file. Use its values for missing connection parameters. Use defaults otherwise.
-    db_connection = load_db_connection()
     connection_passed_in = data["database"].get("hostname", None) is not None
 
     dbos_dbport: Optional[int] = None
@@ -370,49 +397,22 @@ def process_config(
             dbos_dbport = int(dbport_env)
         except ValueError:
             pass
-    dbos_dblocalsuffix: Optional[bool] = None
-    dblocalsuffix_env = os.getenv("DBOS_DBLOCALSUFFIX")
-    if dblocalsuffix_env:
-        try:
-            dbos_dblocalsuffix = dblocalsuffix_env.casefold() == "true".casefold()
-        except ValueError:
-            pass
 
     data["database"]["hostname"] = (
-        os.getenv("DBOS_DBHOST")
-        or data["database"].get("hostname")
-        or db_connection.get("hostname")
-        or "localhost"
+        os.getenv("DBOS_DBHOST") or data["database"].get("hostname") or "localhost"
     )
 
-    data["database"]["port"] = (
-        dbos_dbport or data["database"].get("port") or db_connection.get("port") or 5432
-    )
+    data["database"]["port"] = dbos_dbport or data["database"].get("port") or 5432
     data["database"]["username"] = (
-        os.getenv("DBOS_DBUSER")
-        or data["database"].get("username")
-        or db_connection.get("username")
-        or "postgres"
+        os.getenv("DBOS_DBUSER") or data["database"].get("username") or "postgres"
     )
     data["database"]["password"] = (
         os.getenv("DBOS_DBPASSWORD")
         or data["database"].get("password")
-        or db_connection.get("password")
         or os.environ.get("PGPASSWORD")
         or "dbos"
     )
 
-    local_suffix = False
-    dbcon_local_suffix = db_connection.get("local_suffix")
-    if dbcon_local_suffix is not None:
-        local_suffix = dbcon_local_suffix
-    db_local_suffix = data["database"].get("local_suffix")
-    if db_local_suffix is not None:
-        local_suffix = db_local_suffix
-    if dbos_dblocalsuffix is not None:
-        local_suffix = dbos_dblocalsuffix
-    data["database"]["local_suffix"] = local_suffix
-
     if not data["database"].get("app_db_pool_size"):
         data["database"]["app_db_pool_size"] = 20
     if not data["database"].get("sys_db_pool_size"):
@@ -427,10 +427,6 @@ def process_config(
     elif "run_admin_server" not in data["runtimeConfig"]:
         data["runtimeConfig"]["run_admin_server"] = True
 
-    # Check the connectivity to the database and make sure it's properly configured
-    # Note, never use db wizard if the DBOS is running in debug mode (i.e. DBOS_DEBUG_WORKFLOW_ID env var is set)
-    debugWorkflowId = os.getenv("DBOS_DEBUG_WORKFLOW_ID")
-
     # Pretty-print where we've loaded database connection information from, respecting the log level
     if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
         d = data["database"]
@@ -443,21 +439,11 @@ def process_config(
             print(
                 f"[bold blue]Using database connection string: {conn_string}[/bold blue]"
             )
-        elif db_connection.get("hostname"):
-            print(
-                f"[bold blue]Loading database connection string from .dbos/db_connection: {conn_string}[/bold blue]"
-            )
         else:
             print(
                 f"[bold blue]Using default database connection string: {conn_string}[/bold blue]"
             )
 
-    if use_db_wizard and debugWorkflowId is None:
-        data = db_wizard(data)
-
-    if "local_suffix" in data["database"] and data["database"]["local_suffix"]:
-        data["database"]["app_db_name"] = f"{data['database']['app_db_name']}_local"
-
     # Return data as ConfigFile type
     return data
 

dbos/_debug.py

@@ -28,7 +28,7 @@ def debug_workflow(workflow_id: str, entrypoint: Union[str, PythonModule]) -> No
 
     DBOS.logger.info(f"Debugging workflow {workflow_id}...")
     DBOS.launch(debug_mode=True)
-    handle = DBOS.execute_workflow_id(workflow_id)
+    handle = DBOS._execute_workflow_id(workflow_id)
     handle.get_result()
     DBOS.logger.info("Workflow Debugging complete. Exiting process.")
 

dbos/_docker_pg_helper.py

@@ -0,0 +1,191 @@
+import logging
+import os
+import subprocess
+import time
+
+import docker
+import psycopg
+from docker.errors import APIError, NotFound
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+from typing import Any, Dict, Optional, Tuple
+
+
+def start_docker_pg() -> None:
+    """
+    Starts a PostgreSQL database in a Docker container.
+
+    This function checks if Docker is installed, and if so, starts a local PostgreSQL
+    database in a Docker container. It configures the database with default settings
+    and provides connection information upon successful startup.
+
+    The function uses environment variable PGPASSWORD if available, otherwise
+    defaults to 'dbos' as the database password.
+
+    Returns:
+        None
+
+    Raises:
+        Exception: If there is an error starting the Docker container or if the
+                   PostgreSQL service does not become available within the timeout period.
+    """
+
+    logging.info("Attempting to create a Docker Postgres container...")
+    has_docker = check_docker_installed()
+
+    pool_config = {
+        "host": "localhost",
+        "port": 5432,
+        "password": os.environ.get("PGPASSWORD", "dbos"),
+        "user": "postgres",
+        "database": "postgres",
+        "connect_timeout": 2,
+    }
+
+    # If Docker is installed, start a local Docker based Postgres
+    if has_docker:
+        start_docker_postgres(pool_config)
+        logging.info(
+            f"Postgres available at postgresql://postgres:{pool_config['password']}@{pool_config['host']}:{pool_config['port']}"
+        )
+    else:
+        logging.warning("Docker not detected locally")
+
+
+def check_db_connectivity(config: Dict[str, Any]) -> Optional[Exception]:
+    conn = None
+    try:
+        conn = psycopg.connect(
+            host=config["host"],
+            port=config["port"],
+            user=config["user"],
+            password=config["password"],
+            dbname=config["database"],
+            connect_timeout=config.get("connect_timeout", 30),
+        )
+        cursor = conn.cursor()
+        cursor.execute("SELECT 1;")
+        cursor.close()
+        return None
+    except Exception as error:
+        return error
+    finally:
+        if conn is not None:
+            conn.close()
+
+
+def exec_sync(cmd: str) -> Tuple[str, str]:
+    result = subprocess.run(cmd, shell=True, text=True, capture_output=True, check=True)
+    return result.stdout, result.stderr
+
+
+def start_docker_postgres(pool_config: Dict[str, Any]) -> bool:
+    logging.info("Starting a Postgres Docker container...")
+    container_name = "dbos-db"
+    pg_data = "/var/lib/postgresql/data"
+
+    try:
+        client = docker.from_env()
+
+        # Check if the container already exists
+        try:
+            container = client.containers.get(container_name)
+            if container.status == "running":
+                logging.info(f"Container '{container_name}' is already running.")
+                return True
+            elif container.status == "exited":
+                container.start()
+                logging.info(
+                    f"Container '{container_name}' was stopped and has been restarted."
+                )
+                return True
+        except NotFound:
+            # Container doesn't exist, proceed with creation
+            pass
+
+        # Create and start the container
+        container = client.containers.run(
+            image="pgvector/pgvector:pg16",
+            name=container_name,
+            detach=True,
+            environment={
+                "POSTGRES_PASSWORD": pool_config["password"],
+                "PGDATA": pg_data,
+            },
+            ports={"5432/tcp": pool_config["port"]},
+            volumes={pg_data: {"bind": pg_data, "mode": "rw"}},
+            remove=True,  # Equivalent to --rm
+        )
+
+        logging.info(f"Created container: {container.id}")
+
+    except APIError as e:
+        raise Exception(f"Docker API error: {str(e)}")
+
+    # Wait for PostgreSQL to be ready
+    attempts = 30
+    while attempts > 0:
+        if attempts % 5 == 0:
+            logging.info("Waiting for Postgres Docker container to start...")
+
+        if check_db_connectivity(pool_config) is None:
+            return True
+
+        attempts -= 1
+        time.sleep(1)
+
+    raise Exception(
+        f"Failed to start Docker container: Container {container_name} did not start in time."
+    )
+
+
+def check_docker_installed() -> bool:
+    """
+    Check if Docker is installed and running using the docker library.
+
+    Returns:
+        bool: True if Docker is installed and running, False otherwise.
+    """
+    try:
+        client = docker.from_env()
+        client.ping()  # type: ignore
+        return True
+    except Exception:
+        return False
+
+
+def stop_docker_pg() -> None:
+    """
+    Stops the Docker Postgres container.
+
+    Returns:
+        bool: True if the container was successfully stopped, False if it wasn't running
+
+    Raises:
+        Exception: If there was an error stopping the container
+    """
+    logger = logging.getLogger()
+    container_name = "dbos-db"
+    try:
+        logger.info(f"Stopping Docker Postgres container {container_name}...")
+
+        client = docker.from_env()
+
+        try:
+            container = client.containers.get(container_name)
+
+            if container.status == "running":
+                container.stop()
+                logger.info(
+                    f"Successfully stopped Docker Postgres container {container_name}."
+                )
+            else:
+                logger.info(f"Container {container_name} exists but is not running.")
+
+        except docker.errors.NotFound:
+            logger.info(f"Container {container_name} does not exist.")
+
+    except Exception as error:
+        error_message = str(error)
+        logger.error(f"Failed to stop Docker Postgres container: {error_message}")
+        raise

dbos/_error.py

@@ -26,6 +26,29 @@ class DBOSException(Exception):
         return f"DBOS Error: {self.message}"
 
 
+class DBOSBaseException(BaseException):
+    """
+    This class is for DBOS exceptions that should not be caught by user code.
+    It inherits from BaseException instead of Exception so it cannot be caught
+    except by code specifically trying to catch it.
+
+    Attributes:
+        message(str): The error message string
+        dbos_error_code(DBOSErrorCode): The error code, from the `DBOSErrorCode` enum
+    """
+
+    def __init__(self, message: str, dbos_error_code: Optional[int] = None):
+        self.message = message
+        self.dbos_error_code = dbos_error_code
+        self.status_code: Optional[int] = None
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        if self.dbos_error_code:
+            return f"DBOS Error {self.dbos_error_code}: {self.message}"
+        return f"DBOS Error: {self.message}"
+
+
 class DBOSErrorCode(Enum):
     ConflictingIDError = 1
     RecoveryError = 2
@@ -37,17 +60,13 @@ class DBOSErrorCode(Enum):
     NotAuthorized = 8
     ConflictingWorkflowError = 9
     WorkflowCancelled = 10
+    UnexpectedStep = 11
     ConflictingRegistrationError = 25
 
 
-class DBOSWorkflowConflictIDError(DBOSException):
-    """Exception raised when a workflow database record already exists."""
-
-    def __init__(self, workflow_id: str):
-        super().__init__(
-            f"Conflicting workflow ID {workflow_id}",
-            dbos_error_code=DBOSErrorCode.ConflictingIDError.value,
-        )
+#######################################
+## Exception
+#######################################
 
 
 class DBOSConflictingWorkflowError(DBOSException):
@@ -137,8 +156,35 @@ class DBOSMaxStepRetriesExceeded(DBOSException):
         return (self.__class__, (self.step_name, self.max_retries))
 
 
-class DBOSWorkflowCancelledError(DBOSException):
-    """Exception raised when the workflow has already been cancelled."""
+class DBOSConflictingRegistrationError(DBOSException):
+    """Exception raised when conflicting decorators are applied to the same function."""
+
+    def __init__(self, name: str) -> None:
+        super().__init__(
+            f"Operation (Name: {name}) is already registered with a conflicting function type",
+            dbos_error_code=DBOSErrorCode.ConflictingRegistrationError.value,
+        )
+
+
+class DBOSUnexpectedStepError(DBOSException):
+    """Exception raised when a step has an unexpected recorded name."""
+
+    def __init__(
+        self, workflow_id: str, step_id: int, expected_name: str, recorded_name: str
+    ) -> None:
+        super().__init__(
+            f"During execution of workflow {workflow_id} step {step_id}, function {recorded_name} was recorded when {expected_name} was expected. Check that your workflow is deterministic.",
+            dbos_error_code=DBOSErrorCode.UnexpectedStep.value,
+        )
+
+
+#######################################
+## BaseException
+#######################################
+
+
+class DBOSWorkflowCancelledError(DBOSBaseException):
+    """BaseException raised when the workflow has already been cancelled."""
 
     def __init__(self, msg: str) -> None:
         super().__init__(
@@ -147,11 +193,11 @@ class DBOSWorkflowCancelledError(DBOSException):
         )
 
 
-class DBOSConflictingRegistrationError(DBOSException):
-    """Exception raised when conflicting decorators are applied to the same function."""
+class DBOSWorkflowConflictIDError(DBOSBaseException):
+    """BaseException raised when a workflow database record already exists."""
 
-    def __init__(self, name: str) -> None:
+    def __init__(self, workflow_id: str):
         super().__init__(
-            f"Operation (Name: {name}) is already registered with a conflicting function type",
-            dbos_error_code=DBOSErrorCode.ConflictingRegistrationError.value,
+            f"Conflicting workflow ID {workflow_id}",
+            dbos_error_code=DBOSErrorCode.ConflictingIDError.value,
         )

dbos/_event_loop.py

@@ -0,0 +1,67 @@
+import asyncio
+import threading
+from typing import Any, Coroutine, Optional, TypeVar
+
+
+class BackgroundEventLoop:
+    """
+    This is the event loop to which DBOS submits any coroutines that are not started from within an event loop.
+    In particular, coroutines submitted to queues (such as from scheduled workflows) run on this event loop.
+    """
+
+    def __init__(self) -> None:
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._thread: Optional[threading.Thread] = None
+        self._running = False
+        self._ready = threading.Event()
+
+    def start(self) -> None:
+        if self._running:
+            return
+
+        self._thread = threading.Thread(target=self._run_event_loop, daemon=True)
+        self._thread.start()
+        self._ready.wait()  # Wait until the loop is running
+
+    def stop(self) -> None:
+        if not self._running or self._loop is None or self._thread is None:
+            return
+
+        asyncio.run_coroutine_threadsafe(self._shutdown(), self._loop)
+        self._thread.join()
+        self._running = False
+
+    def _run_event_loop(self) -> None:
+        self._loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self._loop)
+
+        self._running = True
+        self._ready.set()  # Signal that the loop is ready
+
+        try:
+            self._loop.run_forever()
+        finally:
+            self._loop.close()
+
+    async def _shutdown(self) -> None:
+        if self._loop is None:
+            raise RuntimeError("Event loop not started")
+        tasks = [
+            task
+            for task in asyncio.all_tasks(self._loop)
+            if task is not asyncio.current_task(self._loop)
+        ]
+
+        for task in tasks:
+            task.cancel()
+
+        await asyncio.gather(*tasks, return_exceptions=True)
+        self._loop.stop()
+
+    T = TypeVar("T")
+
+    def submit_coroutine(self, coro: Coroutine[Any, Any, T]) -> T:
+        """Submit a coroutine to the background event loop"""
+        if self._loop is None:
+            raise RuntimeError("Event loop not started")
+        return asyncio.run_coroutine_threadsafe(coro, self._loop).result()

dbos/_kafka.py

@@ -115,7 +115,7 @@ def kafka_consumer(
                 _in_order_kafka_queues[topic] = queue
         else:
             global _kafka_queue
-            _kafka_queue = Queue("_dbos_internal_queue")
+            _kafka_queue = dbosreg.get_internal_queue()
         stop_event = threading.Event()
         dbosreg.register_poller(
             stop_event, _kafka_consumer_loop, func, config, topics, stop_event, in_order

dbos/_migrations/versions/83f3732ae8e7_workflow_timeout.py

@@ -0,0 +1,44 @@
+"""workflow_timeout
+
+Revision ID: 83f3732ae8e7
+Revises: f4b9b32ba814
+Create Date: 2025-04-16 17:05:36.642395
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "83f3732ae8e7"
+down_revision: Union[str, None] = "f4b9b32ba814"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "workflow_status",
+        sa.Column(
+            "workflow_timeout_ms",
+            sa.BigInteger(),
+            nullable=True,
+        ),
+        schema="dbos",
+    )
+    op.add_column(
+        "workflow_status",
+        sa.Column(
+            "workflow_deadline_epoch_ms",
+            sa.BigInteger(),
+            nullable=True,
+        ),
+        schema="dbos",
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("workflow_status", "workflow_deadline_epoch_ms", schema="dbos")
+    op.drop_column("workflow_status", "workflow_timeout_ms", schema="dbos")

dbos/_queue.py

@@ -82,7 +82,8 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
     while not stop_event.is_set():
         if stop_event.wait(timeout=1):
             return
-        for _, queue in dbos._registry.queue_info_map.items():
+        queues = dict(dbos._registry.queue_info_map)
+        for _, queue in queues.items():
             try:
                 wf_ids = dbos._sys_db.start_queued_workflows(
                     queue, GlobalParams.executor_id, GlobalParams.app_version

dbos/_recovery.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
 def _recover_workflow(
     dbos: "DBOS", workflow: GetPendingWorkflowsOutput
 ) -> "WorkflowHandle[Any]":
-    if workflow.queue_name and workflow.queue_name != "_dbos_internal_queue":
+    if workflow.queue_name:
         cleared = dbos._sys_db.clear_queue_assignment(workflow.workflow_uuid)
         if cleared:
             return dbos.retrieve_workflow(workflow.workflow_uuid)