dbos 1.2.0a6__py3-none-any.whl → 1.3.0__py3-none-any.whl

dbos/_client.py

@@ -3,8 +3,6 @@ import sys
 import uuid
 from typing import Any, Generic, List, Optional, TypedDict, TypeVar
 
-from sqlalchemy import URL
-
 from dbos._app_db import ApplicationDatabase
 from dbos._context import MaxPriority, MinPriority
 
@@ -15,6 +13,7 @@ else:
 
 from dbos import _serialization
 from dbos._dbos import WorkflowHandle, WorkflowHandleAsync
+from dbos._dbos_config import is_valid_database_url
 from dbos._error import DBOSException, DBOSNonExistentWorkflowError
 from dbos._registrations import DEFAULT_MAX_RECOVERY_ATTEMPTS
 from dbos._serialization import WorkflowInputs
@@ -99,6 +98,7 @@ class WorkflowHandleClientAsyncPolling(Generic[R]):
 
 class DBOSClient:
     def __init__(self, database_url: str, *, system_database: Optional[str] = None):
+        assert is_valid_database_url(database_url)
         # We only create database connections but do not run migrations
         self._sys_db = SystemDatabase(
             database_url=database_url,
@@ -141,6 +141,11 @@ class DBOSClient:
             "priority": options.get("priority"),
         }
 
+        inputs: WorkflowInputs = {
+            "args": args,
+            "kwargs": kwargs,
+        }
+
         status: WorkflowStatusInternal = {
             "workflow_uuid": workflow_id,
             "status": WorkflowStatusString.ENQUEUED.value,
@@ -163,18 +168,18 @@ class DBOSClient:
                 int(workflow_timeout * 1000) if workflow_timeout is not None else None
             ),
             "workflow_deadline_epoch_ms": None,
-        }
-
-        inputs: WorkflowInputs = {
-            "args": args,
-            "kwargs": kwargs,
+            "deduplication_id": enqueue_options_internal["deduplication_id"],
+            "priority": (
+                enqueue_options_internal["priority"]
+                if enqueue_options_internal["priority"] is not None
+                else 0
+            ),
+            "inputs": _serialization.serialize_args(inputs),
         }
 
         self._sys_db.init_workflow(
             status,
-            _serialization.serialize_args(inputs),
             max_recovery_attempts=None,
-            enqueue_options=enqueue_options_internal,
         )
         return workflow_id
 
@@ -230,6 +235,9 @@ class DBOSClient:
             "app_version": None,
             "workflow_timeout_ms": None,
             "workflow_deadline_epoch_ms": None,
+            "deduplication_id": None,
+            "priority": 0,
+            "inputs": _serialization.serialize_args({"args": (), "kwargs": {}}),
         }
         with self._sys_db.engine.begin() as conn:
             self._sys_db._insert_workflow_status(

dbos/_core.py

@@ -252,6 +252,10 @@ def _init_workflow(
             raise DBOSNonExistentWorkflowError(wfid)
         return get_status_result
 
+    # If we have a class name, the first arg is the instance and do not serialize
+    if class_name is not None:
+        inputs = {"args": inputs["args"][1:], "kwargs": inputs["kwargs"]}
+
     # Initialize a workflow status object from the context
     status: WorkflowStatusInternal = {
         "workflow_uuid": wfid,
@@ -279,18 +283,25 @@ def _init_workflow(
         "updated_at": None,
         "workflow_timeout_ms": workflow_timeout_ms,
         "workflow_deadline_epoch_ms": workflow_deadline_epoch_ms,
+        "deduplication_id": (
+            enqueue_options["deduplication_id"] if enqueue_options is not None else None
+        ),
+        "priority": (
+            (
+                enqueue_options["priority"]
+                if enqueue_options["priority"] is not None
+                else 0
+            )
+            if enqueue_options is not None
+            else 0
+        ),
+        "inputs": _serialization.serialize_args(inputs),
     }
 
-    # If we have a class name, the first arg is the instance and do not serialize
-    if class_name is not None:
-        inputs = {"args": inputs["args"][1:], "kwargs": inputs["kwargs"]}
-
     # Synchronously record the status and inputs for workflows
     wf_status, workflow_deadline_epoch_ms = dbos._sys_db.init_workflow(
         status,
-        _serialization.serialize_args(inputs),
         max_recovery_attempts=max_recovery_attempts,
-        enqueue_options=enqueue_options,
     )
 
     if workflow_deadline_epoch_ms is not None:
@@ -342,13 +353,12 @@ def _get_wf_invoke_func(
             return recorded_result
         try:
             output = func()
-            status["status"] = "SUCCESS"
-            status["output"] = _serialization.serialize(output)
             if not dbos.debug_mode:
-                if status["queue_name"] is not None:
-                    queue = dbos._registry.queue_info_map[status["queue_name"]]
-                    dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
-                dbos._sys_db.update_workflow_status(status)
+                dbos._sys_db.update_workflow_outcome(
+                    status["workflow_uuid"],
+                    "SUCCESS",
+                    output=_serialization.serialize(output),
+                )
             return output
         except DBOSWorkflowConflictIDError:
             # Await the workflow result
@@ -357,13 +367,12 @@ def _get_wf_invoke_func(
         except DBOSWorkflowCancelledError as error:
             raise
         except Exception as error:
-            status["status"] = "ERROR"
-            status["error"] = _serialization.serialize_exception(error)
             if not dbos.debug_mode:
-                if status["queue_name"] is not None:
-                    queue = dbos._registry.queue_info_map[status["queue_name"]]
-                    dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
-                dbos._sys_db.update_workflow_status(status)
+                dbos._sys_db.update_workflow_outcome(
+                    status["workflow_uuid"],
+                    "ERROR",
+                    error=_serialization.serialize_exception(error),
+                )
             raise
 
     return persist
@@ -432,16 +441,13 @@ def execute_workflow_by_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[An
     status = dbos._sys_db.get_workflow_status(workflow_id)
     if not status:
         raise DBOSRecoveryError(workflow_id, "Workflow status not found")
-    inputs = dbos._sys_db.get_workflow_inputs(workflow_id)
-    if not inputs:
-        raise DBOSRecoveryError(workflow_id, "Workflow inputs not found")
+    inputs = _serialization.deserialize_args(status["inputs"])
     wf_func = dbos._registry.workflow_info_map.get(status["name"], None)
     if not wf_func:
         raise DBOSWorkflowFunctionNotFoundError(
             workflow_id, "Workflow function not found"
         )
     with DBOSContextEnsure():
-        ctx = assert_current_dbos_context()
         # If this function belongs to a configured class, add that class instance as its first argument
         if status["config_name"] is not None:
             config_name = status["config_name"]

dbos/_dbos.py

@@ -92,7 +92,6 @@ from ._dbos_config import (
     DBOSConfig,
     overwrite_config,
     process_config,
-    set_env_vars,
     translate_dbos_config_to_config_file,
 )
 from ._error import (
@@ -101,7 +100,13 @@ from ._error import (
     DBOSNonExistentWorkflowError,
 )
 from ._event_loop import BackgroundEventLoop
-from ._logger import add_otlp_to_all_loggers, config_logger, dbos_logger, init_logger
+from ._logger import (
+    add_otlp_to_all_loggers,
+    add_transformer_to_all_loggers,
+    config_logger,
+    dbos_logger,
+    init_logger,
+)
 from ._workflow_commands import get_workflow, list_workflow_steps
 
 # Most DBOS functions are just any callable F, so decorators / wrappers work on F
@@ -216,6 +221,8 @@ class DBOSRegistry:
         sources = sorted(
             [inspect.getsource(wf) for wf in self.workflow_info_map.values()]
         )
+        # Different DBOS versions should produce different app versions
+        sources.append(GlobalParams.dbos_version)
         for source in sources:
             hasher.update(source.encode("utf-8"))
         return hasher.hexdigest()
@@ -298,7 +305,6 @@ class DBOS:
 
         self._launched: bool = False
         self._debug_mode: bool = False
-        self._configured_threadpool: bool = False
         self._sys_db_field: Optional[SystemDatabase] = None
         self._app_db_field: Optional[ApplicationDatabase] = None
         self._registry: DBOSRegistry = _get_or_create_dbos_registry()
@@ -329,7 +335,6 @@ class DBOS:
         else:
             raise ValueError("No valid configuration was loaded.")
 
-        set_env_vars(self._config)
         config_logger(self._config)
         dbos_tracer.config(self._config)
         dbos_logger.info(f"Initializing DBOS (v{GlobalParams.dbos_version})")
@@ -412,7 +417,7 @@ class DBOS:
                 GlobalParams.executor_id = str(uuid.uuid4())
             dbos_logger.info(f"Executor ID: {GlobalParams.executor_id}")
             dbos_logger.info(f"Application version: {GlobalParams.app_version}")
-            self._executor_field = ThreadPoolExecutor(max_workers=64)
+            self._executor_field = ThreadPoolExecutor(max_workers=sys.maxsize)
             self._background_event_loop.start()
             assert self._config["database_url"] is not None
             assert self._config["database"]["sys_db_engine_kwargs"] is not None
@@ -515,6 +520,7 @@ class DBOS:
             for handler in dbos_logger.handlers:
                 handler.flush()
             add_otlp_to_all_loggers()
+            add_transformer_to_all_loggers()
         except Exception:
             dbos_logger.error(f"DBOS failed to launch: {traceback.format_exc()}")
             raise
@@ -943,11 +949,8 @@ class DBOS:
 
         This function is called before the first call to asyncio.to_thread.
         """
-        if _get_dbos_instance()._configured_threadpool:
-            return
         loop = asyncio.get_running_loop()
         loop.set_default_executor(_get_dbos_instance()._executor)
-        _get_dbos_instance()._configured_threadpool = True
 
     @classmethod
     def resume_workflow(cls, workflow_id: str) -> WorkflowHandle[Any]:

dbos/_dbos_config.py

@@ -91,7 +91,7 @@ class ConfigFile(TypedDict, total=False):
     Data structure containing the DBOS Configuration.
 
     This configuration data is typically loaded from `dbos-config.yaml`.
-    See `https://docs.dbos.dev/api-reference/configuration`_
+    See `https://docs.dbos.dev/python/reference/configuration#dbos-configuration-file`
 
     Attributes:
         name (str): Application name
@@ -329,17 +329,9 @@ def process_config(
     if data.get("database_url") is not None and data["database_url"] != "":
         # Parse the db string and check required fields
         assert data["database_url"] is not None
+        assert is_valid_database_url(data["database_url"])
+
         url = make_url(data["database_url"])
-        required_fields = [
-            ("username", "Username must be specified in the connection URL"),
-            ("password", "Password must be specified in the connection URL"),
-            ("host", "Host must be specified in the connection URL"),
-            ("database", "Database name must be specified in the connection URL"),
-        ]
-        for field_name, error_message in required_fields:
-            field_value = getattr(url, field_name, None)
-            if not field_value:
-                raise DBOSInitializationError(error_message)
 
         if not data["database"].get("sys_db_name"):
             assert url.database is not None
@@ -385,6 +377,9 @@ def process_config(
     if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
         log_url = make_url(data["database_url"]).render_as_string(hide_password=True)
         print(f"[bold blue]Using database connection string: {log_url}[/bold blue]")
+        print(
+            f"[bold blue]Database engine parameters: {data['database']['db_engine_kwargs']}[/bold blue]"
+        )
 
     # Return data as ConfigFile type
     return data
@@ -432,6 +427,21 @@ def configure_db_engine_parameters(
     data["sys_db_engine_kwargs"] = system_engine_kwargs
 
 
+def is_valid_database_url(database_url: str) -> bool:
+    url = make_url(database_url)
+    required_fields = [
+        ("username", "Username must be specified in the connection URL"),
+        ("password", "Password must be specified in the connection URL"),
+        ("host", "Host must be specified in the connection URL"),
+        ("database", "Database name must be specified in the connection URL"),
+    ]
+    for field_name, error_message in required_fields:
+        field_value = getattr(url, field_name, None)
+        if not field_value:
+            raise DBOSInitializationError(error_message)
+    return True
+
+
 def _is_valid_app_name(name: str) -> bool:
     name_len = len(name)
     if name_len < 3 or name_len > 30:
@@ -445,12 +455,6 @@ def _app_name_to_db_name(app_name: str) -> str:
     return name if not name[0].isdigit() else f"_{name}"
 
 
-def set_env_vars(config: ConfigFile) -> None:
-    for env, value in config.get("env", {}).items():
-        if value is not None:
-            os.environ[env] = str(value)
-
-
 def overwrite_config(provided_config: ConfigFile) -> ConfigFile:
     # Load the DBOS configuration file and force the use of:
     # 1. The database url provided by DBOS_DATABASE_URL

dbos/_event_loop.py

@@ -1,6 +1,5 @@
 import asyncio
 import threading
-from concurrent.futures import ThreadPoolExecutor
 from typing import Any, Coroutine, Optional, TypeVar
 
 
@@ -34,17 +33,15 @@ class BackgroundEventLoop:
 
     def _run_event_loop(self) -> None:
         self._loop = asyncio.new_event_loop()
-        with ThreadPoolExecutor(max_workers=64) as thread_pool:
-            self._loop.set_default_executor(thread_pool)
-            asyncio.set_event_loop(self._loop)
+        asyncio.set_event_loop(self._loop)
 
-            self._running = True
-            self._ready.set()  # Signal that the loop is ready
+        self._running = True
+        self._ready.set()  # Signal that the loop is ready
 
-            try:
-                self._loop.run_forever()
-            finally:
-                self._loop.close()
+        try:
+            self._loop.run_forever()
+        finally:
+            self._loop.close()
 
     async def _shutdown(self) -> None:
         if self._loop is None:

dbos/_logger.py

@@ -16,7 +16,7 @@ if TYPE_CHECKING:
     from ._dbos_config import ConfigFile
 
 dbos_logger = logging.getLogger("dbos")
-_otlp_handler, _otlp_transformer = None, None
+_otlp_handler, _dbos_log_transformer = None, None
 
 
 class DBOSLogTransformer(logging.Filter):
@@ -93,25 +93,31 @@ def config_logger(config: "ConfigFile") -> None:
         global _otlp_handler
         _otlp_handler = LoggingHandler(logger_provider=log_provider)
 
-        # Attach DBOS-specific attributes to all log entries.
-        global _otlp_transformer
-        _otlp_transformer = DBOSLogTransformer()
-
         # Direct DBOS logs to OTLP
         dbos_logger.addHandler(_otlp_handler)
-        dbos_logger.addFilter(_otlp_transformer)
+
+    # Attach DBOS-specific attributes to all log entries.
+    global _dbos_log_transformer
+    _dbos_log_transformer = DBOSLogTransformer()
+    dbos_logger.addFilter(_dbos_log_transformer)
 
 
 def add_otlp_to_all_loggers() -> None:
-    if _otlp_handler is not None and _otlp_transformer is not None:
+    if _otlp_handler is not None:
         root = logging.root
-
         root.addHandler(_otlp_handler)
-        root.addFilter(_otlp_transformer)
-
         for logger_name in root.manager.loggerDict:
             if logger_name != dbos_logger.name:
                 logger = logging.getLogger(logger_name)
                 if not logger.propagate:
                     logger.addHandler(_otlp_handler)
-                logger.addFilter(_otlp_transformer)
+
+
+def add_transformer_to_all_loggers() -> None:
+    if _dbos_log_transformer is not None:
+        root = logging.root
+        root.addFilter(_dbos_log_transformer)
+        for logger_name in root.manager.loggerDict:
+            if logger_name != dbos_logger.name:
+                logger = logging.getLogger(logger_name)
+                logger.addFilter(_dbos_log_transformer)

dbos/_migrations/versions/66478e1b95e5_consolidate_queues.py

@@ -0,0 +1,71 @@
+"""consolidate_queues
+
+Revision ID: 66478e1b95e5
+Revises: 933e86bdac6a
+Create Date: 2025-05-21 10:14:25.674613
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "66478e1b95e5"
+down_revision: Union[str, None] = "933e86bdac6a"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add new columns to workflow_status table
+    op.add_column(
+        "workflow_status",
+        sa.Column("started_at_epoch_ms", sa.BigInteger(), nullable=True),
+        schema="dbos",
+    )
+
+    op.add_column(
+        "workflow_status",
+        sa.Column("deduplication_id", sa.Text(), nullable=True),
+        schema="dbos",
+    )
+
+    op.add_column(
+        "workflow_status",
+        sa.Column(
+            "priority", sa.Integer(), nullable=False, server_default=sa.text("'0'::int")
+        ),
+        schema="dbos",
+    )
+
+    # Add unique constraint for deduplication_id
+    op.create_unique_constraint(
+        "uq_workflow_status_queue_name_dedup_id",
+        "workflow_status",
+        ["queue_name", "deduplication_id"],
+        schema="dbos",
+    )
+
+    # Add index on status field
+    op.create_index(
+        "workflow_status_status_index", "workflow_status", ["status"], schema="dbos"
+    )
+
+
+def downgrade() -> None:
+    # Drop indexes
+    op.drop_index(
+        "workflow_status_status_index", table_name="workflow_status", schema="dbos"
+    )
+
+    # Drop unique constraint
+    op.drop_constraint(
+        "uq_workflow_status_queue_name_dedup_id", "workflow_status", schema="dbos"
+    )
+
+    # Drop columns
+    op.drop_column("workflow_status", "priority", schema="dbos")
+    op.drop_column("workflow_status", "deduplication_id", schema="dbos")
+    op.drop_column("workflow_status", "started_at_epoch_ms", schema="dbos")

dbos/_migrations/versions/d994145b47b6_consolidate_inputs.py

@@ -0,0 +1,30 @@
+"""consolidate_inputs
+
+Revision ID: d994145b47b6
+Revises: 66478e1b95e5
+Create Date: 2025-05-23 08:09:15.515009
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d994145b47b6"
+down_revision: Union[str, None] = "66478e1b95e5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "workflow_status",
+        sa.Column("inputs", sa.Text(), nullable=True),
+        schema="dbos",
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("workflow_status", "inputs", schema="dbos")

dbos/_queue.py

@@ -5,8 +5,9 @@ from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, TypedDict
 from psycopg import errors
 from sqlalchemy.exc import OperationalError
 
+from dbos._context import get_local_dbos_context
 from dbos._logger import dbos_logger
-from dbos._utils import GlobalParams
+from dbos._utils import INTERNAL_QUEUE_NAME, GlobalParams
 
 from ._core import P, R, execute_workflow_by_id, start_workflow, start_workflow_async
 
@@ -41,6 +42,7 @@ class Queue:
         limiter: Optional[QueueRateLimit] = None,
         *,  # Disable positional arguments from here on
         worker_concurrency: Optional[int] = None,
+        priority_enabled: bool = False,
     ) -> None:
         if (
             worker_concurrency is not None
@@ -54,10 +56,11 @@ class Queue:
         self.concurrency = concurrency
         self.worker_concurrency = worker_concurrency
         self.limiter = limiter
+        self.priority_enabled = priority_enabled
         from ._dbos import _get_or_create_dbos_registry
 
         registry = _get_or_create_dbos_registry()
-        if self.name in registry.queue_info_map:
+        if self.name in registry.queue_info_map and self.name != INTERNAL_QUEUE_NAME:
             dbos_logger.warning(f"Queue {name} has already been declared")
         registry.queue_info_map[self.name] = self
 
@@ -66,6 +69,16 @@ class Queue:
     ) -> "WorkflowHandle[R]":
         from ._dbos import _get_dbos_instance
 
+        context = get_local_dbos_context()
+        if (
+            context is not None
+            and context.priority is not None
+            and not self.priority_enabled
+        ):
+            dbos_logger.warning(
+                f"Priority is not enabled for queue {self.name}. Setting priority will not have any effect."
+            )
+
         dbos = _get_dbos_instance()
         return start_workflow(dbos, func, self.name, False, *args, **kwargs)
 

dbos/_schemas/system_database.py

@@ -1,6 +1,5 @@
 from sqlalchemy import (
     BigInteger,
-    Boolean,
     Column,
     ForeignKey,
     Index,
@@ -57,8 +56,18 @@ class SystemSchema:
         Column("queue_name", Text, nullable=True),
         Column("workflow_timeout_ms", BigInteger, nullable=True),
         Column("workflow_deadline_epoch_ms", BigInteger, nullable=True),
+        Column("started_at_epoch_ms", BigInteger(), nullable=True),
+        Column("deduplication_id", Text(), nullable=True),
+        Column("inputs", Text()),
+        Column("priority", Integer(), nullable=False, server_default=text("'0'::int")),
         Index("workflow_status_created_at_index", "created_at"),
         Index("workflow_status_executor_id_index", "executor_id"),
+        Index("workflow_status_status_index", "status"),
+        UniqueConstraint(
+            "queue_name",
+            "deduplication_id",
+            name="uq_workflow_status_queue_name_dedup_id",
+        ),
     )
 
     operation_outputs = Table(
@@ -80,21 +89,6 @@ class SystemSchema:
         PrimaryKeyConstraint("workflow_uuid", "function_id"),
     )
 
-    workflow_inputs = Table(
-        "workflow_inputs",
-        metadata_obj,
-        Column(
-            "workflow_uuid",
-            Text,
-            ForeignKey(
-                "workflow_status.workflow_uuid", onupdate="CASCADE", ondelete="CASCADE"
-            ),
-            primary_key=True,
-            nullable=False,
-        ),
-        Column("inputs", Text, nullable=False),
-    )
-
     notifications = Table(
         "notifications",
         metadata_obj,
@@ -138,54 +132,3 @@ class SystemSchema:
         Column("value", Text, nullable=False),
         PrimaryKeyConstraint("workflow_uuid", "key"),
     )
-
-    scheduler_state = Table(
-        "scheduler_state",
-        metadata_obj,
-        Column("workflow_fn_name", Text, primary_key=True, nullable=False),
-        Column("last_run_time", BigInteger, nullable=False),
-    )
-
-    workflow_queue = Table(
-        "workflow_queue",
-        metadata_obj,
-        Column(
-            "workflow_uuid",
-            Text,
-            ForeignKey(
-                "workflow_status.workflow_uuid", onupdate="CASCADE", ondelete="CASCADE"
-            ),
-            nullable=False,
-            primary_key=True,
-        ),
-        # Column("executor_id", Text), # This column is deprecated. Do *not* use it.
-        Column("queue_name", Text, nullable=False),
-        Column(
-            "created_at_epoch_ms",
-            BigInteger,
-            nullable=False,
-            server_default=text("(EXTRACT(epoch FROM now()) * 1000::numeric)::bigint"),
-        ),
-        Column(
-            "started_at_epoch_ms",
-            BigInteger(),
-        ),
-        Column(
-            "completed_at_epoch_ms",
-            BigInteger(),
-        ),
-        Column(
-            "deduplication_id",
-            Text,
-            nullable=True,
-        ),
-        Column(
-            "priority",
-            Integer,
-            nullable=False,
-            server_default=text("'0'::int"),
-        ),
-        UniqueConstraint(
-            "queue_name", "deduplication_id", name="uq_workflow_queue_name_dedup_id"
-        ),
-    )