dbos 1.3.0a2__py3-none-any.whl → 1.3.0a4__py3-none-any.whl

dbos/_client.py

@@ -163,6 +163,12 @@ class DBOSClient:
                 int(workflow_timeout * 1000) if workflow_timeout is not None else None
             ),
             "workflow_deadline_epoch_ms": None,
+            "deduplication_id": enqueue_options_internal["deduplication_id"],
+            "priority": (
+                enqueue_options_internal["priority"]
+                if enqueue_options_internal["priority"] is not None
+                else 0
+            ),
         }
 
         inputs: WorkflowInputs = {
@@ -174,7 +180,6 @@ class DBOSClient:
             status,
             _serialization.serialize_args(inputs),
             max_recovery_attempts=None,
-            enqueue_options=enqueue_options_internal,
         )
         return workflow_id
 
@@ -230,6 +235,8 @@ class DBOSClient:
             "app_version": None,
             "workflow_timeout_ms": None,
             "workflow_deadline_epoch_ms": None,
+            "deduplication_id": None,
+            "priority": 0,
         }
         with self._sys_db.engine.begin() as conn:
             self._sys_db._insert_workflow_status(

dbos/_core.py

@@ -279,6 +279,18 @@ def _init_workflow(
         "updated_at": None,
         "workflow_timeout_ms": workflow_timeout_ms,
         "workflow_deadline_epoch_ms": workflow_deadline_epoch_ms,
+        "deduplication_id": (
+            enqueue_options["deduplication_id"] if enqueue_options is not None else None
+        ),
+        "priority": (
+            (
+                enqueue_options["priority"]
+                if enqueue_options["priority"] is not None
+                else 0
+            )
+            if enqueue_options is not None
+            else 0
+        ),
     }
 
     # If we have a class name, the first arg is the instance and do not serialize
@@ -290,7 +302,6 @@ def _init_workflow(
         status,
         _serialization.serialize_args(inputs),
         max_recovery_attempts=max_recovery_attempts,
-        enqueue_options=enqueue_options,
     )
 
     if workflow_deadline_epoch_ms is not None:
@@ -342,13 +353,12 @@ def _get_wf_invoke_func(
             return recorded_result
         try:
             output = func()
-            status["status"] = "SUCCESS"
-            status["output"] = _serialization.serialize(output)
             if not dbos.debug_mode:
-                if status["queue_name"] is not None:
-                    queue = dbos._registry.queue_info_map[status["queue_name"]]
-                    dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
-                dbos._sys_db.update_workflow_status(status)
+                dbos._sys_db.update_workflow_outcome(
+                    status["workflow_uuid"],
+                    "SUCCESS",
+                    output=_serialization.serialize(output),
+                )
             return output
         except DBOSWorkflowConflictIDError:
             # Await the workflow result
@@ -357,13 +367,12 @@ def _get_wf_invoke_func(
         except DBOSWorkflowCancelledError as error:
             raise
         except Exception as error:
-            status["status"] = "ERROR"
-            status["error"] = _serialization.serialize_exception(error)
             if not dbos.debug_mode:
-                if status["queue_name"] is not None:
-                    queue = dbos._registry.queue_info_map[status["queue_name"]]
-                    dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
-                dbos._sys_db.update_workflow_status(status)
+                dbos._sys_db.update_workflow_outcome(
+                    status["workflow_uuid"],
+                    "ERROR",
+                    error=_serialization.serialize_exception(error),
+                )
             raise
 
     return persist

dbos/_dbos.py

@@ -215,6 +215,8 @@ class DBOSRegistry:
         sources = sorted(
             [inspect.getsource(wf) for wf in self.workflow_info_map.values()]
         )
+        # Different DBOS versions should produce different app versions
+        sources.append(GlobalParams.dbos_version)
         for source in sources:
             hasher.update(source.encode("utf-8"))
         return hasher.hexdigest()

dbos/_migrations/versions/66478e1b95e5_consolidate_queues.py

@@ -0,0 +1,71 @@
+"""consolidate_queues
+
+Revision ID: 66478e1b95e5
+Revises: 933e86bdac6a
+Create Date: 2025-05-21 10:14:25.674613
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "66478e1b95e5"
+down_revision: Union[str, None] = "933e86bdac6a"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add new columns to workflow_status table
+    op.add_column(
+        "workflow_status",
+        sa.Column("started_at_epoch_ms", sa.BigInteger(), nullable=True),
+        schema="dbos",
+    )
+
+    op.add_column(
+        "workflow_status",
+        sa.Column("deduplication_id", sa.Text(), nullable=True),
+        schema="dbos",
+    )
+
+    op.add_column(
+        "workflow_status",
+        sa.Column(
+            "priority", sa.Integer(), nullable=False, server_default=sa.text("'0'::int")
+        ),
+        schema="dbos",
+    )
+
+    # Add unique constraint for deduplication_id
+    op.create_unique_constraint(
+        "uq_workflow_status_queue_name_dedup_id",
+        "workflow_status",
+        ["queue_name", "deduplication_id"],
+        schema="dbos",
+    )
+
+    # Add index on status field
+    op.create_index(
+        "workflow_status_status_index", "workflow_status", ["status"], schema="dbos"
+    )
+
+
+def downgrade() -> None:
+    # Drop indexes
+    op.drop_index(
+        "workflow_status_status_index", table_name="workflow_status", schema="dbos"
+    )
+
+    # Drop unique constraint
+    op.drop_constraint(
+        "uq_workflow_status_queue_name_dedup_id", "workflow_status", schema="dbos"
+    )
+
+    # Drop columns
+    op.drop_column("workflow_status", "priority", schema="dbos")
+    op.drop_column("workflow_status", "deduplication_id", schema="dbos")
+    op.drop_column("workflow_status", "started_at_epoch_ms", schema="dbos")

dbos/_queue.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, TypedDict
 from psycopg import errors
 from sqlalchemy.exc import OperationalError
 
+from dbos._context import get_local_dbos_context
 from dbos._logger import dbos_logger
 from dbos._utils import GlobalParams
 
@@ -41,6 +42,7 @@ class Queue:
         limiter: Optional[QueueRateLimit] = None,
         *,  # Disable positional arguments from here on
         worker_concurrency: Optional[int] = None,
+        priority_enabled: bool = False,
     ) -> None:
         if (
             worker_concurrency is not None
@@ -54,6 +56,7 @@ class Queue:
         self.concurrency = concurrency
         self.worker_concurrency = worker_concurrency
         self.limiter = limiter
+        self.priority_enabled = priority_enabled
         from ._dbos import _get_or_create_dbos_registry
 
         registry = _get_or_create_dbos_registry()
@@ -66,6 +69,14 @@ class Queue:
     ) -> "WorkflowHandle[R]":
         from ._dbos import _get_dbos_instance
 
+        context = get_local_dbos_context()
+        if (
+            context is not None
+            and context.priority is not None
+            and not self.priority_enabled
+        ):
+            dbos_logger.warning(f"Priority is not enabled for queue {self.name}. Setting priority will not have any effect.")
+
         dbos = _get_dbos_instance()
         return start_workflow(dbos, func, self.name, False, *args, **kwargs)
 

dbos/_schemas/system_database.py

@@ -1,6 +1,5 @@
 from sqlalchemy import (
     BigInteger,
-    Boolean,
     Column,
     ForeignKey,
     Index,
@@ -57,8 +56,17 @@ class SystemSchema:
         Column("queue_name", Text, nullable=True),
         Column("workflow_timeout_ms", BigInteger, nullable=True),
         Column("workflow_deadline_epoch_ms", BigInteger, nullable=True),
+        Column("started_at_epoch_ms", BigInteger(), nullable=True),
+        Column("deduplication_id", Text(), nullable=True),
+        Column("priority", Integer(), nullable=False, server_default=text("'0'::int")),
         Index("workflow_status_created_at_index", "created_at"),
         Index("workflow_status_executor_id_index", "executor_id"),
+        Index("workflow_status_status_index", "status"),
+        UniqueConstraint(
+            "queue_name",
+            "deduplication_id",
+            name="uq_workflow_status_queue_name_dedup_id",
+        ),
     )
 
     operation_outputs = Table(
@@ -138,54 +146,3 @@ class SystemSchema:
         Column("value", Text, nullable=False),
         PrimaryKeyConstraint("workflow_uuid", "key"),
     )
-
-    scheduler_state = Table(
-        "scheduler_state",
-        metadata_obj,
-        Column("workflow_fn_name", Text, primary_key=True, nullable=False),
-        Column("last_run_time", BigInteger, nullable=False),
-    )
-
-    workflow_queue = Table(
-        "workflow_queue",
-        metadata_obj,
-        Column(
-            "workflow_uuid",
-            Text,
-            ForeignKey(
-                "workflow_status.workflow_uuid", onupdate="CASCADE", ondelete="CASCADE"
-            ),
-            nullable=False,
-            primary_key=True,
-        ),
-        # Column("executor_id", Text), # This column is deprecated. Do *not* use it.
-        Column("queue_name", Text, nullable=False),
-        Column(
-            "created_at_epoch_ms",
-            BigInteger,
-            nullable=False,
-            server_default=text("(EXTRACT(epoch FROM now()) * 1000::numeric)::bigint"),
-        ),
-        Column(
-            "started_at_epoch_ms",
-            BigInteger(),
-        ),
-        Column(
-            "completed_at_epoch_ms",
-            BigInteger(),
-        ),
-        Column(
-            "deduplication_id",
-            Text,
-            nullable=True,
-        ),
-        Column(
-            "priority",
-            Integer,
-            nullable=False,
-            server_default=text("'0'::int"),
-        ),
-        UniqueConstraint(
-            "queue_name", "deduplication_id", name="uq_workflow_queue_name_dedup_id"
-        ),
-    )

dbos/_sys_db.py

@@ -136,13 +136,17 @@ class WorkflowStatusInternal(TypedDict):
     # The deadline of a workflow, computed by adding its timeout to its start time.
     # Deadlines propagate to children. When the deadline is reached, the workflow is cancelled.
     workflow_deadline_epoch_ms: Optional[int]
+    # Unique ID for deduplication on a queue
+    deduplication_id: Optional[str]
+    # Priority of the workflow on the queue, starting from 1 ~ 2,147,483,647. Default 0 (highest priority).
+    priority: int
 
 
 class EnqueueOptionsInternal(TypedDict):
-    deduplication_id: Optional[str]  # Unique ID for deduplication on a queue
-    priority: Optional[
-        int
-    ]  # Priority of the workflow on the queue, starting from 1 ~ 2,147,483,647. Default 0 (highest priority).
+    # Unique ID for deduplication on a queue
+    deduplication_id: Optional[str]
+    # Priority of the workflow on the queue, starting from 1 ~ 2,147,483,647. Default 0 (highest priority).
+    priority: Optional[int]
 
 
 class RecordedResult(TypedDict):
@@ -456,6 +460,8 @@ class SystemDatabase:
                 ),
                 workflow_timeout_ms=status["workflow_timeout_ms"],
                 workflow_deadline_epoch_ms=status["workflow_deadline_epoch_ms"],
+                deduplication_id=status["deduplication_id"],
+                priority=status["priority"],
             )
             .on_conflict_do_update(
                 index_elements=["workflow_uuid"],
@@ -465,7 +471,18 @@ class SystemDatabase:
 
         cmd = cmd.returning(SystemSchema.workflow_status.c.recovery_attempts, SystemSchema.workflow_status.c.status, SystemSchema.workflow_status.c.workflow_deadline_epoch_ms, SystemSchema.workflow_status.c.name, SystemSchema.workflow_status.c.class_name, SystemSchema.workflow_status.c.config_name, SystemSchema.workflow_status.c.queue_name)  # type: ignore
 
-        results = conn.execute(cmd)
+        try:
+            results = conn.execute(cmd)
+        except DBAPIError as dbapi_error:
+            # Unique constraint violation for the deduplication ID
+            if dbapi_error.orig.sqlstate == "23505":  # type: ignore
+                assert status["deduplication_id"] is not None
+                assert status["queue_name"] is not None
+                raise DBOSQueueDeduplicatedError(
+                    status["workflow_uuid"],
+                    status["queue_name"],
+                    status["deduplication_id"],
+                )
         row = results.fetchone()
         if row is not None:
             # Check the started workflow matches the expected name, class_name, config_name, and queue_name
@@ -495,12 +512,6 @@ class SystemDatabase:
                 and max_recovery_attempts is not None
                 and recovery_attempts > max_recovery_attempts + 1
             ):
-                delete_cmd = sa.delete(SystemSchema.workflow_queue).where(
-                    SystemSchema.workflow_queue.c.workflow_uuid
-                    == status["workflow_uuid"]
-                )
-                conn.execute(delete_cmd)
-
                 dlq_cmd = (
                     sa.update(SystemSchema.workflow_status)
                     .where(
@@ -513,6 +524,8 @@ class SystemDatabase:
                     )
                     .values(
                         status=WorkflowStatusString.RETRIES_EXCEEDED.value,
+                        deduplication_id=None,
+                        started_at_epoch_ms=None,
                         queue_name=None,
                     )
                 )
@@ -526,44 +539,28 @@ class SystemDatabase:
         return wf_status, workflow_deadline_epoch_ms
 
     @db_retry()
-    def update_workflow_status(
+    def update_workflow_outcome(
         self,
-        status: WorkflowStatusInternal,
+        workflow_id: str,
+        status: WorkflowStatuses,
+        *,
+        output: Optional[str] = None,
+        error: Optional[str] = None,
     ) -> None:
         if self._debug_mode:
             raise Exception("called update_workflow_status in debug mode")
-        wf_status: WorkflowStatuses = status["status"]
         with self.engine.begin() as c:
             c.execute(
-                pg.insert(SystemSchema.workflow_status)
+                sa.update(SystemSchema.workflow_status)
                 .values(
-                    workflow_uuid=status["workflow_uuid"],
-                    status=status["status"],
-                    name=status["name"],
-                    class_name=status["class_name"],
-                    config_name=status["config_name"],
-                    output=status["output"],
-                    error=status["error"],
-                    executor_id=status["executor_id"],
-                    application_version=status["app_version"],
-                    application_id=status["app_id"],
-                    authenticated_user=status["authenticated_user"],
-                    authenticated_roles=status["authenticated_roles"],
-                    assumed_role=status["assumed_role"],
-                    queue_name=status["queue_name"],
-                    recovery_attempts=(
-                        1 if wf_status != WorkflowStatusString.ENQUEUED.value else 0
-                    ),
-                )
-                .on_conflict_do_update(
-                    index_elements=["workflow_uuid"],
-                    set_=dict(
-                        status=status["status"],
-                        output=status["output"],
-                        error=status["error"],
-                        updated_at=func.extract("epoch", func.now()) * 1000,
-                    ),
+                    status=status,
+                    output=output,
+                    error=error,
+                    # As the workflow is complete, remove its deduplication ID
+                    deduplication_id=None,
+                    updated_at=func.extract("epoch", func.now()) * 1000,
                 )
+                .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
             )
 
     def cancel_workflow(
@@ -585,18 +582,15 @@ class SystemDatabase:
                 or row[0] == WorkflowStatusString.ERROR.value
             ):
                 return
-            # Remove the workflow from the queues table so it does not block the table
-            c.execute(
-                sa.delete(SystemSchema.workflow_queue).where(
-                    SystemSchema.workflow_queue.c.workflow_uuid == workflow_id
-                )
-            )
-            # Set the workflow's status to CANCELLED
+            # Set the workflow's status to CANCELLED and remove it from any queue it is on
             c.execute(
                 sa.update(SystemSchema.workflow_status)
                 .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
                 .values(
                     status=WorkflowStatusString.CANCELLED.value,
+                    queue_name=None,
+                    deduplication_id=None,
+                    started_at_epoch_ms=None,
                 )
             )
 
@@ -620,19 +614,6 @@ class SystemDatabase:
                 or status == WorkflowStatusString.ERROR.value
             ):
                 return
-            # Remove the workflow from the queues table so resume can safely be called on an ENQUEUED workflow
-            c.execute(
-                sa.delete(SystemSchema.workflow_queue).where(
-                    SystemSchema.workflow_queue.c.workflow_uuid == workflow_id
-                )
-            )
-            # Enqueue the workflow on the internal queue
-            c.execute(
-                pg.insert(SystemSchema.workflow_queue).values(
-                    workflow_uuid=workflow_id,
-                    queue_name=INTERNAL_QUEUE_NAME,
-                )
-            )
             # Set the workflow's status to ENQUEUED and clear its recovery attempts and deadline.
             c.execute(
                 sa.update(SystemSchema.workflow_status)
@@ -642,6 +623,8 @@ class SystemDatabase:
                     queue_name=INTERNAL_QUEUE_NAME,
                     recovery_attempts=0,
                     workflow_deadline_epoch_ms=None,
+                    deduplication_id=None,
+                    started_at_epoch_ms=None,
                 )
             )
 
@@ -720,14 +703,6 @@ class SystemDatabase:
                 )
 
                 c.execute(insert_stmt)
-
-            # Enqueue the forked workflow on the internal queue
-            c.execute(
-                pg.insert(SystemSchema.workflow_queue).values(
-                    workflow_uuid=forked_workflow_id,
-                    queue_name=INTERNAL_QUEUE_NAME,
-                )
-            )
         return forked_workflow_id
 
     @db_retry()
@@ -753,6 +728,8 @@ class SystemDatabase:
                     SystemSchema.workflow_status.c.application_id,
                     SystemSchema.workflow_status.c.workflow_deadline_epoch_ms,
                     SystemSchema.workflow_status.c.workflow_timeout_ms,
+                    SystemSchema.workflow_status.c.deduplication_id,
+                    SystemSchema.workflow_status.c.priority,
                 ).where(SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid)
             ).fetchone()
             if row is None:
@@ -777,6 +754,8 @@ class SystemDatabase:
                 "app_id": row[13],
                 "workflow_deadline_epoch_ms": row[14],
                 "workflow_timeout_ms": row[15],
+                "deduplication_id": row[16],
+                "priority": row[17],
             }
             return status
 
@@ -972,37 +951,40 @@ class SystemDatabase:
         """
         Retrieve a list of queued workflows result and inputs based on the input criteria. The result is a list of external-facing workflow status objects.
         """
-        query = sa.select(
-            SystemSchema.workflow_status.c.workflow_uuid,
-            SystemSchema.workflow_status.c.status,
-            SystemSchema.workflow_status.c.name,
-            SystemSchema.workflow_status.c.recovery_attempts,
-            SystemSchema.workflow_status.c.config_name,
-            SystemSchema.workflow_status.c.class_name,
-            SystemSchema.workflow_status.c.authenticated_user,
-            SystemSchema.workflow_status.c.authenticated_roles,
-            SystemSchema.workflow_status.c.assumed_role,
-            SystemSchema.workflow_status.c.queue_name,
-            SystemSchema.workflow_status.c.executor_id,
-            SystemSchema.workflow_status.c.created_at,
-            SystemSchema.workflow_status.c.updated_at,
-            SystemSchema.workflow_status.c.application_version,
-            SystemSchema.workflow_status.c.application_id,
-            SystemSchema.workflow_inputs.c.inputs,
-            SystemSchema.workflow_status.c.output,
-            SystemSchema.workflow_status.c.error,
-            SystemSchema.workflow_status.c.workflow_deadline_epoch_ms,
-            SystemSchema.workflow_status.c.workflow_timeout_ms,
-        ).select_from(
-            SystemSchema.workflow_queue.join(
-                SystemSchema.workflow_status,
-                SystemSchema.workflow_queue.c.workflow_uuid
-                == SystemSchema.workflow_status.c.workflow_uuid,
-            ).join(
+        query = (
+            sa.select(
+                SystemSchema.workflow_status.c.workflow_uuid,
+                SystemSchema.workflow_status.c.status,
+                SystemSchema.workflow_status.c.name,
+                SystemSchema.workflow_status.c.recovery_attempts,
+                SystemSchema.workflow_status.c.config_name,
+                SystemSchema.workflow_status.c.class_name,
+                SystemSchema.workflow_status.c.authenticated_user,
+                SystemSchema.workflow_status.c.authenticated_roles,
+                SystemSchema.workflow_status.c.assumed_role,
+                SystemSchema.workflow_status.c.queue_name,
+                SystemSchema.workflow_status.c.executor_id,
+                SystemSchema.workflow_status.c.created_at,
+                SystemSchema.workflow_status.c.updated_at,
+                SystemSchema.workflow_status.c.application_version,
+                SystemSchema.workflow_status.c.application_id,
+                SystemSchema.workflow_inputs.c.inputs,
+                SystemSchema.workflow_status.c.output,
+                SystemSchema.workflow_status.c.error,
+                SystemSchema.workflow_status.c.workflow_deadline_epoch_ms,
+                SystemSchema.workflow_status.c.workflow_timeout_ms,
+            )
+            .join(
                 SystemSchema.workflow_inputs,
-                SystemSchema.workflow_queue.c.workflow_uuid
+                SystemSchema.workflow_status.c.workflow_uuid
                 == SystemSchema.workflow_inputs.c.workflow_uuid,
             )
+            .where(
+                sa.and_(
+                    SystemSchema.workflow_status.c.queue_name.isnot(None),
+                    SystemSchema.workflow_status.c.status.in_(["ENQUEUED", "PENDING"]),
+                )
+            )
         )
         if input["sort_desc"]:
             query = query.order_by(SystemSchema.workflow_status.c.created_at.desc())
@@ -1014,7 +996,7 @@ class SystemDatabase:
 
         if input.get("queue_name"):
             query = query.where(
-                SystemSchema.workflow_queue.c.queue_name == input["queue_name"]
+                SystemSchema.workflow_status.c.queue_name == input["queue_name"]
             )
 
         if input.get("status"):
@@ -1693,51 +1675,6 @@ class SystemDatabase:
             )
         return value
 
-    def _enqueue(
-        self,
-        workflow_id: str,
-        queue_name: str,
-        conn: sa.Connection,
-        *,
-        enqueue_options: Optional[EnqueueOptionsInternal],
-    ) -> None:
-        if self._debug_mode:
-            raise Exception("called enqueue in debug mode")
-        try:
-            deduplication_id = (
-                enqueue_options["deduplication_id"]
-                if enqueue_options is not None
-                else None
-            )
-            priority = (
-                enqueue_options["priority"] if enqueue_options is not None else None
-            )
-            # Default to 0 (highest priority) if not provided
-            if priority is None:
-                priority = 0
-            query = (
-                pg.insert(SystemSchema.workflow_queue)
-                .values(
-                    workflow_uuid=workflow_id,
-                    queue_name=queue_name,
-                    deduplication_id=deduplication_id,
-                    priority=priority,
-                )
-                .on_conflict_do_nothing(
-                    index_elements=SystemSchema.workflow_queue.primary_key.columns
-                )
-            )  # Ignore primary key constraint violation
-            conn.execute(query)
-        except DBAPIError as dbapi_error:
-            # Unique constraint violation for the deduplication ID
-            if dbapi_error.orig.sqlstate == "23505":  # type: ignore
-                assert (
-                    deduplication_id is not None
-                ), f"deduplication_id should not be None. Workflow ID: {workflow_id}, Queue name: {queue_name}."
-                raise DBOSQueueDeduplicatedError(
-                    workflow_id, queue_name, deduplication_id
-                )
-
     def start_queued_workflows(
         self, queue: "Queue", executor_id: str, app_version: str
     ) -> List[str]:
@@ -1755,13 +1692,14 @@ class SystemDatabase:
             if queue.limiter is not None:
                 query = (
                     sa.select(sa.func.count())
-                    .select_from(SystemSchema.workflow_queue)
-                    .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
+                    .select_from(SystemSchema.workflow_status)
+                    .where(SystemSchema.workflow_status.c.queue_name == queue.name)
                     .where(
-                        SystemSchema.workflow_queue.c.started_at_epoch_ms.isnot(None)
+                        SystemSchema.workflow_status.c.status
+                        != WorkflowStatusString.ENQUEUED.value
                     )
                     .where(
-                        SystemSchema.workflow_queue.c.started_at_epoch_ms
+                        SystemSchema.workflow_status.c.started_at_epoch_ms
                         > start_time_ms - limiter_period_ms
                     )
                 )
@@ -1769,64 +1707,57 @@ class SystemDatabase:
                 if num_recent_queries >= queue.limiter["limit"]:
                     return []
 
-            # Count how many workflows on this queue are currently PENDING both locally and globally.
-            pending_tasks_query = (
-                sa.select(
-                    SystemSchema.workflow_status.c.executor_id,
-                    sa.func.count().label("task_count"),
-                )
-                .select_from(
-                    SystemSchema.workflow_queue.join(
-                        SystemSchema.workflow_status,
-                        SystemSchema.workflow_queue.c.workflow_uuid
-                        == SystemSchema.workflow_status.c.workflow_uuid,
-                    )
-                )
-                .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
-                .where(
-                    SystemSchema.workflow_status.c.status
-                    == WorkflowStatusString.PENDING.value
-                )
-                .group_by(SystemSchema.workflow_status.c.executor_id)
-            )
-            pending_workflows = c.execute(pending_tasks_query).fetchall()
-            pending_workflows_dict = {row[0]: row[1] for row in pending_workflows}
-            local_pending_workflows = pending_workflows_dict.get(executor_id, 0)
-
             # Compute max_tasks, the number of workflows that can be dequeued given local and global concurrency limits,
             max_tasks = float("inf")
-            if queue.worker_concurrency is not None:
-                # Print a warning if the local concurrency limit is violated
-                if local_pending_workflows > queue.worker_concurrency:
-                    dbos_logger.warning(
-                        f"The number of local pending workflows ({local_pending_workflows}) on queue {queue.name} exceeds the local concurrency limit ({queue.worker_concurrency})"
+            if queue.worker_concurrency is not None or queue.concurrency is not None:
+                # Count how many workflows on this queue are currently PENDING both locally and globally.
+                pending_tasks_query = (
+                    sa.select(
+                        SystemSchema.workflow_status.c.executor_id,
+                        sa.func.count().label("task_count"),
+                    )
+                    .select_from(SystemSchema.workflow_status)
+                    .where(SystemSchema.workflow_status.c.queue_name == queue.name)
+                    .where(
+                        SystemSchema.workflow_status.c.status
+                        == WorkflowStatusString.PENDING.value
+                    )
+                    .group_by(SystemSchema.workflow_status.c.executor_id)
+                )
+                pending_workflows = c.execute(pending_tasks_query).fetchall()
+                pending_workflows_dict = {row[0]: row[1] for row in pending_workflows}
+                local_pending_workflows = pending_workflows_dict.get(executor_id, 0)
+
+                if queue.worker_concurrency is not None:
+                    # Print a warning if the local concurrency limit is violated
+                    if local_pending_workflows > queue.worker_concurrency:
+                        dbos_logger.warning(
+                            f"The number of local pending workflows ({local_pending_workflows}) on queue {queue.name} exceeds the local concurrency limit ({queue.worker_concurrency})"
+                        )
+                    max_tasks = max(
+                        0, queue.worker_concurrency - local_pending_workflows
                     )
-                max_tasks = max(0, queue.worker_concurrency - local_pending_workflows)
 
-            if queue.concurrency is not None:
-                global_pending_workflows = sum(pending_workflows_dict.values())
-                # Print a warning if the global concurrency limit is violated
-                if global_pending_workflows > queue.concurrency:
-                    dbos_logger.warning(
-                        f"The total number of pending workflows ({global_pending_workflows}) on queue {queue.name} exceeds the global concurrency limit ({queue.concurrency})"
+                if queue.concurrency is not None:
+                    global_pending_workflows = sum(pending_workflows_dict.values())
+                    # Print a warning if the global concurrency limit is violated
+                    if global_pending_workflows > queue.concurrency:
+                        dbos_logger.warning(
+                            f"The total number of pending workflows ({global_pending_workflows}) on queue {queue.name} exceeds the global concurrency limit ({queue.concurrency})"
+                        )
+                    available_tasks = max(
+                        0, queue.concurrency - global_pending_workflows
                     )
-                available_tasks = max(0, queue.concurrency - global_pending_workflows)
-                max_tasks = min(max_tasks, available_tasks)
+                    max_tasks = min(max_tasks, available_tasks)
 
             # Retrieve the first max_tasks workflows in the queue.
             # Only retrieve workflows of the local version (or without version set)
             query = (
                 sa.select(
-                    SystemSchema.workflow_queue.c.workflow_uuid,
-                )
-                .select_from(
-                    SystemSchema.workflow_queue.join(
-                        SystemSchema.workflow_status,
-                        SystemSchema.workflow_queue.c.workflow_uuid
-                        == SystemSchema.workflow_status.c.workflow_uuid,
-                    )
+                    SystemSchema.workflow_status.c.workflow_uuid,
                 )
-                .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
+                .select_from(SystemSchema.workflow_status)
+                .where(SystemSchema.workflow_status.c.queue_name == queue.name)
                 .where(
                     SystemSchema.workflow_status.c.status
                     == WorkflowStatusString.ENQUEUED.value
@@ -1838,12 +1769,15 @@ class SystemDatabase:
                         SystemSchema.workflow_status.c.application_version.is_(None),
                     )
                 )
-                .order_by(
-                    SystemSchema.workflow_queue.c.priority.asc(),
-                    SystemSchema.workflow_queue.c.created_at_epoch_ms.asc(),
-                )
                 .with_for_update(nowait=True)  # Error out early
             )
+            if queue.priority_enabled:
+                query = query.order_by(
+                    SystemSchema.workflow_status.c.priority.asc(),
+                    SystemSchema.workflow_status.c.created_at.asc(),
+                )
+            else:
+                query = query.order_by(SystemSchema.workflow_status.c.created_at.asc())
             # Apply limit only if max_tasks is finite
             if max_tasks != float("inf"):
                 query = query.limit(int(max_tasks))
@@ -1873,6 +1807,7 @@ class SystemDatabase:
                         status=WorkflowStatusString.PENDING.value,
                         application_version=app_version,
                         executor_id=executor_id,
+                        started_at_epoch_ms=start_time_ms,
                         # If a timeout is set, set the deadline on dequeue
                         workflow_deadline_epoch_ms=sa.case(
                             (
@@ -1892,82 +1827,31 @@ class SystemDatabase:
                     )
                 )
                 # Then give it a start time
-                c.execute(
-                    SystemSchema.workflow_queue.update()
-                    .where(SystemSchema.workflow_queue.c.workflow_uuid == id)
-                    .values(started_at_epoch_ms=start_time_ms)
-                )
                 ret_ids.append(id)
 
-            # If we have a limiter, garbage-collect all completed workflows started
-            # before the period. If there's no limiter, there's no need--they were
-            # deleted on completion.
-            if queue.limiter is not None:
-                c.execute(
-                    sa.delete(SystemSchema.workflow_queue)
-                    .where(SystemSchema.workflow_queue.c.completed_at_epoch_ms != None)
-                    .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
-                    .where(
-                        SystemSchema.workflow_queue.c.started_at_epoch_ms
-                        < start_time_ms - limiter_period_ms
-                    )
-                )
-
             # Return the IDs of all functions we started
             return ret_ids
 
-    @db_retry()
-    def remove_from_queue(self, workflow_id: str, queue: "Queue") -> None:
-        if self._debug_mode:
-            raise Exception("called remove_from_queue in debug mode")
-
-        with self.engine.begin() as c:
-            if queue.limiter is None:
-                c.execute(
-                    sa.delete(SystemSchema.workflow_queue).where(
-                        SystemSchema.workflow_queue.c.workflow_uuid == workflow_id
-                    )
-                )
-            else:
-                c.execute(
-                    sa.update(SystemSchema.workflow_queue)
-                    .where(SystemSchema.workflow_queue.c.workflow_uuid == workflow_id)
-                    .values(completed_at_epoch_ms=int(time.time() * 1000))
-                )
-
     def clear_queue_assignment(self, workflow_id: str) -> bool:
         if self._debug_mode:
             raise Exception("called clear_queue_assignment in debug mode")
 
-        with self.engine.connect() as conn:
-            with conn.begin() as transaction:
-                # Reset the start time in the queue to mark it as not started
-                res = conn.execute(
-                    sa.update(SystemSchema.workflow_queue)
-                    .where(SystemSchema.workflow_queue.c.workflow_uuid == workflow_id)
-                    .where(
-                        SystemSchema.workflow_queue.c.completed_at_epoch_ms.is_(None)
-                    )
-                    .values(started_at_epoch_ms=None)
+        with self.engine.begin() as c:
+            # Reset the status of the task to "ENQUEUED"
+            res = c.execute(
+                sa.update(SystemSchema.workflow_status)
+                .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
+                .where(SystemSchema.workflow_status.c.queue_name.isnot(None))
+                .where(
+                    SystemSchema.workflow_status.c.status
+                    == WorkflowStatusString.PENDING.value
                 )
-
-                # If no rows were affected, the workflow is not anymore in the queue or was already completed
-                if res.rowcount == 0:
-                    transaction.rollback()
-                    return False
-
-                # Reset the status of the task to "ENQUEUED"
-                res = conn.execute(
-                    sa.update(SystemSchema.workflow_status)
-                    .where(SystemSchema.workflow_status.c.workflow_uuid == workflow_id)
-                    .values(status=WorkflowStatusString.ENQUEUED.value)
+                .values(
+                    status=WorkflowStatusString.ENQUEUED.value, started_at_epoch_ms=None
                 )
-                if res.rowcount == 0:
-                    # This should never happen
-                    raise Exception(
-                        f"UNREACHABLE: Workflow {workflow_id} is found in the workflow_queue table but not found in the workflow_status table"
-                    )
-                return True
+            )
+            # If no rows were affected, the workflow is not anymore in the queue or was already completed
+            return res.rowcount > 0
 
     T = TypeVar("T")
 
@@ -2012,7 +1896,6 @@ class SystemDatabase:
         inputs: str,
         *,
         max_recovery_attempts: Optional[int],
-        enqueue_options: Optional[EnqueueOptionsInternal],
     ) -> tuple[WorkflowStatuses, Optional[int]]:
         """
         Synchronously record the status and inputs for workflows in a single transaction
@@ -2021,19 +1904,7 @@ class SystemDatabase:
             wf_status, workflow_deadline_epoch_ms = self._insert_workflow_status(
                 status, conn, max_recovery_attempts=max_recovery_attempts
             )
-            # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
             self._update_workflow_inputs(status["workflow_uuid"], inputs, conn)
-
-            if (
-                status["queue_name"] is not None
-                and wf_status == WorkflowStatusString.ENQUEUED.value
-            ):
-                self._enqueue(
-                    status["workflow_uuid"],
-                    status["queue_name"],
-                    conn,
-                    enqueue_options=enqueue_options,
-                )
         return wf_status, workflow_deadline_epoch_ms
 
     def check_connection(self) -> None:

{dbos-1.3.0a2.dist-info → dbos-1.3.0a4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 1.3.0a2
+Version: 1.3.0a4
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-1.3.0a2.dist-info → dbos-1.3.0a4.dist-info}/RECORD

@@ -1,19 +1,19 @@
-dbos-1.3.0a2.dist-info/METADATA,sha256=FLGWD6E0OqgjOJ8RnpSPGTK9sAi0s4F_7_I_vfu51KE,13267
-dbos-1.3.0a2.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
-dbos-1.3.0a2.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
-dbos-1.3.0a2.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
+dbos-1.3.0a4.dist-info/METADATA,sha256=SxzfQt29v1tlpqECaOISk3fOPzN9vCsagpgQ5n3TqwM,13267
+dbos-1.3.0a4.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
+dbos-1.3.0a4.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
+dbos-1.3.0a4.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
 dbos/__init__.py,sha256=NssPCubaBxdiKarOWa-wViz1hdJSkmBGcpLX_gQ4NeA,891
 dbos/__main__.py,sha256=G7Exn-MhGrVJVDbgNlpzhfh8WMX_72t3_oJaFT9Lmt8,653
 dbos/_admin_server.py,sha256=TWXi4drrzKFpKkUmEJpJkQBZxAtOalnhtYicEn2nDK0,10618
 dbos/_app_db.py,sha256=0PKqpxJ3EbIaak3Wl0lNl3hXvhBfz4EEHaCw1bUOvIM,9937
 dbos/_classproperty.py,sha256=f0X-_BySzn3yFDRKB2JpCbLYQ9tLwt1XftfshvY7CBs,626
-dbos/_client.py,sha256=BZ5mROMnHrWyMsMj8gYCfey79Zc4eZp1Srlrgel485o,14302
+dbos/_client.py,sha256=jrvbv33KMkoACD4-wE72a6-Mp2fx0tagh_MnHY93Apk,14576
 dbos/_conductor/conductor.py,sha256=o0IaZjwnZ2TOyHeP2H4iSX6UnXLXQ4uODvWAKD9hHMs,21703
 dbos/_conductor/protocol.py,sha256=wgOFZxmS81bv0WCB9dAyg0s6QzldpzVKQDoSPeaX0Ws,6967
 dbos/_context.py,sha256=5ajoWAmToAfzzmMLylnJZoL4Ny9rBwZWuG05sXadMIA,24798
-dbos/_core.py,sha256=m2i9lsHjNKTi8BQyiSOUBrAVH5OvMoBswNZPRpMVIC0,48662
+dbos/_core.py,sha256=-yhsWn5TqDEDsIE_fY1O5qxu295kYFnz8IJnS9luhTE,48656
 dbos/_croniter.py,sha256=XHAyUyibs_59sJQfSNWkP7rqQY6_XrlfuuCxk4jYqek,47559
-dbos/_dbos.py,sha256=ccVC97MgJr979z9sc0HhVLrdi1BJ9GzaMJSKFiPyKck,47041
+dbos/_dbos.py,sha256=Y---ozwd9OXzA1o-oWEbnafQkUibblE6o1kUuVZa_90,47163
 dbos/_dbos_config.py,sha256=JWVuPE_Ifyr-pYHFxclFalB_HZ8ETFCGNJzBHGpClXw,20347
 dbos/_debug.py,sha256=MNlQVZ6TscGCRQeEEL0VE8Uignvr6dPeDDDefS3xgIE,1823
 dbos/_docker_pg_helper.py,sha256=tLJXWqZ4S-ExcaPnxg_i6cVxL6ZxrYlZjaGsklY-s2I,6115
@@ -30,6 +30,7 @@ dbos/_migrations/versions/04ca4f231047_workflow_queues_executor_id.py,sha256=ICL
 dbos/_migrations/versions/27ac6900c6ad_add_queue_dedup.py,sha256=56w1v6TdofW3V18iwm0MP0SAeSaAUPSS40HIcn6qYIE,1072
 dbos/_migrations/versions/50f3227f0b4b_fix_job_queue.py,sha256=ZBYrtTdxy64HxIAlOes89fVIk2P1gNaJack7wuC_epg,873
 dbos/_migrations/versions/5c361fc04708_added_system_tables.py,sha256=Xr9hBDJjkAtymlauOmAy00yUHj0VVUaEz7kNwEM9IwE,6403
+dbos/_migrations/versions/66478e1b95e5_consolidate_queues.py,sha256=Qo9C8pFSdN0GPM0fN-DI5GPRegXq99Mig2me04IXfLI,1894
 dbos/_migrations/versions/83f3732ae8e7_workflow_timeout.py,sha256=Q_R35pb8AfVI3sg5mzKwyoPfYB88Ychcc8gwxpM9R7A,1035
 dbos/_migrations/versions/933e86bdac6a_add_queue_priority.py,sha256=yZX2kGF33skpXIBdMXtDNx-Nl_orFatKeHB8c-3K8-c,773
 dbos/_migrations/versions/a3b18ad34abe_added_triggers.py,sha256=Rv0ZsZYZ_WdgGEULYsPfnp4YzaO5L198gDTgYY39AVA,2022
@@ -38,16 +39,16 @@ dbos/_migrations/versions/d76646551a6c_workflow_queue.py,sha256=G942nophZ2uC2vc4
 dbos/_migrations/versions/eab0cc1d9a14_job_queue.py,sha256=uvhFOtqbBreCePhAxZfIT0qCAI7BiZTou9wt6QnbY7c,1412
 dbos/_migrations/versions/f4b9b32ba814_functionname_childid_op_outputs.py,sha256=m90Lc5YH0ZISSq1MyxND6oq3RZrZKrIqEsZtwJ1jWxA,1049
 dbos/_outcome.py,sha256=EXxBg4jXCVJsByDQ1VOCIedmbeq_03S6d-p1vqQrLFU,6810
-dbos/_queue.py,sha256=oDQcydDwYM68U5KQKN6iZiSC-4LXye6KFmSJ7ohG048,3558
+dbos/_queue.py,sha256=w3vjOqU0NNdQlUFcvhhdRIzepcS2gtWJCgzobR1dJ8U,4009
 dbos/_recovery.py,sha256=jVMexjfCCNopzyn8gVQzJCmGJaP9G3C1EFaoCQ_Nh7g,2564
 dbos/_registrations.py,sha256=CZt1ElqDjCT7hz6iyT-1av76Yu-iuwu_c9lozO87wvM,7303
 dbos/_roles.py,sha256=iOsgmIAf1XVzxs3gYWdGRe1B880YfOw5fpU7Jwx8_A8,2271
 dbos/_scheduler.py,sha256=SR1oRZRcVzYsj-JauV2LA8JtwTkt8mru7qf6H1AzQ1U,2027
 dbos/_schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_schemas/application_database.py,sha256=SypAS9l9EsaBHFn9FR8jmnqt01M74d9AF1AMa4m2hhI,1040
-dbos/_schemas/system_database.py,sha256=3Z0L72bOgHnusK1hBaETWU9RfiLBP0QnS-fdu41i0yY,5835
+dbos/_schemas/system_database.py,sha256=5ebwz_h-Yz11u_eYbs8Cy8YinDpxvyiw8mEc0qjR5bo,4796
 dbos/_serialization.py,sha256=bWuwhXSQcGmiazvhJHA5gwhrRWxtmFmcCFQSDJnqqkU,3666
-dbos/_sys_db.py,sha256=KP2wIoytbtFoNzx3-SmobkAn3HhE6tZy_9Vvh8qpUQ8,85780
+dbos/_sys_db.py,sha256=wAmqy4oznS3BkG-Q0K0PnpodrecuUSP9UyXGQbTYkk4,80429
 dbos/_templates/dbos-db-starter/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
 dbos/_templates/dbos-db-starter/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_templates/dbos-db-starter/__package/main.py.dbos,sha256=aQnBPSSQpkB8ERfhf7gB7P9tsU6OPKhZscfeh0yiaD8,2702
@@ -67,4 +68,4 @@ dbos/cli/cli.py,sha256=EemOMqNpzSU2BQhAxV_e59pBRITDLwt49HF6W3uWBZg,20775
 dbos/dbos-config.schema.json,sha256=CjaspeYmOkx6Ip_pcxtmfXJTn_YGdSx_0pcPBF7KZmo,6060
 dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
 version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
-dbos-1.3.0a2.dist-info/RECORD,,
+dbos-1.3.0a4.dist-info/RECORD,,