dbos 0.8.0a3__py3-none-any.whl → 0.8.0a10__py3-none-any.whl

dbos/core.py

@@ -38,6 +38,7 @@ from dbos.error import (
     DBOSWorkflowFunctionNotFoundError,
 )
 from dbos.registrations import (
+    DEFAULT_MAX_RECOVERY_ATTEMPTS,
     get_config_name,
     get_dbos_class_name,
     get_dbos_func_name,
@@ -118,6 +119,7 @@ def _init_workflow(
     config_name: Optional[str],
     temp_wf_type: Optional[str],
     queue: Optional[str] = None,
+    max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS,
 ) -> WorkflowStatusInternal:
     wfid = (
         ctx.workflow_id
@@ -157,7 +159,9 @@ def _init_workflow(
         # Synchronously record the status and inputs for workflows and single-step workflows
         # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
         # TODO: Make this transactional (and with the queue step below)
-        dbos._sys_db.update_workflow_status(status, False, ctx.in_recovery)
+        dbos._sys_db.update_workflow_status(
+            status, False, ctx.in_recovery, max_recovery_attempts=max_recovery_attempts
+        )
         dbos._sys_db.update_workflow_inputs(wfid, utils.serialize_args(inputs))
     else:
         # Buffer the inputs for single-transaction workflows, but don't buffer the status
@@ -181,7 +185,8 @@ def _execute_workflow(
         status["status"] = "SUCCESS"
         status["output"] = utils.serialize(output)
         if status["queue_name"] is not None:
-            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+            queue = dbos._registry.queue_info_map[status["queue_name"]]
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
         dbos._sys_db.buffer_workflow_status(status)
     except DBOSWorkflowConflictIDError:
         # Retrieve the workflow handle and wait for the result.
@@ -195,7 +200,8 @@ def _execute_workflow(
         status["status"] = "ERROR"
         status["error"] = utils.serialize_exception(error)
         if status["queue_name"] is not None:
-            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+            queue = dbos._registry.queue_info_map[status["queue_name"]]
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
         dbos._sys_db.update_workflow_status(status)
         raise
 
@@ -218,7 +224,7 @@ def _execute_workflow_wthread(
         with EnterDBOSWorkflow(attributes):
             try:
                 return _execute_workflow(dbos, status, func, *args, **kwargs)
-            except Exception as e:
+            except Exception:
                 dbos.logger.error(
                     f"Exception encountered in asynchronous workflow: {traceback.format_exc()}"
                 )
@@ -289,10 +295,15 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
                 )
 
 
-def _workflow_wrapper(dbosreg: "_DBOSRegistry", func: F) -> F:
+def _workflow_wrapper(
+    dbosreg: "_DBOSRegistry",
+    func: F,
+    max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS,
+) -> F:
     func.__orig_func = func  # type: ignore
 
     fi = get_or_create_func_info(func)
+    fi.max_recovery_attempts = max_recovery_attempts
 
     @wraps(func)
     def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -325,17 +336,21 @@ def _workflow_wrapper(dbosreg: "_DBOSRegistry", func: F) -> F:
                 class_name=get_dbos_class_name(fi, func, args),
                 config_name=get_config_name(fi, func, args),
                 temp_wf_type=get_temp_workflow_type(func),
+                max_recovery_attempts=max_recovery_attempts,
             )
 
+            dbos.logger.debug(
+                f"Running workflow, id: {ctx.workflow_id}, name: {get_dbos_func_name(func)}"
+            )
             return _execute_workflow(dbos, status, func, *args, **kwargs)
 
     wrapped_func = cast(F, wrapper)
     return wrapped_func
 
 
-def _workflow(reg: "_DBOSRegistry") -> Callable[[F], F]:
+def _workflow(reg: "_DBOSRegistry", max_recovery_attempts: int) -> Callable[[F], F]:
     def _workflow_decorator(func: F) -> F:
-        wrapped_func = _workflow_wrapper(reg, func)
+        wrapped_func = _workflow_wrapper(reg, func, max_recovery_attempts)
         reg.register_wf_function(func.__qualname__, wrapped_func)
         return wrapped_func
 
@@ -401,6 +416,7 @@ def _start_workflow(
         config_name=get_config_name(fi, func, gin_args),
         temp_wf_type=get_temp_workflow_type(func),
         queue=queue_name,
+        max_recovery_attempts=fi.max_recovery_attempts,
     )
 
     if not execute_workflow:
@@ -477,6 +493,9 @@ def _transaction(
                                     )
                                 )
                                 if recorded_output:
+                                    dbos.logger.debug(
+                                        f"Replaying transaction, id: {ctx.function_id}, name: {attributes['name']}"
+                                    )
                                     if recorded_output["error"]:
                                         deserialized_error = (
                                             utils.deserialize_exception(
@@ -493,6 +512,11 @@ def _transaction(
                                         raise Exception(
                                             "Output and error are both None"
                                         )
+                                else:
+                                    dbos.logger.debug(
+                                        f"Running transaction, id: {ctx.function_id}, name: {attributes['name']}"
+                                    )
+
                                 output = func(*args, **kwargs)
                                 txn_output["output"] = utils.serialize(output)
                                 assert (
@@ -590,6 +614,9 @@ def _step(
                     ctx.workflow_id, ctx.function_id
                 )
                 if recorded_output:
+                    dbos.logger.debug(
+                        f"Replaying step, id: {ctx.function_id}, name: {attributes['name']}"
+                    )
                     if recorded_output["error"] is not None:
                         deserialized_error = utils.deserialize_exception(
                             recorded_output["error"]
@@ -599,6 +626,11 @@ def _step(
                         return utils.deserialize(recorded_output["output"])
                     else:
                         raise Exception("Output and error are both None")
+                else:
+                    dbos.logger.debug(
+                        f"Running step, id: {ctx.function_id}, name: {attributes['name']}"
+                    )
+
                 output = None
                 error = None
                 local_max_attempts = max_attempts if retries_allowed else 1

dbos/dbos.py

@@ -5,6 +5,7 @@ import json
 import os
 import sys
 import threading
+import traceback
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from logging import Logger
@@ -42,6 +43,7 @@ from dbos.decorators import classproperty
 from dbos.queue import Queue, queue_thread
 from dbos.recovery import _recover_pending_workflows, _startup_recovery_thread
 from dbos.registrations import (
+    DEFAULT_MAX_RECOVERY_ATTEMPTS,
     DBOSClassInfo,
     get_or_create_class_info,
     set_dbos_func_name,
@@ -338,43 +340,47 @@ class DBOS:
             _dbos_global_instance._launch()
 
     def _launch(self) -> None:
-        if self._launched:
-            dbos_logger.warning(f"DBOS was already launched")
-            return
-        self._launched = True
-        self._executor_field = ThreadPoolExecutor(max_workers=64)
-        self._sys_db_field = SystemDatabase(self.config)
-        self._app_db_field = ApplicationDatabase(self.config)
-        self._admin_server_field = AdminServer(dbos=self)
-
-        if not os.environ.get("DBOS__VMID"):
-            workflow_ids = self._sys_db.get_pending_workflows("local")
-            self._executor.submit(_startup_recovery_thread, self, workflow_ids)
-
-        # Listen to notifications
-        self._executor.submit(self._sys_db._notification_listener)
-
-        # Start flush workflow buffers thread
-        self._executor.submit(self._sys_db.flush_workflow_buffers)
-
-        # Start the queue thread
-        evt = threading.Event()
-        self.stop_events.append(evt)
-        self._executor.submit(queue_thread, evt, self)
-
-        # Grab any pollers that were deferred and start them
-        for evt, func, args, kwargs in self._registry.pollers:
+        try:
+            if self._launched:
+                dbos_logger.warning(f"DBOS was already launched")
+                return
+            self._launched = True
+            self._executor_field = ThreadPoolExecutor(max_workers=64)
+            self._sys_db_field = SystemDatabase(self.config)
+            self._app_db_field = ApplicationDatabase(self.config)
+            self._admin_server_field = AdminServer(dbos=self)
+
+            if not os.environ.get("DBOS__VMID"):
+                workflow_ids = self._sys_db.get_pending_workflows("local")
+                self._executor.submit(_startup_recovery_thread, self, workflow_ids)
+
+            # Listen to notifications
+            self._executor.submit(self._sys_db._notification_listener)
+
+            # Start flush workflow buffers thread
+            self._executor.submit(self._sys_db.flush_workflow_buffers)
+
+            # Start the queue thread
+            evt = threading.Event()
             self.stop_events.append(evt)
-            self._executor.submit(func, *args, **kwargs)
-        self._registry.pollers = []
+            self._executor.submit(queue_thread, evt, self)
 
-        dbos_logger.info("DBOS launched")
+            # Grab any pollers that were deferred and start them
+            for evt, func, args, kwargs in self._registry.pollers:
+                self.stop_events.append(evt)
+                self._executor.submit(func, *args, **kwargs)
+            self._registry.pollers = []
 
-        # Flush handlers and add OTLP to all loggers if enabled
-        # to enable their export in DBOS Cloud
-        for handler in dbos_logger.handlers:
-            handler.flush()
-        add_otlp_to_all_loggers()
+            dbos_logger.info("DBOS launched")
+
+            # Flush handlers and add OTLP to all loggers if enabled
+            # to enable their export in DBOS Cloud
+            for handler in dbos_logger.handlers:
+                handler.flush()
+            add_otlp_to_all_loggers()
+        except Exception:
+            dbos_logger.error(f"DBOS failed to launch: {traceback.format_exc()}")
+            raise
 
     def _destroy(self) -> None:
         self._initialized = False
@@ -401,9 +407,11 @@ class DBOS:
 
     # Decorators for DBOS functionality
     @classmethod
-    def workflow(cls) -> Callable[[F], F]:
+    def workflow(
+        cls, *, max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS
+    ) -> Callable[[F], F]:
         """Decorate a function for use as a DBOS workflow."""
-        return _workflow(_get_or_create_dbos_registry())
+        return _workflow(_get_or_create_dbos_registry(), max_recovery_attempts)
 
     @classmethod
     def transaction(
@@ -542,6 +550,7 @@ class DBOS:
             recovery_attempts=stat["recovery_attempts"],
             class_name=stat["class_name"],
             config_name=stat["config_name"],
+            queue_name=stat["queue_name"],
             authenticated_user=stat["authenticated_user"],
             assumed_role=stat["assumed_role"],
             authenticated_roles=(
@@ -748,6 +757,7 @@ class WorkflowStatus:
         name(str): The workflow function name
         class_name(str): For member functions, the name of the class containing the workflow function
         config_name(str): For instance member functions, the name of the class instance for the execution
+        queue_name(str): For workflows that are or were queued, the queue name
         authenticated_user(str): The user who invoked the workflow
         assumed_role(str): The access role used by the user to allow access to the workflow function
         authenticated_roles(List[str]): List of all access roles available to the authenticated user
@@ -760,6 +770,7 @@ class WorkflowStatus:
     name: str
     class_name: Optional[str]
     config_name: Optional[str]
+    queue_name: Optional[str]
     authenticated_user: Optional[str]
     assumed_role: Optional[str]
     authenticated_roles: Optional[List[str]]

dbos/error.py

@@ -32,6 +32,7 @@ class DBOSErrorCode(Enum):
     InitializationError = 3
     WorkflowFunctionNotFound = 4
     NonExistentWorkflowError = 5
+    DeadLetterQueueError = 6
     MaxStepRetriesExceeded = 7
     NotAuthorized = 8
 
@@ -86,6 +87,16 @@ class DBOSNonExistentWorkflowError(DBOSException):
         )
 
 
+class DBOSDeadLetterQueueError(DBOSException):
+    """Exception raised when a workflow database record does not exist for a given ID."""
+
+    def __init__(self, wf_id: str, max_retries: int):
+        super().__init__(
+            f"Workflow {wf_id} has been moved to the dead-letter queue after exceeding the maximum of ${max_retries} retries",
+            dbos_error_code=DBOSErrorCode.DeadLetterQueueError.value,
+        )
+
+
 class DBOSNotAuthorizedError(DBOSException):
     """Exception raised by DBOS role-based security when the user is not authorized to access a function."""
 

dbos/migrations/versions/50f3227f0b4b_fix_job_queue.py

@@ -1,4 +1,5 @@
-"""fix_job_queue
+"""
+Fix job queue PK.
 
 Revision ID: 50f3227f0b4b
 Revises: eab0cc1d9a14

dbos/migrations/versions/d76646551a6b_job_queue_limiter.py

@@ -0,0 +1,43 @@
+"""
+Adjust workflow queue to add columns for rate limiter.
+
+Revision ID: d76646551a6b
+Revises: 50f3227f0b4b
+Create Date: 2024-09-25 14:48:10.218015
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d76646551a6b"
+down_revision: Union[str, None] = "50f3227f0b4b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "job_queue",
+        sa.Column(
+            "started_at_epoch_ms",
+            sa.BigInteger(),
+        ),
+        schema="dbos",
+    )
+    op.add_column(
+        "job_queue",
+        sa.Column(
+            "completed_at_epoch_ms",
+            sa.BigInteger(),
+        ),
+        schema="dbos",
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("job_queue", "started_at_epoch_ms", schema="dbos")
+    op.drop_column("job_queue", "completed_at_epoch_ms", schema="dbos")

dbos/migrations/versions/d76646551a6c_workflow_queue.py

@@ -0,0 +1,28 @@
+"""workflow_queue
+
+Revision ID: d76646551a6c
+Revises: d76646551a6b
+Create Date: 2024-09-27 12:00:00.0
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d76646551a6c"
+down_revision: Union[str, None] = "d76646551a6b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.rename_table("job_queue", "workflow_queue", schema="dbos")
+    op.execute("CREATE VIEW dbos.job_queue AS SELECT * FROM dbos.workflow_queue;")
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW dbos.job_queue;")
+    op.rename_table("workflow_queue", "job_queue", schema="dbos")

dbos/migrations/versions/eab0cc1d9a14_job_queue.py

@@ -1,4 +1,5 @@
-"""job_queue
+"""
+Add workflow queue table.
 
 Revision ID: eab0cc1d9a14
 Revises: a3b18ad34abe

dbos/queue.py

@@ -1,7 +1,6 @@
 import threading
-import time
 import traceback
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, TypedDict
 
 from dbos.core import P, R, _execute_workflow_id, _start_workflow
 
@@ -9,10 +8,35 @@ if TYPE_CHECKING:
     from dbos.dbos import DBOS, Workflow, WorkflowHandle
 
 
+class QueueRateLimit(TypedDict):
+    """
+    Limit the maximum number of workflows from this queue that can be started in a given period.
+
+    If the limit is 5 and the period is 10, no more than 5 functions can be
+    started per 10 seconds.
+    """
+
+    limit: int
+    period: float
+
+
 class Queue:
-    def __init__(self, name: str, concurrency: Optional[int] = None) -> None:
+    """
+    Workflow queue.
+
+    Workflow queues allow workflows to be started at a later time, based on concurrency and
+    rate limits.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        concurrency: Optional[int] = None,
+        limiter: Optional[QueueRateLimit] = None,
+    ) -> None:
         self.name = name
         self.concurrency = concurrency
+        self.limiter = limiter
         from dbos.dbos import _get_or_create_dbos_registry
 
         registry = _get_or_create_dbos_registry()
@@ -29,12 +53,11 @@ class Queue:
 
 def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
     while not stop_event.is_set():
-        time.sleep(1)
-        for queue_name, queue in dbos._registry.queue_info_map.items():
+        if stop_event.wait(timeout=1):
+            return
+        for _, queue in dbos._registry.queue_info_map.items():
             try:
-                wf_ids = dbos._sys_db.start_queued_workflows(
-                    queue_name, queue.concurrency
-                )
+                wf_ids = dbos._sys_db.start_queued_workflows(queue)
                 for id in wf_ids:
                     _execute_workflow_id(dbos, id)
             except Exception:

dbos/registrations.py

@@ -3,6 +3,8 @@ from enum import Enum
 from types import FunctionType
 from typing import Any, Callable, List, Literal, Optional, Tuple, Type, cast
 
+DEFAULT_MAX_RECOVERY_ATTEMPTS = 50
+
 
 def get_dbos_func_name(f: Any) -> str:
     if hasattr(f, "dbos_function_name"):
@@ -47,6 +49,7 @@ class DBOSFuncInfo:
         self.class_info: Optional[DBOSClassInfo] = None
         self.func_type: DBOSFuncType = DBOSFuncType.Unknown
         self.required_roles: Optional[List[str]] = None
+        self.max_recovery_attempts = DEFAULT_MAX_RECOVERY_ATTEMPTS
 
 
 def get_or_create_class_info(cls: Type[Any]) -> DBOSClassInfo:

dbos/request.py

@@ -13,6 +13,7 @@ class Address(NamedTuple):
 class Request:
     """
     Serializable HTTP Request object.
+
     Attributes:
         base_url(str): Base of URL requested, as in application code
         client(Optional[Address]): HTTP Client

dbos/scheduler/scheduler.py

@@ -2,6 +2,7 @@ import threading
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Callable
 
+from dbos.logger import dbos_logger
 from dbos.queue import Queue
 
 if TYPE_CHECKING:
@@ -18,7 +19,12 @@ scheduler_queue: Queue
 def scheduler_loop(
     func: ScheduledWorkflow, cron: str, stop_event: threading.Event
 ) -> None:
-    iter = croniter(cron, datetime.now(timezone.utc), second_at_beginning=True)
+    try:
+        iter = croniter(cron, datetime.now(timezone.utc), second_at_beginning=True)
+    except Exception as e:
+        dbos_logger.error(
+            f'Cannot run scheduled function {func.__name__}. Invalid crontab "{cron}"'
+        )
     while not stop_event.is_set():
         nextExecTime = iter.get_next(datetime)
         sleepTime = nextExecTime - datetime.now(timezone.utc)

dbos/schemas/system_database.py

@@ -142,8 +142,8 @@ class SystemSchema:
         Column("last_run_time", BigInteger, nullable=False),
     )
 
-    job_queue = Table(
-        "job_queue",
+    workflow_queue = Table(
+        "workflow_queue",
         metadata_obj,
         Column(
             "workflow_uuid",
@@ -161,4 +161,12 @@ class SystemSchema:
             nullable=False,
             server_default=text("(EXTRACT(epoch FROM now()) * 1000::numeric)::bigint"),
         ),
+        Column(
+            "started_at_epoch_ms",
+            BigInteger(),
+        ),
+        Column(
+            "completed_at_epoch_ms",
+            BigInteger(),
+        ),
     )

dbos/system_database.py

@@ -4,6 +4,7 @@ import threading
 import time
 from enum import Enum
 from typing import (
+    TYPE_CHECKING,
     Any,
     Dict,
     List,
@@ -24,12 +25,20 @@ from alembic.config import Config
 from sqlalchemy.exc import DBAPIError
 
 import dbos.utils as utils
-from dbos.error import DBOSNonExistentWorkflowError, DBOSWorkflowConflictIDError
+from dbos.error import (
+    DBOSDeadLetterQueueError,
+    DBOSNonExistentWorkflowError,
+    DBOSWorkflowConflictIDError,
+)
+from dbos.registrations import DEFAULT_MAX_RECOVERY_ATTEMPTS
 
 from .dbos_config import ConfigFile
 from .logger import dbos_logger
 from .schemas.system_database import SystemSchema
 
+if TYPE_CHECKING:
+    from .queue import Queue
+
 
 class WorkflowStatusString(Enum):
     """Enumeration of values allowed for `WorkflowSatusInternal.status`."""
@@ -233,7 +242,9 @@ class SystemDatabase:
         status: WorkflowStatusInternal,
         replace: bool = True,
         in_recovery: bool = False,
+        *,
         conn: Optional[sa.Connection] = None,
+        max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS,
     ) -> None:
         cmd = pg.insert(SystemSchema.workflow_status).values(
             workflow_uuid=status["workflow_uuid"],
@@ -271,12 +282,36 @@ class SystemDatabase:
             )
         else:
             cmd = cmd.on_conflict_do_nothing()
+        cmd = cmd.returning(SystemSchema.workflow_status.c.recovery_attempts)  # type: ignore
 
         if conn is not None:
-            conn.execute(cmd)
+            results = conn.execute(cmd)
         else:
             with self.engine.begin() as c:
-                c.execute(cmd)
+                results = c.execute(cmd)
+        if in_recovery:
+            row = results.fetchone()
+            if row is not None:
+                recovery_attempts: int = row[0]
+                if recovery_attempts > max_recovery_attempts:
+                    with self.engine.begin() as c:
+                        c.execute(
+                            sa.update(SystemSchema.workflow_status)
+                            .where(
+                                SystemSchema.workflow_status.c.workflow_uuid
+                                == status["workflow_uuid"]
+                            )
+                            .where(
+                                SystemSchema.workflow_status.c.status
+                                == WorkflowStatusString.PENDING.value
+                            )
+                            .values(
+                                status=WorkflowStatusString.RETRIES_EXCEEDED.value,
+                            )
+                        )
+                    raise DBOSDeadLetterQueueError(
+                        status["workflow_uuid"], max_recovery_attempts
+                    )
 
         # Record we have exported status for this single-transaction workflow
         if status["workflow_uuid"] in self._temp_txn_wf_ids:
@@ -617,7 +652,14 @@ class SystemDatabase:
                 workflow_uuid, function_id, conn=c
             )
             if recorded_output is not None:
+                dbos_logger.debug(
+                    f"Replaying send, id: {function_id}, destination_uuid: {destination_uuid}, topic: {topic}"
+                )
                 return  # Already sent before
+            else:
+                dbos_logger.debug(
+                    f"Running send, id: {function_id}, destination_uuid: {destination_uuid}, topic: {topic}"
+                )
 
             try:
                 c.execute(
@@ -653,10 +695,13 @@ class SystemDatabase:
         # First, check for previous executions.
         recorded_output = self.check_operation_execution(workflow_uuid, function_id)
         if recorded_output is not None:
+            dbos_logger.debug(f"Replaying recv, id: {function_id}, topic: {topic}")
             if recorded_output["output"] is not None:
                 return utils.deserialize(recorded_output["output"])
             else:
                 raise Exception("No output recorded in the last recv")
+        else:
+            dbos_logger.debug(f"Running recv, id: {function_id}, topic: {topic}")
 
         # Insert a condition to the notifications map, so the listener can notify it when a message is received.
         payload = f"{workflow_uuid}::{topic}"
@@ -799,9 +844,11 @@ class SystemDatabase:
         recorded_output = self.check_operation_execution(workflow_uuid, function_id)
         end_time: float
         if recorded_output is not None:
+            dbos_logger.debug(f"Replaying sleep, id: {function_id}, seconds: {seconds}")
             assert recorded_output["output"] is not None, "no recorded end time"
             end_time = utils.deserialize(recorded_output["output"])
         else:
+            dbos_logger.debug(f"Running sleep, id: {function_id}, seconds: {seconds}")
             end_time = time.time() + seconds
             try:
                 self.record_operation_result(
@@ -831,7 +878,10 @@ class SystemDatabase:
                 workflow_uuid, function_id, conn=c
             )
             if recorded_output is not None:
+                dbos_logger.debug(f"Replaying set_event, id: {function_id}, key: {key}")
                 return  # Already sent before
+            else:
+                dbos_logger.debug(f"Running set_event, id: {function_id}, key: {key}")
 
             c.execute(
                 pg.insert(SystemSchema.workflow_events)
@@ -872,10 +922,17 @@ class SystemDatabase:
                 caller_ctx["workflow_uuid"], caller_ctx["function_id"]
             )
             if recorded_output is not None:
+                dbos_logger.debug(
+                    f"Replaying get_event, id: {caller_ctx['function_id']}, key: {key}"
+                )
                 if recorded_output["output"] is not None:
                     return utils.deserialize(recorded_output["output"])
                 else:
                     raise Exception("No output recorded in the last get_event")
+            else:
+                dbos_logger.debug(
+                    f"Running get_event, id: {caller_ctx['function_id']}, key: {key}"
+                )
 
         payload = f"{target_uuid}::{key}"
         condition = threading.Condition()
@@ -926,7 +983,7 @@ class SystemDatabase:
         return value
 
     def _flush_workflow_status_buffer(self) -> None:
-        """Export the workflow status buffer to the database, up to the batch size"""
+        """Export the workflow status buffer to the database, up to the batch size."""
         if len(self._workflow_status_buffer) == 0:
             return
 
@@ -1022,7 +1079,7 @@ class SystemDatabase:
     def enqueue(self, workflow_id: str, queue_name: str) -> None:
         with self.engine.begin() as c:
             c.execute(
-                pg.insert(SystemSchema.job_queue)
+                pg.insert(SystemSchema.workflow_queue)
                 .values(
                     workflow_uuid=workflow_id,
                     queue_name=queue_name,
@@ -1030,22 +1087,63 @@ class SystemDatabase:
                 .on_conflict_do_nothing()
             )
 
-    def start_queued_workflows(
-        self, queue_name: str, concurrency: Optional[int]
-    ) -> List[str]:
+    def start_queued_workflows(self, queue: "Queue") -> List[str]:
+        start_time_ms = int(time.time() * 1000)
+        if queue.limiter is not None:
+            limiter_period_ms = int(queue.limiter["period"] * 1000)
         with self.engine.begin() as c:
-            query = sa.select(SystemSchema.job_queue.c.workflow_uuid).where(
-                SystemSchema.job_queue.c.queue_name == queue_name
+            # Execute with snapshot isolation to ensure multiple workers respect limits
+            c.execute(sa.text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"))
+
+            # If there is a limiter, compute how many functions have started in its period.
+            if queue.limiter is not None:
+                query = (
+                    sa.select(sa.func.count())
+                    .select_from(SystemSchema.workflow_queue)
+                    .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
+                    .where(
+                        SystemSchema.workflow_queue.c.started_at_epoch_ms.isnot(None)
+                    )
+                    .where(
+                        SystemSchema.workflow_queue.c.started_at_epoch_ms
+                        > start_time_ms - limiter_period_ms
+                    )
+                )
+                num_recent_queries = c.execute(query).fetchone()[0]  # type: ignore
+                if num_recent_queries >= queue.limiter["limit"]:
+                    return []
+
+            # Select not-yet-completed functions in the queue ordered by the
+            # time at which they were enqueued.
+            # If there is a concurrency limit N, select only the N most recent
+            # functions, else select all of them.
+            query = (
+                sa.select(
+                    SystemSchema.workflow_queue.c.workflow_uuid,
+                    SystemSchema.workflow_queue.c.started_at_epoch_ms,
+                )
+                .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
+                .where(SystemSchema.workflow_queue.c.completed_at_epoch_ms == None)
+                .order_by(SystemSchema.workflow_queue.c.created_at_epoch_ms.asc())
             )
-            if concurrency is not None:
-                query = query.order_by(
-                    SystemSchema.job_queue.c.created_at_epoch_ms.asc()
-                ).limit(concurrency)
+            if queue.concurrency is not None:
+                query = query.limit(queue.concurrency)
+
+            # From the functions retrieved, get the workflow IDs of the functions
+            # that have not yet been started so we can start them.
             rows = c.execute(query).fetchall()
-            dequeued_ids: List[str] = [row[0] for row in rows]
-            ret_ids = []
+            dequeued_ids: List[str] = [row[0] for row in rows if row[1] is None]
+            ret_ids: list[str] = []
             for id in dequeued_ids:
-                result = c.execute(
+
+                # If we have a limiter, stop starting functions when the number
+                # of functions started this period exceeds the limit.
+                if queue.limiter is not None:
+                    if len(ret_ids) + num_recent_queries >= queue.limiter["limit"]:
+                        break
+
+                # To start a function, first set its status to PENDING
+                c.execute(
                     SystemSchema.workflow_status.update()
                     .where(SystemSchema.workflow_status.c.workflow_uuid == id)
                     .where(
@@ -1054,14 +1152,43 @@ class SystemDatabase:
                     )
                     .values(status=WorkflowStatusString.PENDING.value)
                 )
-                if result.rowcount > 0:
-                    ret_ids.append(id)
+
+                # Then give it a start time
+                c.execute(
+                    SystemSchema.workflow_queue.update()
+                    .where(SystemSchema.workflow_queue.c.workflow_uuid == id)
+                    .values(started_at_epoch_ms=start_time_ms)
+                )
+                ret_ids.append(id)
+
+            # If we have a limiter, garbage-collect all completed functions started
+            # before the period. If there's no limiter, there's no need--they were
+            # deleted on completion.
+            if queue.limiter is not None:
+                c.execute(
+                    sa.delete(SystemSchema.workflow_queue)
+                    .where(SystemSchema.workflow_queue.c.completed_at_epoch_ms != None)
+                    .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
+                    .where(
+                        SystemSchema.workflow_queue.c.started_at_epoch_ms
+                        < start_time_ms - limiter_period_ms
+                    )
+                )
+
+            # Return the IDs of all functions we started
             return ret_ids
 
-    def remove_from_queue(self, workflow_id: str) -> None:
+    def remove_from_queue(self, workflow_id: str, queue: "Queue") -> None:
         with self.engine.begin() as c:
-            c.execute(
-                sa.delete(SystemSchema.job_queue).where(
-                    SystemSchema.job_queue.c.workflow_uuid == workflow_id
+            if queue.limiter is None:
+                c.execute(
+                    sa.delete(SystemSchema.workflow_queue).where(
+                        SystemSchema.workflow_queue.c.workflow_uuid == workflow_id
+                    )
+                )
+            else:
+                c.execute(
+                    sa.update(SystemSchema.workflow_queue)
+                    .where(SystemSchema.workflow_queue.c.workflow_uuid == workflow_id)
+                    .values(completed_at_epoch_ms=int(time.time() * 1000))
                 )
-            )

{dbos-0.8.0a3.dist-info → dbos-0.8.0a10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.8.0a3
+Version: 0.8.0a10
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.8.0a3.dist-info → dbos-0.8.0a10.dist-info}/RECORD

@@ -1,18 +1,18 @@
-dbos-0.8.0a3.dist-info/METADATA,sha256=u3PDfNC0j5qeu05Uq1834FHAScD6ECq64z9_XOQqdiw,5010
-dbos-0.8.0a3.dist-info/WHEEL,sha256=Vza3XR51HW1KmFP0iIMUVYIvz0uQuKJpIXKYOBGQyFQ,90
-dbos-0.8.0a3.dist-info/entry_points.txt,sha256=z6GcVANQV7Uw_82H9Ob2axJX6V3imftyZsljdh-M1HU,54
-dbos-0.8.0a3.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
+dbos-0.8.0a10.dist-info/METADATA,sha256=Ql94ETlUkbD-PYbjc6ZvVlXj42jnHlLZqp-we0XwkF8,5011
+dbos-0.8.0a10.dist-info/WHEEL,sha256=Vza3XR51HW1KmFP0iIMUVYIvz0uQuKJpIXKYOBGQyFQ,90
+dbos-0.8.0a10.dist-info/entry_points.txt,sha256=z6GcVANQV7Uw_82H9Ob2axJX6V3imftyZsljdh-M1HU,54
+dbos-0.8.0a10.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
 dbos/__init__.py,sha256=-h1QgWNL11CiLlHEKa2ycAJVJw5SXYZ4BGNNWBAiE9k,726
 dbos/admin_sever.py,sha256=Qg5T3YRrbPW05PR_99yAaxgo1ugQrAp_uTeTqSfjm_k,3397
 dbos/application_database.py,sha256=knFK8We8y6WrIpnFCKvFq5hvSuFQqUuJqOqDpSVMCPI,5521
 dbos/cli.py,sha256=z5dXbbnGWzSC3E1rfS8Lp1_OIImzcDKM7jP-iu_Q4aI,8602
 dbos/context.py,sha256=4MsxZdoh1WIsgoUsaxo0B6caGN6xq2WC60MzbBppzGk,17738
-dbos/core.py,sha256=nV0w0wCkKm0VRVbUO0DgRnVqX5ue1Bn37UyAsdSXl48,29342
+dbos/core.py,sha256=ggsRC2XicvNI1qqruEFoqxoTU5oSSnhMZvDih3AG_3A,30879
 dbos/dbos-config.schema.json,sha256=azpfmoDZg7WfSy3kvIsk9iEiKB_-VZt03VEOoXJAkqE,5331
-dbos/dbos.py,sha256=RtDcvKe4sm1TlnCGU4cyex-UI7hMMlhgzmOl1NuRLo4,29294
+dbos/dbos.py,sha256=qQOHcgaT4rru1gZJy1nFrIvz8YzEhDfMJmN9eCskPZ4,29857
 dbos/dbos_config.py,sha256=NJVze2GkKgYUmcPP31Unb-QpsA0TzImEeQGJgVq6W6k,5352
 dbos/decorators.py,sha256=lbPefsLK6Cya4cb7TrOcLglOpGT3pc6qjZdsQKlfZLg,629
-dbos/error.py,sha256=c2y7d3Cbb-ZOO-M9txcCxuyPE1bdnwKrJnXgJXYb-pQ,3437
+dbos/error.py,sha256=UETk8CoZL-TO2Utn1-E7OSWelhShWmKM-fOlODMR9PE,3893
 dbos/fastapi.py,sha256=gx9hlpxYOiwbuhSlbY9bn5C-F_FsCbrJvkX9ZAvDG6U,3418
 dbos/flask.py,sha256=azr4geMEGuuTBCyxIZmgDmmP-6s_pTIF-lGyp9Q4IB8,2430
 dbos/kafka.py,sha256=LH3hbNapnkjLcuXNUtdGU0398JafWb-t0GwUl3LOzkc,3645
@@ -20,22 +20,24 @@ dbos/kafka_message.py,sha256=NYvOXNG3Qn7bghn1pv3fg4Pbs86ILZGcK4IB-MLUNu0,409
 dbos/logger.py,sha256=D-aFSZUCHBP34J1IZ5YNkTrJW-rDiH3py_v9jLU4Yrk,3565
 dbos/migrations/env.py,sha256=38SIGVbmn_VV2x2u1aHLcPOoWgZ84eCymf3g_NljmbU,1626
 dbos/migrations/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvDgl93dMj8,635
-dbos/migrations/versions/50f3227f0b4b_fix_job_queue.py,sha256=ZtnsZFMuon-D0n8V5BR10jQEqJPUsYsOwt29FAoKG8g,868
+dbos/migrations/versions/50f3227f0b4b_fix_job_queue.py,sha256=ZBYrtTdxy64HxIAlOes89fVIk2P1gNaJack7wuC_epg,873
 dbos/migrations/versions/5c361fc04708_added_system_tables.py,sha256=QMgFMb0aLgC25YicsvPSr6AHRCA6Zd66hyaRUhwKzrQ,6404
 dbos/migrations/versions/a3b18ad34abe_added_triggers.py,sha256=Rv0ZsZYZ_WdgGEULYsPfnp4YzaO5L198gDTgYY39AVA,2022
-dbos/migrations/versions/eab0cc1d9a14_job_queue.py,sha256=_9-FCW-zOpCQfblTS_yRLtFiUaWlC1tM4BoKBTDeH9k,1395
+dbos/migrations/versions/d76646551a6b_job_queue_limiter.py,sha256=8PyFi8rd6CN-mUro43wGhsg5wcQWKZPRHD6jw8R5pVc,986
+dbos/migrations/versions/d76646551a6c_workflow_queue.py,sha256=G942nophZ2uC2vc4hGBC02Ptng1715roTjY3xiyzZU4,729
+dbos/migrations/versions/eab0cc1d9a14_job_queue.py,sha256=uvhFOtqbBreCePhAxZfIT0qCAI7BiZTou9wt6QnbY7c,1412
 dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
-dbos/queue.py,sha256=ngY1MN3xD7jAvEXlrl_D16FUPpP_vpRgbyERLfPyU9Y,1437
+dbos/queue.py,sha256=DT5dFIDZGnC4GpgI1Tph8fh5VvClpokugv-2ow4qyiQ,1947
 dbos/recovery.py,sha256=zqtO_ExGoIErLMVnbneU3VeHLVWvhV4jnfqssAVlQQk,2016
-dbos/registrations.py,sha256=gMI-u05tv5bpvyddQGtoUgCsqARx51aOY7p0JXPafQo,6539
-dbos/request.py,sha256=-FIwtknayvRl6OjvqO4V2GySVzSdP1Ft3cc9ZBS-PLY,928
+dbos/registrations.py,sha256=mei6q6_3R5uei8i_Wo_TqGZs85s10shOekDX41sFYD0,6642
+dbos/request.py,sha256=cX1B3Atlh160phgS35gF1VEEV4pD126c9F3BDgBmxZU,929
 dbos/roles.py,sha256=7Lh7uwUq1dpa6TXCOHre4mPTd5qmXzK_QPkvYR52DXg,2285
 dbos/scheduler/croniter.py,sha256=hbhgfsHBqclUS8VeLnJ9PSE9Z54z6mi4nnrr1aUXn0k,47561
-dbos/scheduler/scheduler.py,sha256=Sz4EIpAtur7so2YajTic64GrTpa4qPw8QxXn0M34v80,1360
+dbos/scheduler/scheduler.py,sha256=KpcBid6qIbqLqLdrQQqEQnRBTvo_XwtVuvUba3Ed5Go,1560
 dbos/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/schemas/application_database.py,sha256=q_Wr2XbiZNBYFkOtu7uKavo1T_cSOBblxKGHThYGGsY,962
-dbos/schemas/system_database.py,sha256=ed4c1UntsD-cqXD0ekM4jvcYYEViavDh_G6c0pVDe7k,4938
-dbos/system_database.py,sha256=Os1-qqnrirKm_K37zuMgk14mm4ziXNYDjZnM1o8A450,41864
+dbos/schemas/system_database.py,sha256=7iw7eHJzEvkatHMOaHORoSvtfisF73wW5j8hRt_Ph14,5126
+dbos/system_database.py,sha256=jS0JV3HW2nxKlCVAahTkraXpUpfy1pv2eirzQFpX6J4,48067
 dbos/templates/hello/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
 dbos/templates/hello/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/templates/hello/__package/main.py,sha256=eI0SS9Nwj-fldtiuSzIlIG6dC91GXXwdRsoHxv6S_WI,2719
@@ -49,4 +51,4 @@ dbos/templates/hello/start_postgres_docker.py,sha256=lQVLlYO5YkhGPEgPqwGc7Y8uDKs
 dbos/tracer.py,sha256=GaXDhdKKF_IQp5SAMipGXiDVwteRKjNbrXyYCH1mor0,2520
 dbos/utils.py,sha256=lwRymY-y7GprAS8pKmbICQvOJd5eGxKGTxCMFn0OwaQ,1739
 version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
-dbos-0.8.0a3.dist-info/RECORD,,
+dbos-0.8.0a10.dist-info/RECORD,,