dbos 0.8.0a3__tar.gz → 0.8.0a10__tar.gz

{dbos-0.8.0a3 → dbos-0.8.0a10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.8.0a3
+Version: 0.8.0a10
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/core.py

@@ -38,6 +38,7 @@ from dbos.error import (
     DBOSWorkflowFunctionNotFoundError,
 )
 from dbos.registrations import (
+    DEFAULT_MAX_RECOVERY_ATTEMPTS,
     get_config_name,
     get_dbos_class_name,
     get_dbos_func_name,
@@ -118,6 +119,7 @@ def _init_workflow(
     config_name: Optional[str],
     temp_wf_type: Optional[str],
     queue: Optional[str] = None,
+    max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS,
 ) -> WorkflowStatusInternal:
     wfid = (
         ctx.workflow_id
@@ -157,7 +159,9 @@ def _init_workflow(
         # Synchronously record the status and inputs for workflows and single-step workflows
         # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
         # TODO: Make this transactional (and with the queue step below)
-        dbos._sys_db.update_workflow_status(status, False, ctx.in_recovery)
+        dbos._sys_db.update_workflow_status(
+            status, False, ctx.in_recovery, max_recovery_attempts=max_recovery_attempts
+        )
         dbos._sys_db.update_workflow_inputs(wfid, utils.serialize_args(inputs))
     else:
         # Buffer the inputs for single-transaction workflows, but don't buffer the status
@@ -181,7 +185,8 @@ def _execute_workflow(
         status["status"] = "SUCCESS"
         status["output"] = utils.serialize(output)
         if status["queue_name"] is not None:
-            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+            queue = dbos._registry.queue_info_map[status["queue_name"]]
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
         dbos._sys_db.buffer_workflow_status(status)
     except DBOSWorkflowConflictIDError:
         # Retrieve the workflow handle and wait for the result.
@@ -195,7 +200,8 @@ def _execute_workflow(
         status["status"] = "ERROR"
         status["error"] = utils.serialize_exception(error)
         if status["queue_name"] is not None:
-            dbos._sys_db.remove_from_queue(status["workflow_uuid"])
+            queue = dbos._registry.queue_info_map[status["queue_name"]]
+            dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
         dbos._sys_db.update_workflow_status(status)
         raise
 
@@ -218,7 +224,7 @@ def _execute_workflow_wthread(
         with EnterDBOSWorkflow(attributes):
             try:
                 return _execute_workflow(dbos, status, func, *args, **kwargs)
-            except Exception as e:
+            except Exception:
                 dbos.logger.error(
                     f"Exception encountered in asynchronous workflow: {traceback.format_exc()}"
                 )
@@ -289,10 +295,15 @@ def _execute_workflow_id(dbos: "DBOS", workflow_id: str) -> "WorkflowHandle[Any]
                 )
 
 
-def _workflow_wrapper(dbosreg: "_DBOSRegistry", func: F) -> F:
+def _workflow_wrapper(
+    dbosreg: "_DBOSRegistry",
+    func: F,
+    max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS,
+) -> F:
     func.__orig_func = func  # type: ignore
 
     fi = get_or_create_func_info(func)
+    fi.max_recovery_attempts = max_recovery_attempts
 
     @wraps(func)
     def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -325,17 +336,21 @@ def _workflow_wrapper(dbosreg: "_DBOSRegistry", func: F) -> F:
                 class_name=get_dbos_class_name(fi, func, args),
                 config_name=get_config_name(fi, func, args),
                 temp_wf_type=get_temp_workflow_type(func),
+                max_recovery_attempts=max_recovery_attempts,
             )
 
+            dbos.logger.debug(
+                f"Running workflow, id: {ctx.workflow_id}, name: {get_dbos_func_name(func)}"
+            )
             return _execute_workflow(dbos, status, func, *args, **kwargs)
 
     wrapped_func = cast(F, wrapper)
     return wrapped_func
 
 
-def _workflow(reg: "_DBOSRegistry") -> Callable[[F], F]:
+def _workflow(reg: "_DBOSRegistry", max_recovery_attempts: int) -> Callable[[F], F]:
     def _workflow_decorator(func: F) -> F:
-        wrapped_func = _workflow_wrapper(reg, func)
+        wrapped_func = _workflow_wrapper(reg, func, max_recovery_attempts)
         reg.register_wf_function(func.__qualname__, wrapped_func)
         return wrapped_func
 
@@ -401,6 +416,7 @@ def _start_workflow(
         config_name=get_config_name(fi, func, gin_args),
         temp_wf_type=get_temp_workflow_type(func),
         queue=queue_name,
+        max_recovery_attempts=fi.max_recovery_attempts,
     )
 
     if not execute_workflow:
@@ -477,6 +493,9 @@ def _transaction(
                                     )
                                 )
                                 if recorded_output:
+                                    dbos.logger.debug(
+                                        f"Replaying transaction, id: {ctx.function_id}, name: {attributes['name']}"
+                                    )
                                     if recorded_output["error"]:
                                         deserialized_error = (
                                             utils.deserialize_exception(
@@ -493,6 +512,11 @@ def _transaction(
                                         raise Exception(
                                             "Output and error are both None"
                                         )
+                                else:
+                                    dbos.logger.debug(
+                                        f"Running transaction, id: {ctx.function_id}, name: {attributes['name']}"
+                                    )
+
                                 output = func(*args, **kwargs)
                                 txn_output["output"] = utils.serialize(output)
                                 assert (
@@ -590,6 +614,9 @@ def _step(
                     ctx.workflow_id, ctx.function_id
                 )
                 if recorded_output:
+                    dbos.logger.debug(
+                        f"Replaying step, id: {ctx.function_id}, name: {attributes['name']}"
+                    )
                     if recorded_output["error"] is not None:
                         deserialized_error = utils.deserialize_exception(
                             recorded_output["error"]
@@ -599,6 +626,11 @@ def _step(
                         return utils.deserialize(recorded_output["output"])
                     else:
                         raise Exception("Output and error are both None")
+                else:
+                    dbos.logger.debug(
+                        f"Running step, id: {ctx.function_id}, name: {attributes['name']}"
+                    )
+
                 output = None
                 error = None
                 local_max_attempts = max_attempts if retries_allowed else 1

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/dbos.py

@@ -5,6 +5,7 @@ import json
 import os
 import sys
 import threading
+import traceback
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from logging import Logger
@@ -42,6 +43,7 @@ from dbos.decorators import classproperty
 from dbos.queue import Queue, queue_thread
 from dbos.recovery import _recover_pending_workflows, _startup_recovery_thread
 from dbos.registrations import (
+    DEFAULT_MAX_RECOVERY_ATTEMPTS,
     DBOSClassInfo,
     get_or_create_class_info,
     set_dbos_func_name,
@@ -338,43 +340,47 @@ class DBOS:
             _dbos_global_instance._launch()
 
     def _launch(self) -> None:
-        if self._launched:
-            dbos_logger.warning(f"DBOS was already launched")
-            return
-        self._launched = True
-        self._executor_field = ThreadPoolExecutor(max_workers=64)
-        self._sys_db_field = SystemDatabase(self.config)
-        self._app_db_field = ApplicationDatabase(self.config)
-        self._admin_server_field = AdminServer(dbos=self)
-
-        if not os.environ.get("DBOS__VMID"):
-            workflow_ids = self._sys_db.get_pending_workflows("local")
-            self._executor.submit(_startup_recovery_thread, self, workflow_ids)
-
-        # Listen to notifications
-        self._executor.submit(self._sys_db._notification_listener)
-
-        # Start flush workflow buffers thread
-        self._executor.submit(self._sys_db.flush_workflow_buffers)
-
-        # Start the queue thread
-        evt = threading.Event()
-        self.stop_events.append(evt)
-        self._executor.submit(queue_thread, evt, self)
-
-        # Grab any pollers that were deferred and start them
-        for evt, func, args, kwargs in self._registry.pollers:
+        try:
+            if self._launched:
+                dbos_logger.warning(f"DBOS was already launched")
+                return
+            self._launched = True
+            self._executor_field = ThreadPoolExecutor(max_workers=64)
+            self._sys_db_field = SystemDatabase(self.config)
+            self._app_db_field = ApplicationDatabase(self.config)
+            self._admin_server_field = AdminServer(dbos=self)
+
+            if not os.environ.get("DBOS__VMID"):
+                workflow_ids = self._sys_db.get_pending_workflows("local")
+                self._executor.submit(_startup_recovery_thread, self, workflow_ids)
+
+            # Listen to notifications
+            self._executor.submit(self._sys_db._notification_listener)
+
+            # Start flush workflow buffers thread
+            self._executor.submit(self._sys_db.flush_workflow_buffers)
+
+            # Start the queue thread
+            evt = threading.Event()
             self.stop_events.append(evt)
-            self._executor.submit(func, *args, **kwargs)
-        self._registry.pollers = []
+            self._executor.submit(queue_thread, evt, self)
 
-        dbos_logger.info("DBOS launched")
+            # Grab any pollers that were deferred and start them
+            for evt, func, args, kwargs in self._registry.pollers:
+                self.stop_events.append(evt)
+                self._executor.submit(func, *args, **kwargs)
+            self._registry.pollers = []
 
-        # Flush handlers and add OTLP to all loggers if enabled
-        # to enable their export in DBOS Cloud
-        for handler in dbos_logger.handlers:
-            handler.flush()
-        add_otlp_to_all_loggers()
+            dbos_logger.info("DBOS launched")
+
+            # Flush handlers and add OTLP to all loggers if enabled
+            # to enable their export in DBOS Cloud
+            for handler in dbos_logger.handlers:
+                handler.flush()
+            add_otlp_to_all_loggers()
+        except Exception:
+            dbos_logger.error(f"DBOS failed to launch: {traceback.format_exc()}")
+            raise
 
     def _destroy(self) -> None:
         self._initialized = False
@@ -401,9 +407,11 @@ class DBOS:
 
     # Decorators for DBOS functionality
     @classmethod
-    def workflow(cls) -> Callable[[F], F]:
+    def workflow(
+        cls, *, max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS
+    ) -> Callable[[F], F]:
         """Decorate a function for use as a DBOS workflow."""
-        return _workflow(_get_or_create_dbos_registry())
+        return _workflow(_get_or_create_dbos_registry(), max_recovery_attempts)
 
     @classmethod
     def transaction(
@@ -542,6 +550,7 @@ class DBOS:
             recovery_attempts=stat["recovery_attempts"],
             class_name=stat["class_name"],
             config_name=stat["config_name"],
+            queue_name=stat["queue_name"],
             authenticated_user=stat["authenticated_user"],
             assumed_role=stat["assumed_role"],
             authenticated_roles=(
@@ -748,6 +757,7 @@ class WorkflowStatus:
         name(str): The workflow function name
         class_name(str): For member functions, the name of the class containing the workflow function
         config_name(str): For instance member functions, the name of the class instance for the execution
+        queue_name(str): For workflows that are or were queued, the queue name
         authenticated_user(str): The user who invoked the workflow
         assumed_role(str): The access role used by the user to allow access to the workflow function
         authenticated_roles(List[str]): List of all access roles available to the authenticated user
@@ -760,6 +770,7 @@ class WorkflowStatus:
     name: str
     class_name: Optional[str]
     config_name: Optional[str]
+    queue_name: Optional[str]
     authenticated_user: Optional[str]
     assumed_role: Optional[str]
     authenticated_roles: Optional[List[str]]

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/error.py

@@ -32,6 +32,7 @@ class DBOSErrorCode(Enum):
     InitializationError = 3
     WorkflowFunctionNotFound = 4
     NonExistentWorkflowError = 5
+    DeadLetterQueueError = 6
     MaxStepRetriesExceeded = 7
     NotAuthorized = 8
 
@@ -86,6 +87,16 @@ class DBOSNonExistentWorkflowError(DBOSException):
         )
 
 
+class DBOSDeadLetterQueueError(DBOSException):
+    """Exception raised when a workflow database record does not exist for a given ID."""
+
+    def __init__(self, wf_id: str, max_retries: int):
+        super().__init__(
+            f"Workflow {wf_id} has been moved to the dead-letter queue after exceeding the maximum of ${max_retries} retries",
+            dbos_error_code=DBOSErrorCode.DeadLetterQueueError.value,
+        )
+
+
 class DBOSNotAuthorizedError(DBOSException):
     """Exception raised by DBOS role-based security when the user is not authorized to access a function."""
 

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/migrations/versions/50f3227f0b4b_fix_job_queue.py

@@ -1,4 +1,5 @@
-"""fix_job_queue
+"""
+Fix job queue PK.
 
 Revision ID: 50f3227f0b4b
 Revises: eab0cc1d9a14

dbos-0.8.0a10/dbos/migrations/versions/d76646551a6b_job_queue_limiter.py

@@ -0,0 +1,43 @@
+"""
+Adjust workflow queue to add columns for rate limiter.
+
+Revision ID: d76646551a6b
+Revises: 50f3227f0b4b
+Create Date: 2024-09-25 14:48:10.218015
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d76646551a6b"
+down_revision: Union[str, None] = "50f3227f0b4b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "job_queue",
+        sa.Column(
+            "started_at_epoch_ms",
+            sa.BigInteger(),
+        ),
+        schema="dbos",
+    )
+    op.add_column(
+        "job_queue",
+        sa.Column(
+            "completed_at_epoch_ms",
+            sa.BigInteger(),
+        ),
+        schema="dbos",
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("job_queue", "started_at_epoch_ms", schema="dbos")
+    op.drop_column("job_queue", "completed_at_epoch_ms", schema="dbos")

dbos-0.8.0a10/dbos/migrations/versions/d76646551a6c_workflow_queue.py

@@ -0,0 +1,28 @@
+"""workflow_queue
+
+Revision ID: d76646551a6c
+Revises: d76646551a6b
+Create Date: 2024-09-27 12:00:00.0
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d76646551a6c"
+down_revision: Union[str, None] = "d76646551a6b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.rename_table("job_queue", "workflow_queue", schema="dbos")
+    op.execute("CREATE VIEW dbos.job_queue AS SELECT * FROM dbos.workflow_queue;")
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW dbos.job_queue;")
+    op.rename_table("workflow_queue", "job_queue", schema="dbos")

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/migrations/versions/eab0cc1d9a14_job_queue.py

@@ -1,4 +1,5 @@
-"""job_queue
+"""
+Add workflow queue table.
 
 Revision ID: eab0cc1d9a14
 Revises: a3b18ad34abe

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/queue.py

@@ -1,7 +1,6 @@
 import threading
-import time
 import traceback
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, TypedDict
 
 from dbos.core import P, R, _execute_workflow_id, _start_workflow
 
@@ -9,10 +8,35 @@ if TYPE_CHECKING:
     from dbos.dbos import DBOS, Workflow, WorkflowHandle
 
 
+class QueueRateLimit(TypedDict):
+    """
+    Limit the maximum number of workflows from this queue that can be started in a given period.
+
+    If the limit is 5 and the period is 10, no more than 5 functions can be
+    started per 10 seconds.
+    """
+
+    limit: int
+    period: float
+
+
 class Queue:
-    def __init__(self, name: str, concurrency: Optional[int] = None) -> None:
+    """
+    Workflow queue.
+
+    Workflow queues allow workflows to be started at a later time, based on concurrency and
+    rate limits.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        concurrency: Optional[int] = None,
+        limiter: Optional[QueueRateLimit] = None,
+    ) -> None:
         self.name = name
         self.concurrency = concurrency
+        self.limiter = limiter
         from dbos.dbos import _get_or_create_dbos_registry
 
         registry = _get_or_create_dbos_registry()
@@ -29,12 +53,11 @@ class Queue:
 
 def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
     while not stop_event.is_set():
-        time.sleep(1)
-        for queue_name, queue in dbos._registry.queue_info_map.items():
+        if stop_event.wait(timeout=1):
+            return
+        for _, queue in dbos._registry.queue_info_map.items():
             try:
-                wf_ids = dbos._sys_db.start_queued_workflows(
-                    queue_name, queue.concurrency
-                )
+                wf_ids = dbos._sys_db.start_queued_workflows(queue)
                 for id in wf_ids:
                     _execute_workflow_id(dbos, id)
             except Exception:

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/registrations.py

@@ -3,6 +3,8 @@ from enum import Enum
 from types import FunctionType
 from typing import Any, Callable, List, Literal, Optional, Tuple, Type, cast
 
+DEFAULT_MAX_RECOVERY_ATTEMPTS = 50
+
 
 def get_dbos_func_name(f: Any) -> str:
     if hasattr(f, "dbos_function_name"):
@@ -47,6 +49,7 @@ class DBOSFuncInfo:
         self.class_info: Optional[DBOSClassInfo] = None
         self.func_type: DBOSFuncType = DBOSFuncType.Unknown
         self.required_roles: Optional[List[str]] = None
+        self.max_recovery_attempts = DEFAULT_MAX_RECOVERY_ATTEMPTS
 
 
 def get_or_create_class_info(cls: Type[Any]) -> DBOSClassInfo:

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/request.py

@@ -13,6 +13,7 @@ class Address(NamedTuple):
 class Request:
     """
     Serializable HTTP Request object.
+
     Attributes:
         base_url(str): Base of URL requested, as in application code
         client(Optional[Address]): HTTP Client

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/scheduler/scheduler.py

@@ -2,6 +2,7 @@ import threading
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Callable
 
+from dbos.logger import dbos_logger
 from dbos.queue import Queue
 
 if TYPE_CHECKING:
@@ -18,7 +19,12 @@ scheduler_queue: Queue
 def scheduler_loop(
     func: ScheduledWorkflow, cron: str, stop_event: threading.Event
 ) -> None:
-    iter = croniter(cron, datetime.now(timezone.utc), second_at_beginning=True)
+    try:
+        iter = croniter(cron, datetime.now(timezone.utc), second_at_beginning=True)
+    except Exception as e:
+        dbos_logger.error(
+            f'Cannot run scheduled function {func.__name__}. Invalid crontab "{cron}"'
+        )
     while not stop_event.is_set():
         nextExecTime = iter.get_next(datetime)
         sleepTime = nextExecTime - datetime.now(timezone.utc)

{dbos-0.8.0a3 → dbos-0.8.0a10}/dbos/schemas/system_database.py

@@ -142,8 +142,8 @@ class SystemSchema:
         Column("last_run_time", BigInteger, nullable=False),
     )
 
-    job_queue = Table(
-        "job_queue",
+    workflow_queue = Table(
+        "workflow_queue",
         metadata_obj,
         Column(
             "workflow_uuid",
@@ -161,4 +161,12 @@ class SystemSchema:
             nullable=False,
             server_default=text("(EXTRACT(epoch FROM now()) * 1000::numeric)::bigint"),
         ),
+        Column(
+            "started_at_epoch_ms",
+            BigInteger(),
+        ),
+        Column(
+            "completed_at_epoch_ms",
+            BigInteger(),
+        ),
     )