dbos 0.8.0a7__tar.gz → 0.9.0a0__tar.gz

{dbos-0.8.0a7 → dbos-0.9.0a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.8.0a7
+Version: 0.9.0a0
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/dbos.py

@@ -550,6 +550,7 @@ class DBOS:
             recovery_attempts=stat["recovery_attempts"],
             class_name=stat["class_name"],
             config_name=stat["config_name"],
+            queue_name=stat["queue_name"],
             authenticated_user=stat["authenticated_user"],
             assumed_role=stat["assumed_role"],
             authenticated_roles=(
@@ -756,6 +757,7 @@ class WorkflowStatus:
         name(str): The workflow function name
         class_name(str): For member functions, the name of the class containing the workflow function
         config_name(str): For instance member functions, the name of the class instance for the execution
+        queue_name(str): For workflows that are or were queued, the queue name
         authenticated_user(str): The user who invoked the workflow
         assumed_role(str): The access role used by the user to allow access to the workflow function
         authenticated_roles(List[str]): List of all access roles available to the authenticated user
@@ -768,6 +770,7 @@ class WorkflowStatus:
     name: str
     class_name: Optional[str]
     config_name: Optional[str]
+    queue_name: Optional[str]
     authenticated_user: Optional[str]
     assumed_role: Optional[str]
     authenticated_roles: Optional[List[str]]

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/migrations/versions/50f3227f0b4b_fix_job_queue.py

@@ -1,4 +1,5 @@
-"""fix_job_queue
+"""
+Fix job queue PK.
 
 Revision ID: 50f3227f0b4b
 Revises: eab0cc1d9a14

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/migrations/versions/d76646551a6b_job_queue_limiter.py

@@ -1,4 +1,5 @@
-"""job_queue_limiter
+"""
+Adjust workflow queue to add columns for rate limiter.
 
 Revision ID: d76646551a6b
 Revises: 50f3227f0b4b

dbos-0.9.0a0/dbos/migrations/versions/d76646551a6c_workflow_queue.py

@@ -0,0 +1,28 @@
+"""workflow_queue
+
+Revision ID: d76646551a6c
+Revises: d76646551a6b
+Create Date: 2024-09-27 12:00:00.0
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d76646551a6c"
+down_revision: Union[str, None] = "d76646551a6b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.rename_table("job_queue", "workflow_queue", schema="dbos")
+    op.execute("CREATE VIEW dbos.job_queue AS SELECT * FROM dbos.workflow_queue;")
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW dbos.job_queue;")
+    op.rename_table("workflow_queue", "job_queue", schema="dbos")

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/migrations/versions/eab0cc1d9a14_job_queue.py

@@ -1,4 +1,5 @@
-"""job_queue
+"""
+Add workflow queue table.
 
 Revision ID: eab0cc1d9a14
 Revises: a3b18ad34abe

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/queue.py

@@ -8,21 +8,31 @@ if TYPE_CHECKING:
     from dbos.dbos import DBOS, Workflow, WorkflowHandle
 
 
-# Limit the maximum number of functions from this queue
-# that can be started in a given period. If the limit is 5
-# and the period is 10, no more than 5 functions can be
-# started per 10 seconds.
-class Limiter(TypedDict):
+class QueueRateLimit(TypedDict):
+    """
+    Limit the maximum number of workflows from this queue that can be started in a given period.
+
+    If the limit is 5 and the period is 10, no more than 5 functions can be
+    started per 10 seconds.
+    """
+
     limit: int
     period: float
 
 
 class Queue:
+    """
+    Workflow queue.
+
+    Workflow queues allow workflows to be started at a later time, based on concurrency and
+    rate limits.
+    """
+
     def __init__(
         self,
         name: str,
         concurrency: Optional[int] = None,
-        limiter: Optional[Limiter] = None,
+        limiter: Optional[QueueRateLimit] = None,
     ) -> None:
         self.name = name
         self.concurrency = concurrency

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/request.py

@@ -13,6 +13,7 @@ class Address(NamedTuple):
 class Request:
     """
     Serializable HTTP Request object.
+
     Attributes:
         base_url(str): Base of URL requested, as in application code
         client(Optional[Address]): HTTP Client

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/scheduler/scheduler.py

@@ -2,6 +2,7 @@ import threading
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Callable
 
+from dbos.logger import dbos_logger
 from dbos.queue import Queue
 
 if TYPE_CHECKING:
@@ -18,7 +19,12 @@ scheduler_queue: Queue
 def scheduler_loop(
     func: ScheduledWorkflow, cron: str, stop_event: threading.Event
 ) -> None:
-    iter = croniter(cron, datetime.now(timezone.utc), second_at_beginning=True)
+    try:
+        iter = croniter(cron, datetime.now(timezone.utc), second_at_beginning=True)
+    except Exception as e:
+        dbos_logger.error(
+            f'Cannot run scheduled function {func.__name__}. Invalid crontab "{cron}"'
+        )
     while not stop_event.is_set():
         nextExecTime = iter.get_next(datetime)
         sleepTime = nextExecTime - datetime.now(timezone.utc)

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/schemas/system_database.py

@@ -142,8 +142,8 @@ class SystemSchema:
         Column("last_run_time", BigInteger, nullable=False),
     )
 
-    job_queue = Table(
-        "job_queue",
+    workflow_queue = Table(
+        "workflow_queue",
         metadata_obj,
         Column(
             "workflow_uuid",

{dbos-0.8.0a7 → dbos-0.9.0a0}/dbos/system_database.py

@@ -983,7 +983,7 @@ class SystemDatabase:
         return value
 
     def _flush_workflow_status_buffer(self) -> None:
-        """Export the workflow status buffer to the database, up to the batch size"""
+        """Export the workflow status buffer to the database, up to the batch size."""
         if len(self._workflow_status_buffer) == 0:
             return
 
@@ -1079,7 +1079,7 @@ class SystemDatabase:
     def enqueue(self, workflow_id: str, queue_name: str) -> None:
         with self.engine.begin() as c:
             c.execute(
-                pg.insert(SystemSchema.job_queue)
+                pg.insert(SystemSchema.workflow_queue)
                 .values(
                     workflow_uuid=workflow_id,
                     queue_name=queue_name,
@@ -1099,10 +1099,13 @@ class SystemDatabase:
             if queue.limiter is not None:
                 query = (
                     sa.select(sa.func.count())
-                    .select_from(SystemSchema.job_queue)
-                    .where(SystemSchema.job_queue.c.started_at_epoch_ms.isnot(None))
+                    .select_from(SystemSchema.workflow_queue)
+                    .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
                     .where(
-                        SystemSchema.job_queue.c.started_at_epoch_ms
+                        SystemSchema.workflow_queue.c.started_at_epoch_ms.isnot(None)
+                    )
+                    .where(
+                        SystemSchema.workflow_queue.c.started_at_epoch_ms
                         > start_time_ms - limiter_period_ms
                     )
                 )
@@ -1116,12 +1119,12 @@ class SystemDatabase:
             # functions, else select all of them.
             query = (
                 sa.select(
-                    SystemSchema.job_queue.c.workflow_uuid,
-                    SystemSchema.job_queue.c.started_at_epoch_ms,
+                    SystemSchema.workflow_queue.c.workflow_uuid,
+                    SystemSchema.workflow_queue.c.started_at_epoch_ms,
                 )
-                .where(SystemSchema.job_queue.c.queue_name == queue.name)
-                .where(SystemSchema.job_queue.c.completed_at_epoch_ms == None)
-                .order_by(SystemSchema.job_queue.c.created_at_epoch_ms.asc())
+                .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
+                .where(SystemSchema.workflow_queue.c.completed_at_epoch_ms == None)
+                .order_by(SystemSchema.workflow_queue.c.created_at_epoch_ms.asc())
             )
             if queue.concurrency is not None:
                 query = query.limit(queue.concurrency)
@@ -1152,8 +1155,8 @@ class SystemDatabase:
 
                 # Then give it a start time
                 c.execute(
-                    SystemSchema.job_queue.update()
-                    .where(SystemSchema.job_queue.c.workflow_uuid == id)
+                    SystemSchema.workflow_queue.update()
+                    .where(SystemSchema.workflow_queue.c.workflow_uuid == id)
                     .values(started_at_epoch_ms=start_time_ms)
                 )
                 ret_ids.append(id)
@@ -1163,10 +1166,11 @@ class SystemDatabase:
             # deleted on completion.
             if queue.limiter is not None:
                 c.execute(
-                    sa.delete(SystemSchema.job_queue)
-                    .where(SystemSchema.job_queue.c.completed_at_epoch_ms != None)
+                    sa.delete(SystemSchema.workflow_queue)
+                    .where(SystemSchema.workflow_queue.c.completed_at_epoch_ms != None)
+                    .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
                     .where(
-                        SystemSchema.job_queue.c.started_at_epoch_ms
+                        SystemSchema.workflow_queue.c.started_at_epoch_ms
                         < start_time_ms - limiter_period_ms
                     )
                 )
@@ -1178,13 +1182,13 @@ class SystemDatabase:
         with self.engine.begin() as c:
             if queue.limiter is None:
                 c.execute(
-                    sa.delete(SystemSchema.job_queue).where(
-                        SystemSchema.job_queue.c.workflow_uuid == workflow_id
+                    sa.delete(SystemSchema.workflow_queue).where(
+                        SystemSchema.workflow_queue.c.workflow_uuid == workflow_id
                     )
                 )
             else:
                 c.execute(
-                    sa.update(SystemSchema.job_queue)
-                    .where(SystemSchema.job_queue.c.workflow_uuid == workflow_id)
+                    sa.update(SystemSchema.workflow_queue)
+                    .where(SystemSchema.workflow_queue.c.workflow_uuid == workflow_id)
                     .values(completed_at_epoch_ms=int(time.time() * 1000))
                 )

{dbos-0.8.0a7 → dbos-0.9.0a0}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "0.8.0a7"
+version = "0.9.0a0"
 
 [project.license]
 text = "MIT"

{dbos-0.8.0a7 → dbos-0.9.0a0}/tests/test_queue.py

@@ -15,7 +15,7 @@ def queue_entries_are_cleaned_up(dbos: DBOS) -> bool:
     success = False
     for i in range(max_tries):
         with dbos._sys_db.engine.begin() as c:
-            query = sa.select(sa.func.count()).select_from(SystemSchema.job_queue)
+            query = sa.select(sa.func.count()).select_from(SystemSchema.workflow_queue)
             row = c.execute(query).fetchone()
             assert row is not None
             count = row[0]
@@ -77,6 +77,7 @@ def test_one_at_a_time(dbos: DBOS) -> None:
 
     queue = Queue("test_queue", 1)
     handle1 = queue.enqueue(workflow_one)
+    assert handle1.get_status().queue_name == "test_queue"
     handle2 = queue.enqueue(workflow_two)
 
     main_thread_event.wait()
@@ -120,6 +121,40 @@ def test_one_at_a_time_with_limiter(dbos: DBOS) -> None:
     assert handle2.get_result() == None
     assert flag
     assert wf_counter == 1
+    assert queue_entries_are_cleaned_up(dbos)
+
+
+def test_queue_childwf(dbos: DBOS) -> None:
+    queue = Queue("child_queue", 3)
+
+    @DBOS.workflow()
+    def test_child_wf(val: str) -> str:
+        DBOS.recv("release", 30)
+        return val + "d"
+
+    @DBOS.workflow()
+    def test_workflow(var1: str, var2: str) -> str:
+        wfh1 = queue.enqueue(test_child_wf, var1)
+        wfh2 = queue.enqueue(test_child_wf, var2)
+        wfh3 = queue.enqueue(test_child_wf, var1)
+        wfh4 = queue.enqueue(test_child_wf, var2)
+
+        DBOS.sleep(1)
+        assert wfh4.get_status().status == "ENQUEUED"
+
+        DBOS.send(wfh1.get_workflow_id(), "go", "release")
+        DBOS.send(wfh2.get_workflow_id(), "go", "release")
+        DBOS.send(wfh3.get_workflow_id(), "go", "release")
+        DBOS.send(wfh4.get_workflow_id(), "go", "release")
+
+        return (
+            wfh1.get_result()
+            + wfh2.get_result()
+            + wfh3.get_result()
+            + wfh4.get_result()
+        )
+
+    assert test_workflow("a", "b") == "adbdadbd"
 
 
 def test_queue_step(dbos: DBOS) -> None:
@@ -192,16 +227,96 @@ def test_limiter(dbos: DBOS) -> None:
     # Verify that each "wave" of tasks started at the ~same time.
     for wave in range(num_waves):
         for i in range(wave * limit, (wave + 1) * limit - 1):
-            assert times[i + 1] - times[i] < 0.1
+            assert times[i + 1] - times[i] < 0.2
+
+    # Verify that the gap between "waves" is ~equal to the period
+    for wave in range(num_waves - 1):
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.2
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.2
+
+    # Verify all workflows get the SUCCESS status eventually
+    dbos._sys_db.wait_for_buffer_flush()
+    for h in handles:
+        assert h.get_status().status == WorkflowStatusString.SUCCESS.value
+
+    # Verify all queue entries eventually get cleaned up.
+    assert queue_entries_are_cleaned_up(dbos)
+
+
+def test_multiple_queues(dbos: DBOS) -> None:
+
+    wf_counter = 0
+    flag = False
+    workflow_event = threading.Event()
+    main_thread_event = threading.Event()
+
+    @DBOS.workflow()
+    def workflow_one() -> None:
+        nonlocal wf_counter
+        wf_counter += 1
+        main_thread_event.set()
+        workflow_event.wait()
+
+    @DBOS.workflow()
+    def workflow_two() -> None:
+        nonlocal flag
+        flag = True
+
+    concurrency_queue = Queue("test_concurrency_queue", 1)
+    handle1 = concurrency_queue.enqueue(workflow_one)
+    assert handle1.get_status().queue_name == "test_concurrency_queue"
+    handle2 = concurrency_queue.enqueue(workflow_two)
+
+    @DBOS.workflow()
+    def limited_workflow(var1: str, var2: str) -> float:
+        assert var1 == "abc" and var2 == "123"
+        return time.time()
+
+    limit = 5
+    period = 2
+    limiter_queue = Queue(
+        "test_limit_queue", limiter={"limit": limit, "period": period}
+    )
+
+    handles: list[WorkflowHandle[float]] = []
+    times: list[float] = []
+
+    # Launch a number of tasks equal to three times the limit.
+    # This should lead to three "waves" of the limit tasks being
+    # executed simultaneously, followed by a wait of the period,
+    # followed by the next wave.
+    num_waves = 3
+    for _ in range(limit * num_waves):
+        h = limiter_queue.enqueue(limited_workflow, "abc", "123")
+        handles.append(h)
+    for h in handles:
+        times.append(h.get_result())
+
+    # Verify that each "wave" of tasks started at the ~same time.
+    for wave in range(num_waves):
+        for i in range(wave * limit, (wave + 1) * limit - 1):
+            assert times[i + 1] - times[i] < 0.2
 
     # Verify that the gap between "waves" is ~equal to the period
     for wave in range(num_waves - 1):
-        assert times[limit * wave] - times[limit * wave - 1] < period + 0.1
+        assert times[limit * (wave + 1)] - times[limit * wave] > period - 0.2
+        assert times[limit * (wave + 1)] - times[limit * wave] < period + 0.2
 
     # Verify all workflows get the SUCCESS status eventually
     dbos._sys_db.wait_for_buffer_flush()
     for h in handles:
         assert h.get_status().status == WorkflowStatusString.SUCCESS.value
 
+    # Verify that during all this time, the second task
+    # was not launched on the concurrency-limited queue.
+    # Then, finish the first task and verify the second
+    # task runs on schedule.
+    assert not flag
+    workflow_event.set()
+    assert handle1.get_result() == None
+    assert handle2.get_result() == None
+    assert flag
+    assert wf_counter == 1
+
     # Verify all queue entries eventually get cleaned up.
     assert queue_entries_are_cleaned_up(dbos)