dbos 1.5.0a3__tar.gz → 1.5.0a5__tar.gz

{dbos-1.5.0a3 → dbos-1.5.0a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 1.5.0a3
+Version: 1.5.0a5
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-1.5.0a3 → dbos-1.5.0a5}/dbos/_admin_server.py

@@ -5,7 +5,7 @@ import re
 import threading
 from functools import partial
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from typing import TYPE_CHECKING, Any, List, Optional, TypedDict
+from typing import TYPE_CHECKING, Any, List, Optional, TypedDict, Dict
 
 from dbos._workflow_commands import garbage_collect, global_timeout
 
@@ -24,6 +24,8 @@ _deactivate_path = "/deactivate"
 _workflow_queues_metadata_path = "/dbos-workflow-queues-metadata"
 _garbage_collect_path = "/dbos-garbage-collect"
 _global_timeout_path = "/dbos-global-timeout"
+_queued_workflows_path = "/queues"
+_workflows_path = "/workflows"
 # /workflows/:workflow_id/cancel
 # /workflows/:workflow_id/resume
 # /workflows/:workflow_id/restart
@@ -104,10 +106,24 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
             steps_match = re.match(
                 r"^/workflows/(?P<workflow_id>[^/]+)/steps$", self.path
             )
+            workflow_match = re.match(r"^/workflows/(?P<workflow_id>[^/]+)$", self.path)
 
             if steps_match:
                 workflow_id = steps_match.group("workflow_id")
                 self._handle_steps(workflow_id)
+            elif workflow_match:
+                workflow_id = workflow_match.group("workflow_id")
+                workflows = self.dbos.list_workflows(workflow_ids=[workflow_id])
+                if not workflows:
+                    self.send_response(404)
+                    self._end_headers()
+                    return
+                response_body = json.dumps(workflows[0].__dict__).encode("utf-8")
+                self.send_response(200)
+                self.send_header("Content-Type", "application/json")
+                self.send_header("Content-Length", str(len(response_body)))
+                self._end_headers()
+                self.wfile.write(response_body)
             else:
                 self.send_response(404)
                 self._end_headers()
@@ -126,6 +142,32 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
             self.send_response(200)
             self._end_headers()
             self.wfile.write(json.dumps(workflow_ids).encode("utf-8"))
+        elif self.path == _workflows_path:
+            try:
+                filters = json.loads(post_data.decode("utf-8")) if post_data else {}
+                self._handle_workflows(filters)
+            except (json.JSONDecodeError, AttributeError) as e:
+                self.send_response(400)
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+                self.wfile.write(
+                    json.dumps({"error": f"Invalid JSON input: {str(e)}"}).encode(
+                        "utf-8"
+                    )
+                )
+        elif self.path == _queued_workflows_path:
+            try:
+                filters = json.loads(post_data.decode("utf-8")) if post_data else {}
+                self._handle_queued_workflows(filters)
+            except (json.JSONDecodeError, AttributeError) as e:
+                self.send_response(400)
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+                self.wfile.write(
+                    json.dumps({"error": f"Invalid JSON input: {str(e)}"}).encode(
+                        "utf-8"
+                    )
+                )
         elif self.path == _garbage_collect_path:
             inputs = json.loads(post_data.decode("utf-8"))
             cutoff_epoch_timestamp_ms = inputs.get("cutoff_epoch_timestamp_ms", None)
@@ -139,12 +181,11 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
             self._end_headers()
         elif self.path == _global_timeout_path:
             inputs = json.loads(post_data.decode("utf-8"))
-            timeout_ms = inputs.get("timeout_ms", None)
-            global_timeout(self.dbos, timeout_ms)
+            cutoff_epoch_timestamp_ms = inputs.get("cutoff_epoch_timestamp_ms", None)
+            global_timeout(self.dbos, cutoff_epoch_timestamp_ms)
             self.send_response(204)
             self._end_headers()
         else:
-
             restart_match = re.match(
                 r"^/workflows/(?P<workflow_id>[^/]+)/restart$", self.path
             )
@@ -283,6 +324,50 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
         self._end_headers()
         self.wfile.write(json_steps)
 
+    def _handle_workflows(self, filters: Dict[str, Any]) -> None:
+        workflows = self.dbos.list_workflows(
+            workflow_ids=filters.get("workflow_ids"),
+            name=filters.get("name"),
+            start_time=filters.get("start_time"),
+            end_time=filters.get("end_time"),
+            status=filters.get("status"),
+            app_version=filters.get("application_version"),
+            limit=filters.get("limit"),
+            offset=filters.get("offset"),
+            sort_desc=filters.get("sort_desc", False),
+            workflow_id_prefix=filters.get("workflow_id_prefix"),
+        )
+
+        response_body = json.dumps(
+            [workflow.__dict__ for workflow in workflows]
+        ).encode("utf-8")
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(response_body)))
+        self._end_headers()
+        self.wfile.write(response_body)
+
+    def _handle_queued_workflows(self, filters: Dict[str, Any]) -> None:
+        workflows = self.dbos.list_queued_workflows(
+            queue_name=filters.get("queue_name"),
+            name=filters.get("name"),
+            start_time=filters.get("start_time"),
+            end_time=filters.get("end_time"),
+            status=filters.get("status"),
+            limit=filters.get("limit"),
+            offset=filters.get("offset"),
+            sort_desc=filters.get("sort_desc", False),
+        )
+
+        response_body = json.dumps(
+            [workflow.__dict__ for workflow in workflows]
+        ).encode("utf-8")
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(response_body)))
+        self._end_headers()
+        self.wfile.write(response_body)
+
 
 # Be consistent with DBOS-TS response.
 class PerfUtilization(TypedDict):

{dbos-1.5.0a3 → dbos-1.5.0a5}/dbos/_conductor/conductor.py

@@ -13,7 +13,9 @@ from websockets.sync.connection import Connection
 from dbos._context import SetWorkflowID
 from dbos._utils import GlobalParams
 from dbos._workflow_commands import (
+    garbage_collect,
     get_workflow,
+    global_timeout,
     list_queued_workflows,
     list_workflow_steps,
     list_workflows,
@@ -356,6 +358,41 @@ class ConductorWebsocket(threading.Thread):
                                 error_message=error_message,
                             )
                             websocket.send(list_steps_response.to_json())
+                        elif msg_type == p.MessageType.RETENTION:
+                            retention_message = p.RetentionRequest.from_json(message)
+                            success = True
+                            try:
+                                garbage_collect(
+                                    self.dbos,
+                                    cutoff_epoch_timestamp_ms=retention_message.body[
+                                        "gc_cutoff_epoch_ms"
+                                    ],
+                                    rows_threshold=retention_message.body[
+                                        "gc_rows_threshold"
+                                    ],
+                                )
+                                if (
+                                    retention_message.body["timeout_cutoff_epoch_ms"]
+                                    is not None
+                                ):
+                                    global_timeout(
+                                        self.dbos,
+                                        retention_message.body[
+                                            "timeout_cutoff_epoch_ms"
+                                        ],
+                                    )
+                            except Exception as e:
+                                error_message = f"Exception encountered during enforcing retention policy: {traceback.format_exc()}"
+                                self.dbos.logger.error(error_message)
+                                success = False
+
+                            retention_response = p.RetentionResponse(
+                                type=p.MessageType.RETENTION,
+                                request_id=base_message.request_id,
+                                success=success,
+                                error_message=error_message,
+                            )
+                            websocket.send(retention_response.to_json())
                         else:
                             self.dbos.logger.warning(
                                 f"Unexpected message type: {msg_type}"

{dbos-1.5.0a3 → dbos-1.5.0a5}/dbos/_conductor/protocol.py

@@ -18,6 +18,7 @@ class MessageType(str, Enum):
     EXIST_PENDING_WORKFLOWS = "exist_pending_workflows"
     LIST_STEPS = "list_steps"
     FORK_WORKFLOW = "fork_workflow"
+    RETENTION = "retention"
 
 
 T = TypeVar("T", bound="BaseMessage")
@@ -280,3 +281,20 @@ class ForkWorkflowRequest(BaseMessage):
 class ForkWorkflowResponse(BaseMessage):
     new_workflow_id: Optional[str]
     error_message: Optional[str] = None
+
+
+class RetentionBody(TypedDict):
+    gc_cutoff_epoch_ms: Optional[int]
+    gc_rows_threshold: Optional[int]
+    timeout_cutoff_epoch_ms: Optional[int]
+
+
+@dataclass
+class RetentionRequest(BaseMessage):
+    body: RetentionBody
+
+
+@dataclass
+class RetentionResponse(BaseMessage):
+    success: bool
+    error_message: Optional[str] = None

{dbos-1.5.0a3 → dbos-1.5.0a5}/dbos/_workflow_commands.py

@@ -141,8 +141,7 @@ def garbage_collect(
         dbos._app_db.garbage_collect(cutoff_epoch_timestamp_ms, pending_workflow_ids)
 
 
-def global_timeout(dbos: "DBOS", timeout_ms: int) -> None:
-    cutoff_epoch_timestamp_ms = int(time.time() * 1000) - timeout_ms
+def global_timeout(dbos: "DBOS", cutoff_epoch_timestamp_ms: int) -> None:
     cutoff_iso = datetime.fromtimestamp(cutoff_epoch_timestamp_ms / 1000).isoformat()
     for workflow in dbos.list_workflows(
         status=WorkflowStatusString.PENDING.value, end_time=cutoff_iso

{dbos-1.5.0a3 → dbos-1.5.0a5}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 ]
 requires-python = ">=3.9"
 readme = "README.md"
-version = "1.5.0a3"
+version = "1.5.0a5"
 
 [project.license]
 text = "MIT"

{dbos-1.5.0a3 → dbos-1.5.0a5}/tests/test_admin_server.py

@@ -3,7 +3,7 @@ import socket
 import threading
 import time
 import uuid
-from datetime import datetime
+from datetime import datetime, timezone
 
 import pytest
 import requests
@@ -455,6 +455,108 @@ def test_admin_workflow_fork(dbos: DBOS, sys_db: SystemDatabase) -> None:
     assert worked, "Workflow did not finish successfully"
 
 
+def test_list_workflows(dbos: DBOS) -> None:
+    # Create workflows for testing
+    @DBOS.workflow()
+    def test_workflow_1() -> None:
+        pass
+
+    @DBOS.workflow()
+    def test_workflow_2() -> None:
+        pass
+
+    # Start workflows
+    handle_1 = DBOS.start_workflow(test_workflow_1)
+    time.sleep(2)  # Sleep for 2 seconds between workflows
+    handle_2 = DBOS.start_workflow(test_workflow_2)
+
+    # Wait for workflows to complete
+    handle_1.get_result()
+    handle_2.get_result()
+
+    # List workflows and dynamically set the filter "name"
+    workflows_list = DBOS.list_workflows()
+    assert (
+        len(workflows_list) >= 2
+    ), f"Expected at least 2 workflows, but got {len(workflows_list)}"
+
+    workflow_ids = [w.workflow_id for w in workflows_list]
+
+    # Convert created_at to ISO 8601 format
+    created_at_second_workflow = workflows_list[1].created_at
+    assert (
+        created_at_second_workflow is not None
+    ), "created_at for the second workflow is None"
+    start_time_filter = datetime.fromtimestamp(
+        created_at_second_workflow / 1000, tz=timezone.utc
+    ).isoformat()
+
+    # Test POST /workflows with filters
+    filters = {
+        "workflow_ids": workflow_ids,
+        "start_time": start_time_filter,
+    }
+    response = requests.post("http://localhost:3001/workflows", json=filters, timeout=5)
+    assert response.status_code == 200
+
+    workflows = response.json()
+    assert len(workflows) == 1, f"Expected 1 workflows, but got {len(workflows)}"
+    assert workflows[0]["workflow_id"] == handle_2.workflow_id, "Workflow ID mismatch"
+
+    # Test POST /workflows without filters
+    response = requests.post("http://localhost:3001/workflows", json={}, timeout=5)
+    assert response.status_code == 200
+
+    workflows = response.json()
+    assert len(workflows) == len(
+        workflows_list
+    ), f"Expected {len(workflows_list)} workflows, but got {len(workflows)}"
+    for workflow in workflows:
+        assert workflow["workflow_id"] in workflow_ids, "Workflow ID mismatch"
+
+
+def test_get_workflow_by_id(dbos: DBOS) -> None:
+    # Create workflows for testing
+    @DBOS.workflow()
+    def test_workflow_1() -> None:
+        pass
+
+    @DBOS.workflow()
+    def test_workflow_2() -> None:
+        pass
+
+    # Start workflows
+    handle_1 = DBOS.start_workflow(test_workflow_1)
+    handle_2 = DBOS.start_workflow(test_workflow_2)
+
+    # Wait for workflows to complete
+    handle_1.get_result()
+    handle_2.get_result()
+
+    # Get the workflow ID of the second workflow
+    workflow_id = handle_2.workflow_id
+
+    # Test GET /workflows/:workflow_id for an existing workflow
+    response = requests.get(f"http://localhost:3001/workflows/{workflow_id}", timeout=5)
+    assert (
+        response.status_code == 200
+    ), f"Expected status code 200, but got {response.status_code}"
+
+    workflow_data = response.json()
+    assert workflow_data["workflow_id"] == workflow_id, "Workflow ID mismatch"
+    assert (
+        workflow_data["status"] == "SUCCESS"
+    ), "Expected workflow status to be SUCCESS"
+
+    # Test GET /workflows/:workflow_id for a non-existing workflow
+    non_existing_workflow_id = "non-existing-id"
+    response = requests.get(
+        f"http://localhost:3001/workflows/{non_existing_workflow_id}", timeout=5
+    )
+    assert (
+        response.status_code == 404
+    ), f"Expected status code 404, but got {response.status_code}"
+
 def test_admin_garbage_collect(dbos: DBOS) -> None:
 
     @DBOS.workflow()
@@ -484,12 +586,57 @@ def test_admin_global_timeout(dbos: DBOS) -> None:
 
     handle = DBOS.start_workflow(workflow)
     time.sleep(1)
-
+    cutoff_epoch_timestamp_ms = int(time.time() * 1000) - 1000
     response = requests.post(
         f"http://localhost:3001/dbos-global-timeout",
-        json={"timeout_ms": 1000},
+        json={"cutoff_epoch_timestamp_ms": cutoff_epoch_timestamp_ms},
         timeout=5,
     )
     response.raise_for_status()
     with pytest.raises(DBOSWorkflowCancelledError):
         handle.get_result()
+
+
+def test_queued_workflows_endpoint(dbos: DBOS) -> None:
+    """Test the /queues endpoint with various filters and scenarios."""
+
+    # Set up a queue for testing
+    test_queue1 = Queue("test-queue-1", concurrency=1)
+    test_queue2 = Queue("test-queue-2", concurrency=1)
+
+    @DBOS.workflow()
+    def blocking_workflow() -> str:
+        while True:
+            time.sleep(0.1)
+
+    # Enqueue some workflows to create queued entries
+    handles = []
+    handles.append(test_queue1.enqueue(blocking_workflow))
+    handles.append(test_queue1.enqueue(blocking_workflow))
+    handles.append(test_queue2.enqueue(blocking_workflow))
+
+    # Test basic queued workflows endpoint
+    response = requests.post("http://localhost:3001/queues", json={}, timeout=5)
+    assert response.status_code == 200, f"Expected status 200, got {response.status_code}"
+
+    queued_workflows = response.json()
+    assert isinstance(queued_workflows, list), "Response should be a list"
+    assert len(queued_workflows) == 3, f"Expected 3 queued workflows, got {len(queued_workflows)}"
+
+    # Test with filters
+    filters = {"queue_name": "test-queue-1", "limit": 1}
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
+
+    filtered_workflows = response.json()
+    assert isinstance(filtered_workflows, list), "Response should be a list"
+    assert len(filtered_workflows) == 1, f"Expected 1 workflow, got {len(filtered_workflows)}"
+
+    # Test with non-existent queue name
+    filters = {"queue_name": "non-existent-queue"}
+    response = requests.post("http://localhost:3001/queues", json=filters, timeout=5)
+    assert response.status_code == 200
+
+    empty_result = response.json()
+    assert isinstance(empty_result, list), "Response should be a list even for non-existent queue"
+    assert len(empty_result) == 0, "Expected no workflows for non-existent queue"

{dbos-1.5.0a3 → dbos-1.5.0a5}/tests/test_workflow_management.py

@@ -737,7 +737,8 @@ def test_global_timeout(dbos: DBOS) -> None:
     # Wait one second, start one final workflow, then timeout all workflows started more than one second ago
     time.sleep(1)
     final_handle = DBOS.start_workflow(blocked_workflow)
-    global_timeout(dbos, 1000)
+    cutoff_epoch_timestamp_ms = int(time.time() * 1000) - 1000
+    global_timeout(dbos, cutoff_epoch_timestamp_ms)
 
     # Verify all workflows started before the global timeout are cancelled
     for handle in handles: