dbos 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbos might be problematic. Click here for more details.

dbos/_context.py CHANGED
@@ -57,6 +57,7 @@ class DBOSContext:
57
57
  self.request: Optional["Request"] = None
58
58
 
59
59
  self.id_assigned_for_next_workflow: str = ""
60
+ self.is_within_set_workflow_id_block: bool = False
60
61
 
61
62
  self.parent_workflow_id: str = ""
62
63
  self.parent_workflow_fid: int = -1
@@ -78,6 +79,7 @@ class DBOSContext:
78
79
  rv.logger = self.logger
79
80
  rv.id_assigned_for_next_workflow = self.id_assigned_for_next_workflow
80
81
  self.id_assigned_for_next_workflow = ""
82
+ rv.is_within_set_workflow_id_block = self.is_within_set_workflow_id_block
81
83
  rv.parent_workflow_id = self.workflow_id
82
84
  rv.parent_workflow_fid = self.function_id
83
85
  rv.in_recovery = self.in_recovery
@@ -95,6 +97,10 @@ class DBOSContext:
95
97
  if len(self.id_assigned_for_next_workflow) > 0:
96
98
  wfid = self.id_assigned_for_next_workflow
97
99
  else:
100
+ if self.is_within_set_workflow_id_block:
101
+ self.logger.warning(
102
+ f"Multiple workflows started in the same SetWorkflowID block. Only the first workflow is assigned the specified workflow ID; subsequent workflows will use a generated workflow ID."
103
+ )
98
104
  wfid = str(uuid.uuid4())
99
105
  return wfid
100
106
 
@@ -286,7 +292,7 @@ class DBOSContextSwap:
286
292
 
287
293
  class SetWorkflowID:
288
294
  """
289
- Set the workflow ID to be used for the enclosed workflow invocation.
295
+ Set the workflow ID to be used for the enclosed workflow invocation. Note: Only the first workflow will be started with the specified workflow ID within a `with SetWorkflowID` block.
290
296
 
291
297
  Typical Usage
292
298
  ```
@@ -311,7 +317,9 @@ class SetWorkflowID:
311
317
  if ctx is None:
312
318
  self.created_ctx = True
313
319
  _set_local_dbos_context(DBOSContext())
314
- assert_current_dbos_context().id_assigned_for_next_workflow = self.wfid
320
+ ctx = assert_current_dbos_context()
321
+ ctx.id_assigned_for_next_workflow = self.wfid
322
+ ctx.is_within_set_workflow_id_block = True
315
323
  return self
316
324
 
317
325
  def __exit__(
@@ -321,6 +329,7 @@ class SetWorkflowID:
321
329
  traceback: Optional[TracebackType],
322
330
  ) -> Literal[False]:
323
331
  # Code to clean up the basic context if we created it
332
+ assert_current_dbos_context().is_within_set_workflow_id_block = False
324
333
  if self.created_ctx:
325
334
  _clear_local_dbos_context()
326
335
  return False # Did not handle
dbos/_core.py CHANGED
@@ -84,7 +84,7 @@ if TYPE_CHECKING:
84
84
  IsolationLevel,
85
85
  )
86
86
 
87
- from sqlalchemy.exc import DBAPIError
87
+ from sqlalchemy.exc import DBAPIError, InvalidRequestError
88
88
 
89
89
  P = ParamSpec("P") # A generic type for workflow parameters
90
90
  R = TypeVar("R", covariant=True) # A generic type for workflow return values
@@ -180,21 +180,24 @@ def _init_workflow(
180
180
  if class_name is not None:
181
181
  inputs = {"args": inputs["args"][1:], "kwargs": inputs["kwargs"]}
182
182
 
183
+ wf_status = status["status"]
183
184
  if temp_wf_type != "transaction" or queue is not None:
184
185
  # Synchronously record the status and inputs for workflows and single-step workflows
185
186
  # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
186
187
  # TODO: Make this transactional (and with the queue step below)
187
- dbos._sys_db.update_workflow_status(
188
+ wf_status = dbos._sys_db.update_workflow_status(
188
189
  status, False, ctx.in_recovery, max_recovery_attempts=max_recovery_attempts
189
190
  )
191
+ # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
190
192
  dbos._sys_db.update_workflow_inputs(wfid, _serialization.serialize_args(inputs))
191
193
  else:
192
194
  # Buffer the inputs for single-transaction workflows, but don't buffer the status
193
195
  dbos._sys_db.buffer_workflow_inputs(wfid, _serialization.serialize_args(inputs))
194
196
 
195
- if queue is not None:
197
+ if queue is not None and wf_status == WorkflowStatusString.ENQUEUED.value:
196
198
  dbos._sys_db.enqueue(wfid, queue)
197
199
 
200
+ status["status"] = wf_status
198
201
  return status
199
202
 
200
203
 
@@ -413,7 +416,16 @@ def start_workflow(
413
416
  max_recovery_attempts=fi.max_recovery_attempts,
414
417
  )
415
418
 
416
- if not execute_workflow:
419
+ wf_status = status["status"]
420
+
421
+ if (
422
+ not execute_workflow
423
+ or wf_status == WorkflowStatusString.ERROR.value
424
+ or wf_status == WorkflowStatusString.SUCCESS.value
425
+ ):
426
+ dbos.logger.debug(
427
+ f"Workflow {new_wf_id} already completed with status {wf_status}. Directly returning a workflow handle."
428
+ )
417
429
  return WorkflowHandlePolling(new_wf_id, dbos)
418
430
 
419
431
  if fself is not None:
@@ -486,7 +498,7 @@ def workflow_wrapper(
486
498
  temp_wf_type=get_temp_workflow_type(func),
487
499
  max_recovery_attempts=max_recovery_attempts,
488
500
  )
489
-
501
+ # TODO: maybe modify the parameters if they've been changed by `_init_workflow`
490
502
  dbos.logger.debug(
491
503
  f"Running workflow, id: {ctx.workflow_id}, name: {get_dbos_func_name(func)}"
492
504
  )
@@ -545,6 +557,7 @@ def decorate_transaction(
545
557
  max_retry_wait_seconds = 2.0
546
558
  while True:
547
559
  has_recorded_error = False
560
+ txn_error: Optional[Exception] = None
548
561
  try:
549
562
  with session.begin():
550
563
  # This must be the first statement in the transaction!
@@ -608,15 +621,24 @@ def decorate_transaction(
608
621
  max_retry_wait_seconds,
609
622
  )
610
623
  continue
624
+ txn_error = dbapi_error
625
+ raise
626
+ except InvalidRequestError as invalid_request_error:
627
+ dbos.logger.error(
628
+ f"InvalidRequestError in transaction {func.__qualname__} \033[1m Hint: Do not call commit() or rollback() within a DBOS transaction.\033[0m"
629
+ )
630
+ txn_error = invalid_request_error
611
631
  raise
612
632
  except Exception as error:
633
+ txn_error = error
634
+ raise
635
+ finally:
613
636
  # Don't record the error if it was already recorded
614
- if not has_recorded_error:
637
+ if txn_error and not has_recorded_error:
615
638
  txn_output["error"] = (
616
- _serialization.serialize_exception(error)
639
+ _serialization.serialize_exception(txn_error)
617
640
  )
618
641
  dbos._app_db.record_transaction_error(txn_output)
619
- raise
620
642
  return output
621
643
 
622
644
  if inspect.iscoroutinefunction(func):
dbos/_db_wizard.py CHANGED
@@ -1,5 +1,7 @@
1
+ import json
2
+ import os
1
3
  import time
2
- from typing import TYPE_CHECKING, Optional
4
+ from typing import TYPE_CHECKING, Optional, TypedDict
3
5
 
4
6
  import docker # type: ignore
5
7
  import typer
@@ -15,8 +17,18 @@ from ._cloudutils.databases import choose_database, get_user_db_credentials
15
17
  from ._error import DBOSInitializationError
16
18
  from ._logger import dbos_logger
17
19
 
20
+ DB_CONNECTION_PATH = os.path.join(".dbos", "db_connection")
18
21
 
19
- def db_connect(config: "ConfigFile", config_file_path: str) -> "ConfigFile":
22
+
23
+ class DatabaseConnection(TypedDict):
24
+ hostname: Optional[str]
25
+ port: Optional[int]
26
+ username: Optional[str]
27
+ password: Optional[str]
28
+ local_suffix: Optional[bool]
29
+
30
+
31
+ def db_wizard(config: "ConfigFile", config_file_path: str) -> "ConfigFile":
20
32
  # 1. Check the connectivity to the database. Return if successful. If cannot connect, continue to the following steps.
21
33
  db_connection_error = _check_db_connectivity(config)
22
34
  if db_connection_error is None:
@@ -82,17 +94,20 @@ def db_connect(config: "ConfigFile", config_file_path: str) -> "ConfigFile":
82
94
  f"Could not connect to the database. Exception: {db_connection_error}"
83
95
  )
84
96
 
85
- # 6. Save the config to the config file and return the updated config.
86
- # TODO: make the config file prettier
87
- with open(config_file_path, "w") as file:
88
- file.write(yaml.dump(config))
89
-
97
+ # 6. Save the config to the database connection file
98
+ updated_connection = DatabaseConnection(
99
+ hostname=config["database"]["hostname"],
100
+ port=config["database"]["port"],
101
+ username=config["database"]["username"],
102
+ password=config["database"]["password"],
103
+ local_suffix=config["database"]["local_suffix"],
104
+ )
105
+ save_db_connection(updated_connection)
90
106
  return config
91
107
 
92
108
 
93
109
  def _start_docker_postgres(config: "ConfigFile") -> bool:
94
110
  print("Starting a Postgres Docker container...")
95
- config["database"]["password"] = "dbos"
96
111
  client = docker.from_env()
97
112
  pg_data = "/var/lib/postgresql/data"
98
113
  container_name = "dbos-db"
@@ -122,7 +137,7 @@ def _start_docker_postgres(config: "ConfigFile") -> bool:
122
137
  continue
123
138
  print("[green]Postgres Docker container started successfully![/green]")
124
139
  break
125
- except Exception as e:
140
+ except:
126
141
  attempts -= 1
127
142
  time.sleep(1)
128
143
 
@@ -151,7 +166,7 @@ def _check_db_connectivity(config: "ConfigFile") -> Optional[Exception]:
151
166
  host=config["database"]["hostname"],
152
167
  port=config["database"]["port"],
153
168
  database="postgres",
154
- query={"connect_timeout": "2"},
169
+ query={"connect_timeout": "1"},
155
170
  )
156
171
  postgres_db_engine = create_engine(postgres_db_url)
157
172
  try:
@@ -168,3 +183,26 @@ def _check_db_connectivity(config: "ConfigFile") -> Optional[Exception]:
168
183
  postgres_db_engine.dispose()
169
184
 
170
185
  return None
186
+
187
+
188
+ def load_db_connection() -> DatabaseConnection:
189
+ try:
190
+ with open(DB_CONNECTION_PATH, "r") as f:
191
+ data = json.load(f)
192
+ return DatabaseConnection(
193
+ hostname=data.get("hostname", None),
194
+ port=data.get("port", None),
195
+ username=data.get("username", None),
196
+ password=data.get("password", None),
197
+ local_suffix=data.get("local_suffix", None),
198
+ )
199
+ except:
200
+ return DatabaseConnection(
201
+ hostname=None, port=None, username=None, password=None, local_suffix=None
202
+ )
203
+
204
+
205
+ def save_db_connection(connection: DatabaseConnection) -> None:
206
+ os.makedirs(".dbos", exist_ok=True)
207
+ with open(DB_CONNECTION_PATH, "w") as f:
208
+ json.dump(connection, f)
dbos/_dbos.py CHANGED
@@ -83,7 +83,7 @@ from ._context import (
83
83
  )
84
84
  from ._dbos_config import ConfigFile, load_config, set_env_vars
85
85
  from ._error import DBOSException, DBOSNonExistentWorkflowError
86
- from ._logger import add_otlp_to_all_loggers, dbos_logger, init_logger
86
+ from ._logger import add_otlp_to_all_loggers, dbos_logger
87
87
  from ._sys_db import SystemDatabase
88
88
 
89
89
  # Most DBOS functions are just any callable F, so decorators / wrappers work on F
dbos/_dbos_config.py CHANGED
@@ -6,12 +6,15 @@ from typing import Any, Dict, List, Optional, TypedDict, cast
6
6
 
7
7
  import yaml
8
8
  from jsonschema import ValidationError, validate
9
+ from rich import print
9
10
  from sqlalchemy import URL
10
11
 
11
- from ._db_wizard import db_connect
12
+ from ._db_wizard import db_wizard, load_db_connection
12
13
  from ._error import DBOSInitializationError
13
14
  from ._logger import config_logger, dbos_logger, init_logger
14
15
 
16
+ DBOS_CONFIG_PATH = "dbos-config.yaml"
17
+
15
18
 
16
19
  class RuntimeConfig(TypedDict, total=False):
17
20
  start: List[str]
@@ -23,7 +26,7 @@ class DatabaseConfig(TypedDict, total=False):
23
26
  hostname: str
24
27
  port: int
25
28
  username: str
26
- password: Optional[str]
29
+ password: str
27
30
  connectionTimeoutMillis: Optional[int]
28
31
  app_db_name: str
29
32
  sys_db_name: Optional[str]
@@ -93,7 +96,7 @@ def _substitute_env_vars(content: str) -> str:
93
96
  return re.sub(regex, replace_func, content)
94
97
 
95
98
 
96
- def get_dbos_database_url(config_file_path: str = "dbos-config.yaml") -> str:
99
+ def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
97
100
  """
98
101
  Retrieve application database URL from configuration `.yaml` file.
99
102
 
@@ -119,7 +122,9 @@ def get_dbos_database_url(config_file_path: str = "dbos-config.yaml") -> str:
119
122
  return db_url.render_as_string(hide_password=False)
120
123
 
121
124
 
122
- def load_config(config_file_path: str = "dbos-config.yaml") -> ConfigFile:
125
+ def load_config(
126
+ config_file_path: str = DBOS_CONFIG_PATH, *, use_db_wizard: bool = True
127
+ ) -> ConfigFile:
123
128
  """
124
129
  Load the DBOS `ConfigFile` from the specified path (typically `dbos-config.yaml`).
125
130
 
@@ -151,6 +156,9 @@ def load_config(config_file_path: str = "dbos-config.yaml") -> ConfigFile:
151
156
  except ValidationError as e:
152
157
  raise DBOSInitializationError(f"Validation error: {e}")
153
158
 
159
+ if "database" not in data:
160
+ data["database"] = {}
161
+
154
162
  if "name" not in data:
155
163
  raise DBOSInitializationError(
156
164
  f"dbos-config.yaml must specify an application name"
@@ -169,8 +177,6 @@ def load_config(config_file_path: str = "dbos-config.yaml") -> ConfigFile:
169
177
  if "runtimeConfig" not in data or "start" not in data["runtimeConfig"]:
170
178
  raise DBOSInitializationError(f"dbos-config.yaml must specify a start command")
171
179
 
172
- data = cast(ConfigFile, data)
173
-
174
180
  if not _is_valid_app_name(data["name"]):
175
181
  raise DBOSInitializationError(
176
182
  f'Invalid app name {data["name"]}. App names must be between 3 and 30 characters long and contain only lowercase letters, numbers, dashes, and underscores.'
@@ -179,10 +185,49 @@ def load_config(config_file_path: str = "dbos-config.yaml") -> ConfigFile:
179
185
  if "app_db_name" not in data["database"]:
180
186
  data["database"]["app_db_name"] = _app_name_to_db_name(data["name"])
181
187
 
188
+ # Load the DB connection file. Use its values for missing fields from dbos-config.yaml. Use defaults otherwise.
189
+ data = cast(ConfigFile, data)
190
+ db_connection = load_db_connection()
191
+ if data["database"].get("hostname"):
192
+ print(
193
+ "[bold blue]Loading database connection parameters from dbos-config.yaml[/bold blue]"
194
+ )
195
+ elif db_connection.get("hostname"):
196
+ print(
197
+ "[bold blue]Loading database connection parameters from .dbos/db_connection[/bold blue]"
198
+ )
199
+ else:
200
+ print(
201
+ "[bold blue]Using default database connection parameters (localhost)[/bold blue]"
202
+ )
203
+
204
+ data["database"]["hostname"] = (
205
+ data["database"].get("hostname") or db_connection.get("hostname") or "localhost"
206
+ )
207
+ data["database"]["port"] = (
208
+ data["database"].get("port") or db_connection.get("port") or 5432
209
+ )
210
+ data["database"]["username"] = (
211
+ data["database"].get("username") or db_connection.get("username") or "postgres"
212
+ )
213
+ data["database"]["password"] = (
214
+ data["database"].get("password")
215
+ or db_connection.get("password")
216
+ or os.environ.get("PGPASSWORD")
217
+ or "dbos"
218
+ )
219
+ data["database"]["local_suffix"] = (
220
+ data["database"].get("local_suffix")
221
+ or db_connection.get("local_suffix")
222
+ or False
223
+ )
224
+
225
+ # Configure the DBOS logger
182
226
  config_logger(data)
183
227
 
184
228
  # Check the connectivity to the database and make sure it's properly configured
185
- data = db_connect(data, config_file_path)
229
+ if use_db_wizard:
230
+ data = db_wizard(data, config_file_path)
186
231
 
187
232
  if "local_suffix" in data["database"] and data["database"]["local_suffix"]:
188
233
  data["database"]["app_db_name"] = f"{data['database']['app_db_name']}_local"
dbos/_error.py CHANGED
@@ -35,6 +35,7 @@ class DBOSErrorCode(Enum):
35
35
  DeadLetterQueueError = 6
36
36
  MaxStepRetriesExceeded = 7
37
37
  NotAuthorized = 8
38
+ ConflictingWorkflowError = 9
38
39
 
39
40
 
40
41
  class DBOSWorkflowConflictIDError(DBOSException):
@@ -47,6 +48,16 @@ class DBOSWorkflowConflictIDError(DBOSException):
47
48
  )
48
49
 
49
50
 
51
+ class DBOSConflictingWorkflowError(DBOSException):
52
+ """Exception raised different workflows started with the same workflow ID."""
53
+
54
+ def __init__(self, workflow_id: str, message: Optional[str] = None):
55
+ super().__init__(
56
+ f"Conflicting workflow invocation with the same ID ({workflow_id}): {message}",
57
+ dbos_error_code=DBOSErrorCode.ConflictingWorkflowError.value,
58
+ )
59
+
60
+
50
61
  class DBOSRecoveryError(DBOSException):
51
62
  """Exception raised when a workflow recovery fails."""
52
63
 
dbos/_kafka.py CHANGED
@@ -1,3 +1,4 @@
1
+ import re
1
2
  import threading
2
3
  from typing import TYPE_CHECKING, Any, Callable, NoReturn
3
4
 
@@ -19,6 +20,14 @@ _kafka_queue: Queue
19
20
  _in_order_kafka_queues: dict[str, Queue] = {}
20
21
 
21
22
 
23
+ def safe_group_name(method_name: str, topics: list[str]) -> str:
24
+ safe_group_id = "-".join(
25
+ re.sub(r"[^a-zA-Z0-9\-]", "", str(r)) for r in [method_name, *topics]
26
+ )
27
+
28
+ return f"dbos-kafka-group-{safe_group_id}"[:255]
29
+
30
+
22
31
  def _kafka_consumer_loop(
23
32
  func: _KafkaConsumerWorkflow,
24
33
  config: dict[str, Any],
@@ -34,6 +43,12 @@ def _kafka_consumer_loop(
34
43
  if "auto.offset.reset" not in config:
35
44
  config["auto.offset.reset"] = "earliest"
36
45
 
46
+ if config.get("group.id") is None:
47
+ config["group.id"] = safe_group_name(func.__qualname__, topics)
48
+ dbos_logger.warning(
49
+ f"Consumer group ID not found. Using generated group.id {config['group.id']}"
50
+ )
51
+
37
52
  consumer = Consumer(config)
38
53
  try:
39
54
  consumer.subscribe(topics)
@@ -71,8 +86,9 @@ def _kafka_consumer_loop(
71
86
  topic=cmsg.topic(),
72
87
  value=cmsg.value(),
73
88
  )
89
+ groupID = config.get("group.id")
74
90
  with SetWorkflowID(
75
- f"kafka-unique-id-{msg.topic}-{msg.partition}-{msg.offset}"
91
+ f"kafka-unique-id-{msg.topic}-{msg.partition}-{groupID}-{msg.offset}"
76
92
  ):
77
93
  if in_order:
78
94
  assert msg.topic is not None
@@ -0,0 +1,34 @@
1
+ """workflow_queues_executor_id
2
+
3
+ Revision ID: 04ca4f231047
4
+ Revises: d76646551a6c
5
+ Create Date: 2025-01-15 15:05:08.043190
6
+
7
+ """
8
+
9
+ from typing import Sequence, Union
10
+
11
+ import sqlalchemy as sa
12
+ from alembic import op
13
+
14
+ # revision identifiers, used by Alembic.
15
+ revision: str = "04ca4f231047"
16
+ down_revision: Union[str, None] = "d76646551a6c"
17
+ branch_labels: Union[str, Sequence[str], None] = None
18
+ depends_on: Union[str, Sequence[str], None] = None
19
+
20
+
21
+ def upgrade() -> None:
22
+ op.add_column(
23
+ "workflow_queue",
24
+ sa.Column(
25
+ "executor_id",
26
+ sa.Text(),
27
+ nullable=True,
28
+ ),
29
+ schema="dbos",
30
+ )
31
+
32
+
33
+ def downgrade() -> None:
34
+ op.drop_column("workflow_queue", "executor_id", schema="dbos")
dbos/_queue.py CHANGED
@@ -2,6 +2,9 @@ import threading
2
2
  import traceback
3
3
  from typing import TYPE_CHECKING, Optional, TypedDict
4
4
 
5
+ from psycopg import errors
6
+ from sqlalchemy.exc import OperationalError
7
+
5
8
  from ._core import P, R, execute_workflow_by_id, start_workflow
6
9
 
7
10
  if TYPE_CHECKING:
@@ -33,9 +36,20 @@ class Queue:
33
36
  name: str,
34
37
  concurrency: Optional[int] = None,
35
38
  limiter: Optional[QueueRateLimit] = None,
39
+ *, # Disable positional arguments from here on
40
+ worker_concurrency: Optional[int] = None,
36
41
  ) -> None:
42
+ if (
43
+ worker_concurrency is not None
44
+ and concurrency is not None
45
+ and worker_concurrency > concurrency
46
+ ):
47
+ raise ValueError(
48
+ "worker_concurrency must be less than or equal to concurrency"
49
+ )
37
50
  self.name = name
38
51
  self.concurrency = concurrency
52
+ self.worker_concurrency = worker_concurrency
39
53
  self.limiter = limiter
40
54
  from ._dbos import _get_or_create_dbos_registry
41
55
 
@@ -60,6 +74,12 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
60
74
  wf_ids = dbos._sys_db.start_queued_workflows(queue, dbos._executor_id)
61
75
  for id in wf_ids:
62
76
  execute_workflow_by_id(dbos, id)
77
+ except OperationalError as e:
78
+ # Ignore serialization error
79
+ if not isinstance(e.orig, errors.SerializationFailure):
80
+ dbos.logger.warning(
81
+ f"Exception encountered in queue thread: {traceback.format_exc()}"
82
+ )
63
83
  except Exception:
64
84
  dbos.logger.warning(
65
85
  f"Exception encountered in queue thread: {traceback.format_exc()}"
@@ -154,6 +154,7 @@ class SystemSchema:
154
154
  nullable=False,
155
155
  primary_key=True,
156
156
  ),
157
+ Column("executor_id", Text),
157
158
  Column("queue_name", Text, nullable=False),
158
159
  Column(
159
160
  "created_at_epoch_ms",
dbos/_sys_db.py CHANGED
@@ -13,7 +13,6 @@ from typing import (
13
13
  Optional,
14
14
  Sequence,
15
15
  Set,
16
- Tuple,
17
16
  TypedDict,
18
17
  cast,
19
18
  )
@@ -23,12 +22,15 @@ import sqlalchemy as sa
23
22
  import sqlalchemy.dialects.postgresql as pg
24
23
  from alembic import command
25
24
  from alembic.config import Config
25
+ from sqlalchemy import or_
26
26
  from sqlalchemy.exc import DBAPIError
27
27
 
28
28
  from . import _serialization
29
29
  from ._dbos_config import ConfigFile
30
30
  from ._error import (
31
+ DBOSConflictingWorkflowError,
31
32
  DBOSDeadLetterQueueError,
33
+ DBOSException,
32
34
  DBOSNonExistentWorkflowError,
33
35
  DBOSWorkflowConflictIDError,
34
36
  )
@@ -249,7 +251,9 @@ class SystemDatabase:
249
251
  *,
250
252
  conn: Optional[sa.Connection] = None,
251
253
  max_recovery_attempts: int = DEFAULT_MAX_RECOVERY_ATTEMPTS,
252
- ) -> None:
254
+ ) -> WorkflowStatuses:
255
+ wf_status: WorkflowStatuses = status["status"]
256
+
253
257
  cmd = pg.insert(SystemSchema.workflow_status).values(
254
258
  workflow_uuid=status["workflow_uuid"],
255
259
  status=status["status"],
@@ -285,49 +289,75 @@ class SystemDatabase:
285
289
  ),
286
290
  )
287
291
  else:
288
- cmd = cmd.on_conflict_do_nothing()
289
- cmd = cmd.returning(SystemSchema.workflow_status.c.recovery_attempts) # type: ignore
292
+ # A blank update so that we can return the existing status
293
+ cmd = cmd.on_conflict_do_update(
294
+ index_elements=["workflow_uuid"],
295
+ set_=dict(
296
+ recovery_attempts=SystemSchema.workflow_status.c.recovery_attempts
297
+ ),
298
+ )
299
+ cmd = cmd.returning(SystemSchema.workflow_status.c.recovery_attempts, SystemSchema.workflow_status.c.status, SystemSchema.workflow_status.c.name, SystemSchema.workflow_status.c.class_name, SystemSchema.workflow_status.c.config_name, SystemSchema.workflow_status.c.queue_name) # type: ignore
290
300
 
291
301
  if conn is not None:
292
302
  results = conn.execute(cmd)
293
303
  else:
294
304
  with self.engine.begin() as c:
295
305
  results = c.execute(cmd)
296
- if in_recovery:
297
- row = results.fetchone()
298
- if row is not None:
299
- recovery_attempts: int = row[0]
300
- if recovery_attempts > max_recovery_attempts:
301
- with self.engine.begin() as c:
302
- c.execute(
303
- sa.delete(SystemSchema.workflow_queue).where(
304
- SystemSchema.workflow_queue.c.workflow_uuid
305
- == status["workflow_uuid"]
306
- )
306
+
307
+ row = results.fetchone()
308
+ if row is not None:
309
+ # Check the started workflow matches the expected name, class_name, config_name, and queue_name
310
+ # A mismatch indicates a workflow starting with the same UUID but different functions, which would throw an exception.
311
+ recovery_attempts: int = row[0]
312
+ wf_status = row[1]
313
+ err_msg: Optional[str] = None
314
+ if row[2] != status["name"]:
315
+ err_msg = f"Workflow already exists with a different function name: {row[2]}, but the provided function name is: {status['name']}"
316
+ elif row[3] != status["class_name"]:
317
+ err_msg = f"Workflow already exists with a different class name: {row[3]}, but the provided class name is: {status['class_name']}"
318
+ elif row[4] != status["config_name"]:
319
+ err_msg = f"Workflow already exists with a different config name: {row[4]}, but the provided config name is: {status['config_name']}"
320
+ elif row[5] != status["queue_name"]:
321
+ # This is a warning because a different queue name is not necessarily an error.
322
+ dbos_logger.warning(
323
+ f"Workflow already exists in queue: {row[5]}, but the provided queue name is: {status['queue_name']}. The queue is not updated."
324
+ )
325
+ if err_msg is not None:
326
+ raise DBOSConflictingWorkflowError(status["workflow_uuid"], err_msg)
327
+
328
+ if in_recovery and recovery_attempts > max_recovery_attempts:
329
+ with self.engine.begin() as c:
330
+ c.execute(
331
+ sa.delete(SystemSchema.workflow_queue).where(
332
+ SystemSchema.workflow_queue.c.workflow_uuid
333
+ == status["workflow_uuid"]
334
+ )
335
+ )
336
+ c.execute(
337
+ sa.update(SystemSchema.workflow_status)
338
+ .where(
339
+ SystemSchema.workflow_status.c.workflow_uuid
340
+ == status["workflow_uuid"]
341
+ )
342
+ .where(
343
+ SystemSchema.workflow_status.c.status
344
+ == WorkflowStatusString.PENDING.value
307
345
  )
308
- c.execute(
309
- sa.update(SystemSchema.workflow_status)
310
- .where(
311
- SystemSchema.workflow_status.c.workflow_uuid
312
- == status["workflow_uuid"]
313
- )
314
- .where(
315
- SystemSchema.workflow_status.c.status
316
- == WorkflowStatusString.PENDING.value
317
- )
318
- .values(
319
- status=WorkflowStatusString.RETRIES_EXCEEDED.value,
320
- queue_name=None,
321
- )
346
+ .values(
347
+ status=WorkflowStatusString.RETRIES_EXCEEDED.value,
348
+ queue_name=None,
322
349
  )
323
- raise DBOSDeadLetterQueueError(
324
- status["workflow_uuid"], max_recovery_attempts
325
350
  )
351
+ raise DBOSDeadLetterQueueError(
352
+ status["workflow_uuid"], max_recovery_attempts
353
+ )
326
354
 
327
355
  # Record we have exported status for this single-transaction workflow
328
356
  if status["workflow_uuid"] in self._temp_txn_wf_ids:
329
357
  self._exported_temp_txn_wf_status.add(status["workflow_uuid"])
330
358
 
359
+ return wf_status
360
+
331
361
  def set_workflow_status(
332
362
  self,
333
363
  workflow_uuid: str,
@@ -349,7 +379,7 @@ class SystemDatabase:
349
379
  stmt = (
350
380
  sa.update(SystemSchema.workflow_status)
351
381
  .where(
352
- SystemSchema.workflow_inputs.c.workflow_uuid == workflow_uuid
382
+ SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid
353
383
  )
354
384
  .values(recovery_attempts=reset_recovery_attempts)
355
385
  )
@@ -405,7 +435,10 @@ class SystemDatabase:
405
435
  res["output"]
406
436
  )
407
437
  return resstat
408
- return None
438
+ else:
439
+ raise DBOSException(
440
+ "Workflow status record not found. This should not happen! \033[1m Hint: Check if your workflow is deterministic.\033[0m"
441
+ )
409
442
  stat = self.get_workflow_status(workflow_uuid)
410
443
  self.record_operation_result(
411
444
  {
@@ -528,18 +561,27 @@ class SystemDatabase:
528
561
  workflow_uuid=workflow_uuid,
529
562
  inputs=inputs,
530
563
  )
531
- .on_conflict_do_nothing()
564
+ .on_conflict_do_update(
565
+ index_elements=["workflow_uuid"],
566
+ set_=dict(workflow_uuid=SystemSchema.workflow_inputs.c.workflow_uuid),
567
+ )
568
+ .returning(SystemSchema.workflow_inputs.c.inputs)
532
569
  )
533
570
  if conn is not None:
534
- conn.execute(cmd)
571
+ row = conn.execute(cmd).fetchone()
535
572
  else:
536
573
  with self.engine.begin() as c:
537
- c.execute(cmd)
538
-
574
+ row = c.execute(cmd).fetchone()
575
+ if row is not None and row[0] != inputs:
576
+ dbos_logger.warning(
577
+ f"Workflow inputs for {workflow_uuid} changed since the first call! Use the original inputs."
578
+ )
579
+ # TODO: actually changing the input
539
580
  if workflow_uuid in self._temp_txn_wf_ids:
540
581
  # Clean up the single-transaction tracking sets
541
582
  self._exported_temp_txn_wf_status.discard(workflow_uuid)
542
583
  self._temp_txn_wf_ids.discard(workflow_uuid)
584
+ return
543
585
 
544
586
  def get_workflow_inputs(
545
587
  self, workflow_uuid: str
@@ -572,12 +614,12 @@ class SystemDatabase:
572
614
  if input.start_time:
573
615
  query = query.where(
574
616
  SystemSchema.workflow_status.c.created_at
575
- >= datetime.datetime.fromisoformat(input.start_time).timestamp()
617
+ >= datetime.datetime.fromisoformat(input.start_time).timestamp() * 1000
576
618
  )
577
619
  if input.end_time:
578
620
  query = query.where(
579
621
  SystemSchema.workflow_status.c.created_at
580
- <= datetime.datetime.fromisoformat(input.end_time).timestamp()
622
+ <= datetime.datetime.fromisoformat(input.end_time).timestamp() * 1000
581
623
  )
582
624
  if input.status:
583
625
  query = query.where(SystemSchema.workflow_status.c.status == input.status)
@@ -1130,27 +1172,38 @@ class SystemDatabase:
1130
1172
  if num_recent_queries >= queue.limiter["limit"]:
1131
1173
  return []
1132
1174
 
1133
- # Select not-yet-completed functions in the queue ordered by the
1134
- # time at which they were enqueued.
1135
- # If there is a concurrency limit N, select only the N most recent
1175
+ # Dequeue functions eligible for this worker and ordered by the time at which they were enqueued.
1176
+ # If there is a global or local concurrency limit N, select only the N oldest enqueued
1136
1177
  # functions, else select all of them.
1137
1178
  query = (
1138
1179
  sa.select(
1139
1180
  SystemSchema.workflow_queue.c.workflow_uuid,
1140
1181
  SystemSchema.workflow_queue.c.started_at_epoch_ms,
1182
+ SystemSchema.workflow_queue.c.executor_id,
1141
1183
  )
1142
1184
  .where(SystemSchema.workflow_queue.c.queue_name == queue.name)
1143
1185
  .where(SystemSchema.workflow_queue.c.completed_at_epoch_ms == None)
1186
+ .where(
1187
+ # Only select functions that have not been started yet or have been started by this worker
1188
+ or_(
1189
+ SystemSchema.workflow_queue.c.executor_id == None,
1190
+ SystemSchema.workflow_queue.c.executor_id == executor_id,
1191
+ )
1192
+ )
1144
1193
  .order_by(SystemSchema.workflow_queue.c.created_at_epoch_ms.asc())
1145
1194
  )
1146
- if queue.concurrency is not None:
1195
+ # Set a dequeue limit if necessary
1196
+ if queue.worker_concurrency is not None:
1197
+ query = query.limit(queue.worker_concurrency)
1198
+ elif queue.concurrency is not None:
1147
1199
  query = query.limit(queue.concurrency)
1148
1200
 
1149
- # From the functions retrieved, get the workflow IDs of the functions
1150
- # that have not yet been started so we can start them.
1151
1201
  rows = c.execute(query).fetchall()
1202
+
1203
+ # Now, get the workflow IDs of functions that have not yet been started
1152
1204
  dequeued_ids: List[str] = [row[0] for row in rows if row[1] is None]
1153
1205
  ret_ids: list[str] = []
1206
+ dbos_logger.debug(f"[{queue.name}] dequeueing {len(dequeued_ids)} task(s)")
1154
1207
  for id in dequeued_ids:
1155
1208
 
1156
1209
  # If we have a limiter, stop starting functions when the number
@@ -1173,11 +1226,11 @@ class SystemDatabase:
1173
1226
  )
1174
1227
  )
1175
1228
 
1176
- # Then give it a start time
1229
+ # Then give it a start time and assign the executor ID
1177
1230
  c.execute(
1178
1231
  SystemSchema.workflow_queue.update()
1179
1232
  .where(SystemSchema.workflow_queue.c.workflow_uuid == id)
1180
- .values(started_at_epoch_ms=start_time_ms)
1233
+ .values(started_at_epoch_ms=start_time_ms, executor_id=executor_id)
1181
1234
  )
1182
1235
  ret_ids.append(id)
1183
1236
 
@@ -9,10 +9,6 @@ runtimeConfig:
9
9
  start:
10
10
  - "fastapi run ${package_name}/main.py"
11
11
  database:
12
- hostname: localhost
13
- port: 5432
14
- username: postgres
15
- password: ${PGPASSWORD}
16
12
  migrate:
17
13
  - ${migration_command}
18
14
  telemetry:
@@ -0,0 +1,172 @@
1
+ from typing import Any, List, Optional, cast
2
+
3
+ import typer
4
+ from rich import print
5
+
6
+ from dbos import DBOS
7
+
8
+ from . import _serialization, load_config
9
+ from ._dbos_config import ConfigFile, _is_valid_app_name
10
+ from ._sys_db import (
11
+ GetWorkflowsInput,
12
+ GetWorkflowsOutput,
13
+ SystemDatabase,
14
+ WorkflowStatuses,
15
+ WorkflowStatusInternal,
16
+ WorkflowStatusString,
17
+ )
18
+
19
+
20
+ class WorkflowInformation:
21
+ workflowUUID: str
22
+ status: WorkflowStatuses
23
+ workflowName: str
24
+ workflowClassName: Optional[str]
25
+ workflowConfigName: Optional[str]
26
+ input: Optional[_serialization.WorkflowInputs] # JSON (jsonpickle)
27
+ output: Optional[str] # JSON (jsonpickle)
28
+ error: Optional[str] # JSON (jsonpickle)
29
+ executor_id: Optional[str]
30
+ app_version: Optional[str]
31
+ app_id: Optional[str]
32
+ request: Optional[str] # JSON (jsonpickle)
33
+ recovery_attempts: Optional[int]
34
+ authenticated_user: Optional[str]
35
+ assumed_role: Optional[str]
36
+ authenticated_roles: Optional[str] # JSON list of roles.
37
+ queue_name: Optional[str]
38
+
39
+
40
+ def _list_workflows(
41
+ config: ConfigFile,
42
+ li: int,
43
+ user: Optional[str],
44
+ starttime: Optional[str],
45
+ endtime: Optional[str],
46
+ status: Optional[str],
47
+ request: bool,
48
+ appversion: Optional[str],
49
+ ) -> List[WorkflowInformation]:
50
+
51
+ sys_db = None
52
+
53
+ try:
54
+ sys_db = SystemDatabase(config)
55
+
56
+ input = GetWorkflowsInput()
57
+ input.authenticated_user = user
58
+ input.start_time = starttime
59
+ input.end_time = endtime
60
+ if status is not None:
61
+ input.status = cast(WorkflowStatuses, status)
62
+ input.application_version = appversion
63
+ input.limit = li
64
+
65
+ output: GetWorkflowsOutput = sys_db.get_workflows(input)
66
+
67
+ infos: List[WorkflowInformation] = []
68
+
69
+ if output.workflow_uuids is None:
70
+ typer.echo("No workflows found")
71
+ return {}
72
+
73
+ for workflow_id in output.workflow_uuids:
74
+ info = _get_workflow_info(
75
+ sys_db, workflow_id, request
76
+ ) # Call the method for each ID
77
+
78
+ if info is not None:
79
+ infos.append(info)
80
+
81
+ return infos
82
+ except Exception as e:
83
+ typer.echo(f"Error listing workflows: {e}")
84
+ return []
85
+ finally:
86
+ if sys_db:
87
+ sys_db.destroy()
88
+
89
+
90
+ def _get_workflow(
91
+ config: ConfigFile, uuid: str, request: bool
92
+ ) -> Optional[WorkflowInformation]:
93
+ sys_db = None
94
+
95
+ try:
96
+ sys_db = SystemDatabase(config)
97
+
98
+ info = _get_workflow_info(sys_db, uuid, request)
99
+ return info
100
+
101
+ except Exception as e:
102
+ typer.echo(f"Error getting workflow: {e}")
103
+ return None
104
+ finally:
105
+ if sys_db:
106
+ sys_db.destroy()
107
+
108
+
109
+ def _cancel_workflow(config: ConfigFile, uuid: str) -> None:
110
+ # config = load_config()
111
+ sys_db = None
112
+
113
+ try:
114
+ sys_db = SystemDatabase(config)
115
+ sys_db.set_workflow_status(uuid, WorkflowStatusString.CANCELLED, False)
116
+ return
117
+
118
+ except Exception as e:
119
+ typer.echo(f"Failed to connect to DBOS system database: {e}")
120
+ return None
121
+ finally:
122
+ if sys_db:
123
+ sys_db.destroy()
124
+
125
+
126
+ def _reattempt_workflow(uuid: str, startNewWorkflow: bool) -> None:
127
+ print(f"Reattempt workflow info for {uuid} not implemented")
128
+ return
129
+
130
+
131
+ def _get_workflow_info(
132
+ sys_db: SystemDatabase, workflowUUID: str, getRequest: bool
133
+ ) -> Optional[WorkflowInformation]:
134
+
135
+ info = sys_db.get_workflow_status(workflowUUID)
136
+ if info is None:
137
+ return None
138
+
139
+ winfo = WorkflowInformation()
140
+
141
+ winfo.workflowUUID = workflowUUID
142
+ winfo.status = info["status"]
143
+ winfo.workflowName = info["name"]
144
+ winfo.workflowClassName = info["class_name"]
145
+ winfo.workflowConfigName = info["config_name"]
146
+ winfo.executor_id = info["executor_id"]
147
+ winfo.app_version = info["app_version"]
148
+ winfo.app_id = info["app_id"]
149
+ winfo.recovery_attempts = info["recovery_attempts"]
150
+ winfo.authenticated_user = info["authenticated_user"]
151
+ winfo.assumed_role = info["assumed_role"]
152
+ winfo.authenticated_roles = info["authenticated_roles"]
153
+ winfo.queue_name = info["queue_name"]
154
+
155
+ # no input field
156
+ input_data = sys_db.get_workflow_inputs(workflowUUID)
157
+ if input_data is not None:
158
+ winfo.input = input_data
159
+
160
+ if info.get("status") == "SUCCESS":
161
+ result = sys_db.await_workflow_result(workflowUUID)
162
+ winfo.output = result
163
+ elif info.get("status") == "ERROR":
164
+ try:
165
+ sys_db.await_workflow_result(workflowUUID)
166
+ except Exception as e:
167
+ winfo.error = str(e)
168
+
169
+ if not getRequest:
170
+ winfo.request = None
171
+
172
+ return winfo
dbos/cli.py CHANGED
@@ -8,6 +8,7 @@ import typing
8
8
  from os import path
9
9
  from typing import Any
10
10
 
11
+ import jsonpickle # type: ignore
11
12
  import sqlalchemy as sa
12
13
  import tomlkit
13
14
  import typer
@@ -17,12 +18,21 @@ from typing_extensions import Annotated
17
18
 
18
19
  from dbos._schemas.system_database import SystemSchema
19
20
 
20
- from . import load_config
21
+ from . import _serialization, load_config
21
22
  from ._app_db import ApplicationDatabase
22
23
  from ._dbos_config import _is_valid_app_name
23
24
  from ._sys_db import SystemDatabase
25
+ from ._workflow_commands import (
26
+ _cancel_workflow,
27
+ _get_workflow,
28
+ _list_workflows,
29
+ _reattempt_workflow,
30
+ )
24
31
 
25
32
  app = typer.Typer()
33
+ workflow = typer.Typer()
34
+
35
+ app.add_typer(workflow, name="workflow", help="Manage DBOS workflows")
26
36
 
27
37
 
28
38
  def _on_windows() -> bool:
@@ -333,5 +343,94 @@ def reset(
333
343
  sys_db.destroy()
334
344
 
335
345
 
346
+ @workflow.command(help="List workflows for your application")
347
+ def list(
348
+ limit: Annotated[
349
+ int,
350
+ typer.Option("--limit", "-l", help="Limit the results returned"),
351
+ ] = 10,
352
+ user: Annotated[
353
+ typing.Optional[str],
354
+ typer.Option("--user", "-u", help="Retrieve workflows run by this user"),
355
+ ] = None,
356
+ starttime: Annotated[
357
+ typing.Optional[str],
358
+ typer.Option(
359
+ "--start-time",
360
+ "-s",
361
+ help="Retrieve workflows starting after this timestamp (ISO 8601 format)",
362
+ ),
363
+ ] = None,
364
+ endtime: Annotated[
365
+ typing.Optional[str],
366
+ typer.Option(
367
+ "--end-time",
368
+ "-e",
369
+ help="Retrieve workflows starting before this timestamp (ISO 8601 format)",
370
+ ),
371
+ ] = None,
372
+ status: Annotated[
373
+ typing.Optional[str],
374
+ typer.Option(
375
+ "--status",
376
+ "-S",
377
+ help="Retrieve workflows with this status (PENDING, SUCCESS, ERROR, RETRIES_EXCEEDED, ENQUEUED, or CANCELLED)",
378
+ ),
379
+ ] = None,
380
+ appversion: Annotated[
381
+ typing.Optional[str],
382
+ typer.Option(
383
+ "--application-version",
384
+ "-v",
385
+ help="Retrieve workflows with this application version",
386
+ ),
387
+ ] = None,
388
+ request: Annotated[
389
+ bool,
390
+ typer.Option("--request", help="Retrieve workflow request information"),
391
+ ] = True,
392
+ appdir: Annotated[
393
+ typing.Optional[str],
394
+ typer.Option("--app-dir", "-d", help="Specify the application root directory"),
395
+ ] = None,
396
+ ) -> None:
397
+ config = load_config()
398
+ workflows = _list_workflows(
399
+ config, limit, user, starttime, endtime, status, request, appversion
400
+ )
401
+ print(jsonpickle.encode(workflows, unpicklable=False))
402
+
403
+
404
+ @workflow.command(help="Retrieve the status of a workflow")
405
+ def get(
406
+ uuid: Annotated[str, typer.Argument()],
407
+ appdir: Annotated[
408
+ typing.Optional[str],
409
+ typer.Option("--app-dir", "-d", help="Specify the application root directory"),
410
+ ] = None,
411
+ request: Annotated[
412
+ bool,
413
+ typer.Option("--request", help="Retrieve workflow request information"),
414
+ ] = True,
415
+ ) -> None:
416
+ config = load_config()
417
+ print(jsonpickle.encode(_get_workflow(config, uuid, request), unpicklable=False))
418
+
419
+
420
+ @workflow.command(
421
+ help="Cancel a workflow so it is no longer automatically retried or restarted"
422
+ )
423
+ def cancel(
424
+ uuid: Annotated[str, typer.Argument()],
425
+ appdir: Annotated[
426
+ typing.Optional[str],
427
+ typer.Option("--app-dir", "-d", help="Specify the application root directory"),
428
+ ] = None,
429
+ ) -> None:
430
+ config = load_config()
431
+ _cancel_workflow(config, uuid)
432
+ print(f"Workflow {uuid} has been cancelled")
433
+
434
+
336
435
  if __name__ == "__main__":
337
436
  app()
@@ -81,13 +81,7 @@
81
81
  "type": "array",
82
82
  "description": "Specify a list of user DB rollback commands to run"
83
83
  }
84
- },
85
- "required": [
86
- "hostname",
87
- "port",
88
- "username",
89
- "password"
90
- ]
84
+ }
91
85
  },
92
86
  "telemetry": {
93
87
  "type": "object",
@@ -181,9 +175,6 @@
181
175
  "type": "string",
182
176
  "deprecated": true
183
177
  }
184
- },
185
- "required": [
186
- "database"
187
- ]
178
+ }
188
179
  }
189
180
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dbos
3
- Version: 0.18.0
3
+ Version: 0.19.0
4
4
  Summary: Ultra-lightweight durable execution in Python
5
5
  Author-Email: "DBOS, Inc." <contact@dbos.dev>
6
6
  License: MIT
@@ -28,14 +28,14 @@ Description-Content-Type: text/markdown
28
28
 
29
29
  <div align="center">
30
30
 
31
- # DBOS Transact: Ultra-Lightweight Durable Execution
31
+ # DBOS Transact: A Lightweight Durable Execution Library Built on Postgres
32
32
 
33
33
  #### [Documentation](https://docs.dbos.dev/) &nbsp;&nbsp;•&nbsp;&nbsp; [Examples](https://docs.dbos.dev/examples) &nbsp;&nbsp;•&nbsp;&nbsp; [Github](https://github.com/dbos-inc) &nbsp;&nbsp;•&nbsp;&nbsp; [Discord](https://discord.com/invite/jsmC6pXGgX)
34
34
  </div>
35
35
 
36
36
  ---
37
37
 
38
- DBOS Transact is a Python library providing **ultra-lightweight durable execution**.
38
+ DBOS Transact is a Python library for **ultra-lightweight durable execution**.
39
39
  For example:
40
40
 
41
41
  ```python
@@ -55,18 +55,23 @@ def workflow()
55
55
 
56
56
  Durable execution means your program is **resilient to any failure**.
57
57
  If it is ever interrupted or crashes, all your workflows will automatically resume from the last completed step.
58
- If you want to see durable execution in action, check out [this demo app](https://demo-widget-store.cloud.dbos.dev/) (source code [here](https://github.com/dbos-inc/dbos-demo-apps/tree/main/python/widget-store)).
59
- No matter how many times you try to crash it, it always resumes from exactly where it left off!
58
+ Durable execution helps solve many common problems:
60
59
 
61
- Under the hood, DBOS Transact works by storing your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
62
- So all you need to use it is a Postgres database to connect to&mdash;there's no need for a "workflow server."
63
- This approach is also incredibly fast, for example [25x faster than AWS Step Functions](https://www.dbos.dev/blog/dbos-vs-aws-step-functions-benchmark).
60
+ - Orchestrating long-running or business-critical workflows so they seamlessly recover from any failure.
61
+ - Running reliable background jobs with no timeouts.
62
+ - Processing incoming events (e.g. from Kafka) exactly once.
63
+ - Running a fault-tolerant distributed task queue.
64
+ - Running a reliable cron scheduler.
65
+ - Operating an AI agent, or anything that connects to an unreliable or non-deterministic API.
64
66
 
65
- Some more cool features include:
67
+ What’s unique about DBOS's implementation of durable execution is that it’s implemented in a **lightweight library** that’s **totally backed by Postgres**.
68
+ To use DBOS, just `pip install` it and annotate your program with DBOS decorators.
69
+ Under the hood, those decorators store your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
70
+ If your program crashes or is interrupted, they automatically recover its workflows from their stored state.
71
+ So all you need to use DBOS is Postgres&mdash;there are no other dependencies you have to manage, no separate workflow server.
66
72
 
67
- - Scheduled jobs&mdash;run your workflows exactly-once per time interval.
68
- - Exactly-once event processing&mdash;use workflows to process incoming events (for example, from a Kafka topic) exactly-once.
69
- - Observability&mdash;all workflows automatically emit [OpenTelemetry](https://opentelemetry.io/) traces.
73
+ One big advantage of this approach is that you can add DBOS to **any** Python application&mdash;**it’s just a library**.
74
+ You can use DBOS to add reliable background jobs or cron scheduling or queues to your app with no external dependencies except Postgres.
70
75
 
71
76
  ## Getting Started
72
77
 
@@ -77,7 +82,7 @@ pip install dbos
77
82
  dbos init --config
78
83
  ```
79
84
 
80
- Then, try it out with this simple program (requires Postgres):
85
+ Then, try it out with this simple program:
81
86
 
82
87
  ```python
83
88
  from fastapi import FastAPI
@@ -107,14 +112,14 @@ def fastapi_endpoint():
107
112
  dbos_workflow()
108
113
  ```
109
114
 
110
- Save the program into `main.py`, edit `dbos-config.yaml` to configure your Postgres connection settings, and start it with `fastapi run`.
115
+ Save the program into `main.py` and start it with `fastapi run`.
111
116
  Visit `localhost:8000` in your browser to start the workflow.
112
117
  When prompted, press `Control + \` to force quit your application.
113
118
  It should crash midway through the workflow, having completed step one but not step two.
114
119
  Then, restart your app with `fastapi run`.
115
120
  It should resume the workflow from where it left off, completing step two without re-executing step one.
116
121
 
117
- To learn how to build more complex workflows, see our [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
122
+ To learn how to build more complex workflows, see the [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
118
123
 
119
124
  ## Documentation
120
125
 
@@ -125,7 +130,7 @@ To learn how to build more complex workflows, see our [programming guide](https:
125
130
 
126
131
  - [**AI-Powered Slackbot**](https://docs.dbos.dev/python/examples/rag-slackbot) &mdash; A Slackbot that answers questions about previous Slack conversations, using DBOS to durably orchestrate its RAG pipeline.
127
132
  - [**Widget Store**](https://docs.dbos.dev/python/examples/widget-store) &mdash; An online storefront that uses DBOS durable workflows to be resilient to any failure.
128
- - [**Earthquake Tracker**](https://docs.dbos.dev/python/examples/earthquake-tracker) &mdash; A real-time earthquake dashboard that uses DBOS to stream data from the USGS into Postgres, then visualizes it with Streamlit.
133
+ - [**Scheduled Reminders**](https://docs.dbos.dev/python/examples/scheduled-reminders) &mdash; In just three lines of code, schedule an email to send days, weeks, or months in the future.
129
134
 
130
135
  More examples [here](https://docs.dbos.dev/examples)!
131
136
 
@@ -1,7 +1,7 @@
1
- dbos-0.18.0.dist-info/METADATA,sha256=Q2jgOnOohb9XzktpZLozPJ-iG4TG99Fli3i917_EKCY,5142
2
- dbos-0.18.0.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
3
- dbos-0.18.0.dist-info/entry_points.txt,sha256=z6GcVANQV7Uw_82H9Ob2axJX6V3imftyZsljdh-M1HU,54
4
- dbos-0.18.0.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
1
+ dbos-0.19.0.dist-info/METADATA,sha256=Xce4k1qg4PeKccISSh9dZEUxiopjpJlzowY5NureWAM,5307
2
+ dbos-0.19.0.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
3
+ dbos-0.19.0.dist-info/entry_points.txt,sha256=z6GcVANQV7Uw_82H9Ob2axJX6V3imftyZsljdh-M1HU,54
4
+ dbos-0.19.0.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
5
5
  dbos/__init__.py,sha256=CxRHBHEthPL4PZoLbZhp3rdm44-KkRTT2-7DkK9d4QQ,724
6
6
  dbos/_admin_server.py,sha256=DOgzVp9kmwiebQqmJB1LcrZnGTxSMbZiGXdenc1wZDg,3163
7
7
  dbos/_app_db.py,sha256=_tv2vmPjjiaikwgxH3mqxgJ4nUUcG2-0uMXKWCqVu1c,5509
@@ -9,20 +9,21 @@ dbos/_classproperty.py,sha256=f0X-_BySzn3yFDRKB2JpCbLYQ9tLwt1XftfshvY7CBs,626
9
9
  dbos/_cloudutils/authentication.py,sha256=V0fCWQN9stCkhbuuxgPTGpvuQcDqfU3KAxPAh01vKW4,5007
10
10
  dbos/_cloudutils/cloudutils.py,sha256=5e3CW1deSW-dI5G3QN0XbiVsBhyqT8wu7fuV2f8wtGU,7688
11
11
  dbos/_cloudutils/databases.py,sha256=x4187Djsyoa-QaG3Kog8JT2_GERsnqa93LIVanmVUmg,8393
12
- dbos/_context.py,sha256=KV3fd3-Rv6EWrYDUdHARxltSlNZGNtQtNSqeQ-gkXE8,18049
13
- dbos/_core.py,sha256=NWJFQX5bECBvKlYH9pVmNJgmqFGYPnkHnOGjOlOQ3Ag,33504
12
+ dbos/_context.py,sha256=RH08s_nee95vgxdz6AsYuVWF1LuJSVtOyIifblsa4pw,18760
13
+ dbos/_core.py,sha256=lonD_iSnZtxhwetDcDlCV3HLr40jjqxUS2Q0TFaIOt0,34784
14
14
  dbos/_croniter.py,sha256=hbhgfsHBqclUS8VeLnJ9PSE9Z54z6mi4nnrr1aUXn0k,47561
15
- dbos/_db_wizard.py,sha256=uSNgJwbN7Mqor8jcenBnrHR2tz123ljtAYUP3Jw-NIY,6341
16
- dbos/_dbos.py,sha256=z12yGw2QHx7BLdxvoI2zJiwDTSes1r48E7qZ7uOn0mw,34723
17
- dbos/_dbos_config.py,sha256=Hs9L-PJhxeGa2R3nDX75ZFOGLRxA3AiXVf7UoemN9lM,6643
18
- dbos/_error.py,sha256=UETk8CoZL-TO2Utn1-E7OSWelhShWmKM-fOlODMR9PE,3893
15
+ dbos/_db_wizard.py,sha256=xgKLna0_6Xi50F3o8msRosXba8NScHlpJR5ICVCkHDQ,7534
16
+ dbos/_dbos.py,sha256=LWFa48CPt7bsNAnMZrNDzHHTFCyMrY-nKbMZwCG_dqY,34710
17
+ dbos/_dbos_config.py,sha256=h_q1gzudhsAMVkGMD0qQ6kLic6YhdJgzm50YFSIx9Bo,8196
18
+ dbos/_error.py,sha256=vtaSsG0QW6cRlwfZ4zzZWy_IHCZlomwSlrDyGWuyn8c,4337
19
19
  dbos/_fastapi.py,sha256=iyefCZq-ZDKRUjN_rgYQmFmyvWf4gPrSlC6CLbfq4a8,3419
20
20
  dbos/_flask.py,sha256=z1cijbTi5Dpq6kqikPCx1LcR2YHHv2oc41NehOWjw74,2431
21
- dbos/_kafka.py,sha256=OmOKfO7_3Z2FUFv_sJaIfebd7xnqtuRRndzNTTufgb8,3654
21
+ dbos/_kafka.py,sha256=o6DbwnsYRDtvVTZVsN7BAK8cdP79AfoWX3Q7CGY2Yuo,4199
22
22
  dbos/_kafka_message.py,sha256=NYvOXNG3Qn7bghn1pv3fg4Pbs86ILZGcK4IB-MLUNu0,409
23
23
  dbos/_logger.py,sha256=iYwbA7DLyXalWa2Yu07HO6Xm301nRuenMU64GgwUMkU,3576
24
24
  dbos/_migrations/env.py,sha256=38SIGVbmn_VV2x2u1aHLcPOoWgZ84eCymf3g_NljmbU,1626
25
25
  dbos/_migrations/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvDgl93dMj8,635
26
+ dbos/_migrations/versions/04ca4f231047_workflow_queues_executor_id.py,sha256=ICLPl8CN9tQXMsLDsAj8z1TsL831-Z3F8jSBvrR-wyw,736
26
27
  dbos/_migrations/versions/50f3227f0b4b_fix_job_queue.py,sha256=ZBYrtTdxy64HxIAlOes89fVIk2P1gNaJack7wuC_epg,873
27
28
  dbos/_migrations/versions/5c361fc04708_added_system_tables.py,sha256=QMgFMb0aLgC25YicsvPSr6AHRCA6Zd66hyaRUhwKzrQ,6404
28
29
  dbos/_migrations/versions/a3b18ad34abe_added_triggers.py,sha256=Rv0ZsZYZ_WdgGEULYsPfnp4YzaO5L198gDTgYY39AVA,2022
@@ -30,7 +31,7 @@ dbos/_migrations/versions/d76646551a6b_job_queue_limiter.py,sha256=8PyFi8rd6CN-m
30
31
  dbos/_migrations/versions/d76646551a6c_workflow_queue.py,sha256=G942nophZ2uC2vc4hGBC02Ptng1715roTjY3xiyzZU4,729
31
32
  dbos/_migrations/versions/eab0cc1d9a14_job_queue.py,sha256=uvhFOtqbBreCePhAxZfIT0qCAI7BiZTou9wt6QnbY7c,1412
32
33
  dbos/_outcome.py,sha256=FDMgWVjZ06vm9xO-38H17mTqBImUYQxgKs_bDCSIAhE,6648
33
- dbos/_queue.py,sha256=5NZ6RfKQd8LQD8EeUXgrwu86r0AadKEqPIMmL_1ORuw,1956
34
+ dbos/_queue.py,sha256=o_aczwualJTMoXb0XXL-Y5QH77OEukWzuerogbWi2ho,2779
34
35
  dbos/_recovery.py,sha256=jbzGYxICA2drzyzlBSy2UiXhKV_16tBVacKQdTkqf-w,2008
35
36
  dbos/_registrations.py,sha256=mei6q6_3R5uei8i_Wo_TqGZs85s10shOekDX41sFYD0,6642
36
37
  dbos/_request.py,sha256=cX1B3Atlh160phgS35gF1VEEV4pD126c9F3BDgBmxZU,929
@@ -38,22 +39,23 @@ dbos/_roles.py,sha256=iOsgmIAf1XVzxs3gYWdGRe1B880YfOw5fpU7Jwx8_A8,2271
38
39
  dbos/_scheduler.py,sha256=0I3e8Y-OIBG3wiUCIskShd-Sk_eUFCFyRB5u4L7IHXI,1940
39
40
  dbos/_schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
40
41
  dbos/_schemas/application_database.py,sha256=KeyoPrF7hy_ODXV7QNike_VFSD74QBRfQ76D7QyE9HI,966
41
- dbos/_schemas/system_database.py,sha256=7iw7eHJzEvkatHMOaHORoSvtfisF73wW5j8hRt_Ph14,5126
42
+ dbos/_schemas/system_database.py,sha256=rwp4EvCSaXcUoMaRczZCvETCxGp72k3-hvLyGUDkih0,5163
42
43
  dbos/_serialization.py,sha256=YCYv0qKAwAZ1djZisBC7khvKqG-5OcIv9t9EC5PFIog,1743
43
- dbos/_sys_db.py,sha256=uZKeCnGc2MgvEd0ID3nReBBZj21HzClP56TFkXTvIZE,49028
44
+ dbos/_sys_db.py,sha256=ha5E11P83oi78L4R7cX_OL_N1Tf2Ir0Xr30GK1_27SA,52290
44
45
  dbos/_templates/hello/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
45
46
  dbos/_templates/hello/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
46
47
  dbos/_templates/hello/__package/main.py,sha256=eI0SS9Nwj-fldtiuSzIlIG6dC91GXXwdRsoHxv6S_WI,2719
47
48
  dbos/_templates/hello/__package/schema.py,sha256=7Z27JGC8yy7Z44cbVXIREYxtUhU4JVkLCp5Q7UahVQ0,260
48
49
  dbos/_templates/hello/alembic.ini,sha256=VKBn4Gy8mMuCdY7Hip1jmo3wEUJ1VG1aW7EqY0_n-as,3695
49
- dbos/_templates/hello/dbos-config.yaml.dbos,sha256=7yu1q8FAgOZnwJtU-e_5qgV-wkHRn6cqo-GEmk9rK8U,577
50
+ dbos/_templates/hello/dbos-config.yaml.dbos,sha256=OMlcpdYUJKjyAme7phOz3pbn9upcIRjm42iwEThWUEQ,495
50
51
  dbos/_templates/hello/migrations/env.py.dbos,sha256=GUV6sjkDzf9Vl6wkGEd0RSkK-ftRfV6EUwSQdd0qFXg,2392
51
52
  dbos/_templates/hello/migrations/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvDgl93dMj8,635
52
53
  dbos/_templates/hello/migrations/versions/2024_07_31_180642_init.py,sha256=U5thFWGqNN4QLrNXT7wUUqftIFDNE5eSdqD8JNW1mec,942
53
54
  dbos/_templates/hello/start_postgres_docker.py,sha256=lQVLlYO5YkhGPEgPqwGc7Y8uDKse9HsWv5fynJEFJHM,1681
54
55
  dbos/_tracer.py,sha256=rvBY1RQU6DO7rL7EnaJJxGcmd4tP_PpGqUEE6imZnhY,2518
55
- dbos/cli.py,sha256=em1uAxrp5yyg53V7ZpmHFtqD6OJp2cMJkG9vGJPoFTA,10904
56
- dbos/dbos-config.schema.json,sha256=00lliu7hGT6NAIZt8UNAn8mEhQ71RGw6Q2CI3nWxULA,5899
56
+ dbos/_workflow_commands.py,sha256=25mLcPifaaQtX_Wzrf2LVq4CtXGDjmLHABimTcOeQuw,4691
57
+ dbos/cli.py,sha256=0E_QDJm3aGjjauUnmrsdZkqc8U49L6j2uPEtA0QRaZE,13946
58
+ dbos/dbos-config.schema.json,sha256=X5TpXNcARGceX0zQs0fVgtZW_Xj9uBbY5afPt9Rz9yk,5741
57
59
  dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
58
60
  version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
59
- dbos-0.18.0.dist-info/RECORD,,
61
+ dbos-0.19.0.dist-info/RECORD,,
File without changes