dbos 0.19.0a4__tar.gz → 0.19.0a9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbos might be problematic. Click here for more details.

Files changed (87) hide show
  1. {dbos-0.19.0a4 → dbos-0.19.0a9}/PKG-INFO +21 -16
  2. {dbos-0.19.0a4 → dbos-0.19.0a9}/README.md +20 -15
  3. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_context.py +11 -2
  4. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_core.py +5 -1
  5. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_error.py +11 -0
  6. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_kafka.py +17 -1
  7. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_queue.py +1 -0
  8. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_sys_db.py +69 -37
  9. dbos-0.19.0a9/dbos/_workflow_commands.py +172 -0
  10. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/cli.py +100 -1
  11. {dbos-0.19.0a4 → dbos-0.19.0a9}/pyproject.toml +1 -1
  12. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/queuedworkflow.py +1 -0
  13. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_classdecorators.py +1 -0
  14. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_dbos.py +6 -3
  15. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_kafka.py +37 -1
  16. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_queue.py +115 -2
  17. dbos-0.19.0a9/tests/test_workflow_cmds.py +216 -0
  18. {dbos-0.19.0a4 → dbos-0.19.0a9}/LICENSE +0 -0
  19. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/__init__.py +0 -0
  20. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_admin_server.py +0 -0
  21. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_app_db.py +0 -0
  22. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_classproperty.py +0 -0
  23. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_cloudutils/authentication.py +0 -0
  24. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_cloudutils/cloudutils.py +0 -0
  25. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_cloudutils/databases.py +0 -0
  26. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_croniter.py +0 -0
  27. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_db_wizard.py +0 -0
  28. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_dbos.py +0 -0
  29. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_dbos_config.py +0 -0
  30. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_fastapi.py +0 -0
  31. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_flask.py +0 -0
  32. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_kafka_message.py +0 -0
  33. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_logger.py +0 -0
  34. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/env.py +0 -0
  35. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/script.py.mako +0 -0
  36. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/04ca4f231047_workflow_queues_executor_id.py +0 -0
  37. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/50f3227f0b4b_fix_job_queue.py +0 -0
  38. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/5c361fc04708_added_system_tables.py +0 -0
  39. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/a3b18ad34abe_added_triggers.py +0 -0
  40. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/d76646551a6b_job_queue_limiter.py +0 -0
  41. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/d76646551a6c_workflow_queue.py +0 -0
  42. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/eab0cc1d9a14_job_queue.py +0 -0
  43. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_outcome.py +0 -0
  44. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_recovery.py +0 -0
  45. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_registrations.py +0 -0
  46. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_request.py +0 -0
  47. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_roles.py +0 -0
  48. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_scheduler.py +0 -0
  49. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_schemas/__init__.py +0 -0
  50. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_schemas/application_database.py +0 -0
  51. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_schemas/system_database.py +0 -0
  52. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_serialization.py +0 -0
  53. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/README.md +0 -0
  54. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/__package/__init__.py +0 -0
  55. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/__package/main.py +0 -0
  56. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/__package/schema.py +0 -0
  57. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/alembic.ini +0 -0
  58. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/dbos-config.yaml.dbos +0 -0
  59. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/migrations/env.py.dbos +0 -0
  60. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/migrations/script.py.mako +0 -0
  61. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/migrations/versions/2024_07_31_180642_init.py +0 -0
  62. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/start_postgres_docker.py +0 -0
  63. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_tracer.py +0 -0
  64. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/dbos-config.schema.json +0 -0
  65. {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/py.typed +0 -0
  66. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/__init__.py +0 -0
  67. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/atexit_no_ctor.py +0 -0
  68. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/atexit_no_launch.py +0 -0
  69. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/classdefs.py +0 -0
  70. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/conftest.py +0 -0
  71. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/more_classdefs.py +0 -0
  72. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_admin_server.py +0 -0
  73. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_async.py +0 -0
  74. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_concurrency.py +0 -0
  75. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_config.py +0 -0
  76. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_croniter.py +0 -0
  77. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_failures.py +0 -0
  78. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_fastapi.py +0 -0
  79. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_fastapi_roles.py +0 -0
  80. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_flask.py +0 -0
  81. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_outcome.py +0 -0
  82. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_package.py +0 -0
  83. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_scheduler.py +0 -0
  84. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_schema_migration.py +0 -0
  85. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_singleton.py +0 -0
  86. {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_spans.py +0 -0
  87. {dbos-0.19.0a4 → dbos-0.19.0a9}/version/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dbos
3
- Version: 0.19.0a4
3
+ Version: 0.19.0a9
4
4
  Summary: Ultra-lightweight durable execution in Python
5
5
  Author-Email: "DBOS, Inc." <contact@dbos.dev>
6
6
  License: MIT
@@ -28,14 +28,14 @@ Description-Content-Type: text/markdown
28
28
 
29
29
  <div align="center">
30
30
 
31
- # DBOS Transact: Ultra-Lightweight Durable Execution
31
+ # DBOS Transact: A Lightweight Durable Execution Library Built on Postgres
32
32
 
33
33
  #### [Documentation](https://docs.dbos.dev/) &nbsp;&nbsp;•&nbsp;&nbsp; [Examples](https://docs.dbos.dev/examples) &nbsp;&nbsp;•&nbsp;&nbsp; [Github](https://github.com/dbos-inc) &nbsp;&nbsp;•&nbsp;&nbsp; [Discord](https://discord.com/invite/jsmC6pXGgX)
34
34
  </div>
35
35
 
36
36
  ---
37
37
 
38
- DBOS Transact is a Python library providing **ultra-lightweight durable execution**.
38
+ DBOS Transact is a Python library for **ultra-lightweight durable execution**.
39
39
  For example:
40
40
 
41
41
  ```python
@@ -55,18 +55,23 @@ def workflow()
55
55
 
56
56
  Durable execution means your program is **resilient to any failure**.
57
57
  If it is ever interrupted or crashes, all your workflows will automatically resume from the last completed step.
58
- If you want to see durable execution in action, check out [this demo app](https://demo-widget-store.cloud.dbos.dev/) (source code [here](https://github.com/dbos-inc/dbos-demo-apps/tree/main/python/widget-store)).
59
- No matter how many times you try to crash it, it always resumes from exactly where it left off!
58
+ Durable execution helps solve many common problems:
60
59
 
61
- Under the hood, DBOS Transact works by storing your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
62
- So all you need to use it is a Postgres database to connect to&mdash;there's no need for a "workflow server."
63
- This approach is also incredibly fast, for example [25x faster than AWS Step Functions](https://www.dbos.dev/blog/dbos-vs-aws-step-functions-benchmark).
60
+ - Orchestrating long-running or business-critical workflows so they seamlessly recover from any failure.
61
+ - Running reliable background jobs with no timeouts.
62
+ - Processing incoming events (e.g. from Kafka) exactly once.
63
+ - Running a fault-tolerant distributed task queue.
64
+ - Running a reliable cron scheduler.
65
+ - Operating an AI agent, or anything that connects to an unreliable or non-deterministic API.
64
66
 
65
- Some more cool features include:
67
+ What’s unique about DBOS's implementation of durable execution is that it’s implemented in a **lightweight library** that’s **totally backed by Postgres**.
68
+ To use DBOS, just `pip install` it and annotate your program with DBOS decorators.
69
+ Under the hood, those decorators store your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
70
+ If your program crashes or is interrupted, they automatically recover its workflows from their stored state.
71
+ So all you need to use DBOS is Postgres&mdash;there are no other dependencies you have to manage, no separate workflow server.
66
72
 
67
- - Scheduled jobs&mdash;run your workflows exactly-once per time interval.
68
- - Exactly-once event processing&mdash;use workflows to process incoming events (for example, from a Kafka topic) exactly-once.
69
- - Observability&mdash;all workflows automatically emit [OpenTelemetry](https://opentelemetry.io/) traces.
73
+ One big advantage of this approach is that you can add DBOS to **any** Python application&mdash;**it’s just a library**.
74
+ You can use DBOS to add reliable background jobs or cron scheduling or queues to your app with no external dependencies except Postgres.
70
75
 
71
76
  ## Getting Started
72
77
 
@@ -77,7 +82,7 @@ pip install dbos
77
82
  dbos init --config
78
83
  ```
79
84
 
80
- Then, try it out with this simple program (requires Postgres):
85
+ Then, try it out with this simple program:
81
86
 
82
87
  ```python
83
88
  from fastapi import FastAPI
@@ -107,14 +112,14 @@ def fastapi_endpoint():
107
112
  dbos_workflow()
108
113
  ```
109
114
 
110
- Save the program into `main.py`, edit `dbos-config.yaml` to configure your Postgres connection settings, and start it with `fastapi run`.
115
+ Save the program into `main.py` and start it with `fastapi run`.
111
116
  Visit `localhost:8000` in your browser to start the workflow.
112
117
  When prompted, press `Control + \` to force quit your application.
113
118
  It should crash midway through the workflow, having completed step one but not step two.
114
119
  Then, restart your app with `fastapi run`.
115
120
  It should resume the workflow from where it left off, completing step two without re-executing step one.
116
121
 
117
- To learn how to build more complex workflows, see our [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
122
+ To learn how to build more complex workflows, see the [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
118
123
 
119
124
  ## Documentation
120
125
 
@@ -125,7 +130,7 @@ To learn how to build more complex workflows, see our [programming guide](https:
125
130
 
126
131
  - [**AI-Powered Slackbot**](https://docs.dbos.dev/python/examples/rag-slackbot) &mdash; A Slackbot that answers questions about previous Slack conversations, using DBOS to durably orchestrate its RAG pipeline.
127
132
  - [**Widget Store**](https://docs.dbos.dev/python/examples/widget-store) &mdash; An online storefront that uses DBOS durable workflows to be resilient to any failure.
128
- - [**Earthquake Tracker**](https://docs.dbos.dev/python/examples/earthquake-tracker) &mdash; A real-time earthquake dashboard that uses DBOS to stream data from the USGS into Postgres, then visualizes it with Streamlit.
133
+ - [**Scheduled Reminders**](https://docs.dbos.dev/python/examples/scheduled-reminders) &mdash; In just three lines of code, schedule an email to send days, weeks, or months in the future.
129
134
 
130
135
  More examples [here](https://docs.dbos.dev/examples)!
131
136
 
@@ -1,14 +1,14 @@
1
1
 
2
2
  <div align="center">
3
3
 
4
- # DBOS Transact: Ultra-Lightweight Durable Execution
4
+ # DBOS Transact: A Lightweight Durable Execution Library Built on Postgres
5
5
 
6
6
  #### [Documentation](https://docs.dbos.dev/) &nbsp;&nbsp;•&nbsp;&nbsp; [Examples](https://docs.dbos.dev/examples) &nbsp;&nbsp;•&nbsp;&nbsp; [Github](https://github.com/dbos-inc) &nbsp;&nbsp;•&nbsp;&nbsp; [Discord](https://discord.com/invite/jsmC6pXGgX)
7
7
  </div>
8
8
 
9
9
  ---
10
10
 
11
- DBOS Transact is a Python library providing **ultra-lightweight durable execution**.
11
+ DBOS Transact is a Python library for **ultra-lightweight durable execution**.
12
12
  For example:
13
13
 
14
14
  ```python
@@ -28,18 +28,23 @@ def workflow()
28
28
 
29
29
  Durable execution means your program is **resilient to any failure**.
30
30
  If it is ever interrupted or crashes, all your workflows will automatically resume from the last completed step.
31
- If you want to see durable execution in action, check out [this demo app](https://demo-widget-store.cloud.dbos.dev/) (source code [here](https://github.com/dbos-inc/dbos-demo-apps/tree/main/python/widget-store)).
32
- No matter how many times you try to crash it, it always resumes from exactly where it left off!
31
+ Durable execution helps solve many common problems:
33
32
 
34
- Under the hood, DBOS Transact works by storing your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
35
- So all you need to use it is a Postgres database to connect to&mdash;there's no need for a "workflow server."
36
- This approach is also incredibly fast, for example [25x faster than AWS Step Functions](https://www.dbos.dev/blog/dbos-vs-aws-step-functions-benchmark).
33
+ - Orchestrating long-running or business-critical workflows so they seamlessly recover from any failure.
34
+ - Running reliable background jobs with no timeouts.
35
+ - Processing incoming events (e.g. from Kafka) exactly once.
36
+ - Running a fault-tolerant distributed task queue.
37
+ - Running a reliable cron scheduler.
38
+ - Operating an AI agent, or anything that connects to an unreliable or non-deterministic API.
37
39
 
38
- Some more cool features include:
40
+ What’s unique about DBOS's implementation of durable execution is that it’s implemented in a **lightweight library** that’s **totally backed by Postgres**.
41
+ To use DBOS, just `pip install` it and annotate your program with DBOS decorators.
42
+ Under the hood, those decorators store your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
43
+ If your program crashes or is interrupted, they automatically recover its workflows from their stored state.
44
+ So all you need to use DBOS is Postgres&mdash;there are no other dependencies you have to manage, no separate workflow server.
39
45
 
40
- - Scheduled jobs&mdash;run your workflows exactly-once per time interval.
41
- - Exactly-once event processing&mdash;use workflows to process incoming events (for example, from a Kafka topic) exactly-once.
42
- - Observability&mdash;all workflows automatically emit [OpenTelemetry](https://opentelemetry.io/) traces.
46
+ One big advantage of this approach is that you can add DBOS to **any** Python application&mdash;**it’s just a library**.
47
+ You can use DBOS to add reliable background jobs or cron scheduling or queues to your app with no external dependencies except Postgres.
43
48
 
44
49
  ## Getting Started
45
50
 
@@ -50,7 +55,7 @@ pip install dbos
50
55
  dbos init --config
51
56
  ```
52
57
 
53
- Then, try it out with this simple program (requires Postgres):
58
+ Then, try it out with this simple program:
54
59
 
55
60
  ```python
56
61
  from fastapi import FastAPI
@@ -80,14 +85,14 @@ def fastapi_endpoint():
80
85
  dbos_workflow()
81
86
  ```
82
87
 
83
- Save the program into `main.py`, edit `dbos-config.yaml` to configure your Postgres connection settings, and start it with `fastapi run`.
88
+ Save the program into `main.py` and start it with `fastapi run`.
84
89
  Visit `localhost:8000` in your browser to start the workflow.
85
90
  When prompted, press `Control + \` to force quit your application.
86
91
  It should crash midway through the workflow, having completed step one but not step two.
87
92
  Then, restart your app with `fastapi run`.
88
93
  It should resume the workflow from where it left off, completing step two without re-executing step one.
89
94
 
90
- To learn how to build more complex workflows, see our [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
95
+ To learn how to build more complex workflows, see the [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
91
96
 
92
97
  ## Documentation
93
98
 
@@ -98,7 +103,7 @@ To learn how to build more complex workflows, see our [programming guide](https:
98
103
 
99
104
  - [**AI-Powered Slackbot**](https://docs.dbos.dev/python/examples/rag-slackbot) &mdash; A Slackbot that answers questions about previous Slack conversations, using DBOS to durably orchestrate its RAG pipeline.
100
105
  - [**Widget Store**](https://docs.dbos.dev/python/examples/widget-store) &mdash; An online storefront that uses DBOS durable workflows to be resilient to any failure.
101
- - [**Earthquake Tracker**](https://docs.dbos.dev/python/examples/earthquake-tracker) &mdash; A real-time earthquake dashboard that uses DBOS to stream data from the USGS into Postgres, then visualizes it with Streamlit.
106
+ - [**Scheduled Reminders**](https://docs.dbos.dev/python/examples/scheduled-reminders) &mdash; In just three lines of code, schedule an email to send days, weeks, or months in the future.
102
107
 
103
108
  More examples [here](https://docs.dbos.dev/examples)!
104
109
 
@@ -57,6 +57,7 @@ class DBOSContext:
57
57
  self.request: Optional["Request"] = None
58
58
 
59
59
  self.id_assigned_for_next_workflow: str = ""
60
+ self.is_within_set_workflow_id_block: bool = False
60
61
 
61
62
  self.parent_workflow_id: str = ""
62
63
  self.parent_workflow_fid: int = -1
@@ -78,6 +79,7 @@ class DBOSContext:
78
79
  rv.logger = self.logger
79
80
  rv.id_assigned_for_next_workflow = self.id_assigned_for_next_workflow
80
81
  self.id_assigned_for_next_workflow = ""
82
+ rv.is_within_set_workflow_id_block = self.is_within_set_workflow_id_block
81
83
  rv.parent_workflow_id = self.workflow_id
82
84
  rv.parent_workflow_fid = self.function_id
83
85
  rv.in_recovery = self.in_recovery
@@ -95,6 +97,10 @@ class DBOSContext:
95
97
  if len(self.id_assigned_for_next_workflow) > 0:
96
98
  wfid = self.id_assigned_for_next_workflow
97
99
  else:
100
+ if self.is_within_set_workflow_id_block:
101
+ self.logger.warning(
102
+ f"Multiple workflows started in the same SetWorkflowID block. Only the first workflow is assigned the specified workflow ID; subsequent workflows will use a generated workflow ID."
103
+ )
98
104
  wfid = str(uuid.uuid4())
99
105
  return wfid
100
106
 
@@ -286,7 +292,7 @@ class DBOSContextSwap:
286
292
 
287
293
  class SetWorkflowID:
288
294
  """
289
- Set the workflow ID to be used for the enclosed workflow invocation.
295
+ Set the workflow ID to be used for the enclosed workflow invocation. Note: Only the first workflow will be started with the specified workflow ID within a `with SetWorkflowID` block.
290
296
 
291
297
  Typical Usage
292
298
  ```
@@ -311,7 +317,9 @@ class SetWorkflowID:
311
317
  if ctx is None:
312
318
  self.created_ctx = True
313
319
  _set_local_dbos_context(DBOSContext())
314
- assert_current_dbos_context().id_assigned_for_next_workflow = self.wfid
320
+ ctx = assert_current_dbos_context()
321
+ ctx.id_assigned_for_next_workflow = self.wfid
322
+ ctx.is_within_set_workflow_id_block = True
315
323
  return self
316
324
 
317
325
  def __exit__(
@@ -321,6 +329,7 @@ class SetWorkflowID:
321
329
  traceback: Optional[TracebackType],
322
330
  ) -> Literal[False]:
323
331
  # Code to clean up the basic context if we created it
332
+ assert_current_dbos_context().is_within_set_workflow_id_block = False
324
333
  if self.created_ctx:
325
334
  _clear_local_dbos_context()
326
335
  return False # Did not handle
@@ -188,6 +188,7 @@ def _init_workflow(
188
188
  wf_status = dbos._sys_db.update_workflow_status(
189
189
  status, False, ctx.in_recovery, max_recovery_attempts=max_recovery_attempts
190
190
  )
191
+ # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
191
192
  dbos._sys_db.update_workflow_inputs(wfid, _serialization.serialize_args(inputs))
192
193
  else:
193
194
  # Buffer the inputs for single-transaction workflows, but don't buffer the status
@@ -422,6 +423,9 @@ def start_workflow(
422
423
  or wf_status == WorkflowStatusString.ERROR.value
423
424
  or wf_status == WorkflowStatusString.SUCCESS.value
424
425
  ):
426
+ dbos.logger.debug(
427
+ f"Workflow {new_wf_id} already completed with status {wf_status}. Directly returning a workflow handle."
428
+ )
425
429
  return WorkflowHandlePolling(new_wf_id, dbos)
426
430
 
427
431
  if fself is not None:
@@ -494,7 +498,7 @@ def workflow_wrapper(
494
498
  temp_wf_type=get_temp_workflow_type(func),
495
499
  max_recovery_attempts=max_recovery_attempts,
496
500
  )
497
-
501
+ # TODO: maybe modify the parameters if they've been changed by `_init_workflow`
498
502
  dbos.logger.debug(
499
503
  f"Running workflow, id: {ctx.workflow_id}, name: {get_dbos_func_name(func)}"
500
504
  )
@@ -35,6 +35,7 @@ class DBOSErrorCode(Enum):
35
35
  DeadLetterQueueError = 6
36
36
  MaxStepRetriesExceeded = 7
37
37
  NotAuthorized = 8
38
+ ConflictingWorkflowError = 9
38
39
 
39
40
 
40
41
  class DBOSWorkflowConflictIDError(DBOSException):
@@ -47,6 +48,16 @@ class DBOSWorkflowConflictIDError(DBOSException):
47
48
  )
48
49
 
49
50
 
51
+ class DBOSConflictingWorkflowError(DBOSException):
52
+ """Exception raised different workflows started with the same workflow ID."""
53
+
54
+ def __init__(self, workflow_id: str, message: Optional[str] = None):
55
+ super().__init__(
56
+ f"Conflicting workflow invocation with the same ID ({workflow_id}): {message}",
57
+ dbos_error_code=DBOSErrorCode.ConflictingWorkflowError.value,
58
+ )
59
+
60
+
50
61
  class DBOSRecoveryError(DBOSException):
51
62
  """Exception raised when a workflow recovery fails."""
52
63
 
@@ -1,3 +1,4 @@
1
+ import re
1
2
  import threading
2
3
  from typing import TYPE_CHECKING, Any, Callable, NoReturn
3
4
 
@@ -19,6 +20,14 @@ _kafka_queue: Queue
19
20
  _in_order_kafka_queues: dict[str, Queue] = {}
20
21
 
21
22
 
23
+ def safe_group_name(method_name: str, topics: list[str]) -> str:
24
+ safe_group_id = "-".join(
25
+ re.sub(r"[^a-zA-Z0-9\-]", "", str(r)) for r in [method_name, *topics]
26
+ )
27
+
28
+ return f"dbos-kafka-group-{safe_group_id}"[:255]
29
+
30
+
22
31
  def _kafka_consumer_loop(
23
32
  func: _KafkaConsumerWorkflow,
24
33
  config: dict[str, Any],
@@ -34,6 +43,12 @@ def _kafka_consumer_loop(
34
43
  if "auto.offset.reset" not in config:
35
44
  config["auto.offset.reset"] = "earliest"
36
45
 
46
+ if config.get("group.id") is None:
47
+ config["group.id"] = safe_group_name(func.__qualname__, topics)
48
+ dbos_logger.warning(
49
+ f"Consumer group ID not found. Using generated group.id {config['group.id']}"
50
+ )
51
+
37
52
  consumer = Consumer(config)
38
53
  try:
39
54
  consumer.subscribe(topics)
@@ -71,8 +86,9 @@ def _kafka_consumer_loop(
71
86
  topic=cmsg.topic(),
72
87
  value=cmsg.value(),
73
88
  )
89
+ groupID = config.get("group.id")
74
90
  with SetWorkflowID(
75
- f"kafka-unique-id-{msg.topic}-{msg.partition}-{msg.offset}"
91
+ f"kafka-unique-id-{msg.topic}-{msg.partition}-{groupID}-{msg.offset}"
76
92
  ):
77
93
  if in_order:
78
94
  assert msg.topic is not None
@@ -36,6 +36,7 @@ class Queue:
36
36
  name: str,
37
37
  concurrency: Optional[int] = None,
38
38
  limiter: Optional[QueueRateLimit] = None,
39
+ *, # Disable positional arguments from here on
39
40
  worker_concurrency: Optional[int] = None,
40
41
  ) -> None:
41
42
  if (
@@ -28,6 +28,7 @@ from sqlalchemy.exc import DBAPIError
28
28
  from . import _serialization
29
29
  from ._dbos_config import ConfigFile
30
30
  from ._error import (
31
+ DBOSConflictingWorkflowError,
31
32
  DBOSDeadLetterQueueError,
32
33
  DBOSException,
33
34
  DBOSNonExistentWorkflowError,
@@ -288,8 +289,14 @@ class SystemDatabase:
288
289
  ),
289
290
  )
290
291
  else:
291
- cmd = cmd.on_conflict_do_nothing()
292
- cmd = cmd.returning(SystemSchema.workflow_status.c.recovery_attempts, SystemSchema.workflow_status.c.status) # type: ignore
292
+ # A blank update so that we can return the existing status
293
+ cmd = cmd.on_conflict_do_update(
294
+ index_elements=["workflow_uuid"],
295
+ set_=dict(
296
+ recovery_attempts=SystemSchema.workflow_status.c.recovery_attempts
297
+ ),
298
+ )
299
+ cmd = cmd.returning(SystemSchema.workflow_status.c.recovery_attempts, SystemSchema.workflow_status.c.status, SystemSchema.workflow_status.c.name, SystemSchema.workflow_status.c.class_name, SystemSchema.workflow_status.c.config_name, SystemSchema.workflow_status.c.queue_name) # type: ignore
293
300
 
294
301
  if conn is not None:
295
302
  results = conn.execute(cmd)
@@ -297,37 +304,53 @@ class SystemDatabase:
297
304
  with self.engine.begin() as c:
298
305
  results = c.execute(cmd)
299
306
 
300
- if in_recovery:
301
- row = results.fetchone()
302
- if row is not None:
303
- recovery_attempts: int = row[0]
304
- wf_status = row[1]
305
- if recovery_attempts > max_recovery_attempts:
306
- with self.engine.begin() as c:
307
- c.execute(
308
- sa.delete(SystemSchema.workflow_queue).where(
309
- SystemSchema.workflow_queue.c.workflow_uuid
310
- == status["workflow_uuid"]
311
- )
307
+ row = results.fetchone()
308
+ if row is not None:
309
+ # Check the started workflow matches the expected name, class_name, config_name, and queue_name
310
+ # A mismatch indicates a workflow starting with the same UUID but different functions, which would throw an exception.
311
+ recovery_attempts: int = row[0]
312
+ wf_status = row[1]
313
+ err_msg: Optional[str] = None
314
+ if row[2] != status["name"]:
315
+ err_msg = f"Workflow already exists with a different function name: {row[2]}, but the provided function name is: {status['name']}"
316
+ elif row[3] != status["class_name"]:
317
+ err_msg = f"Workflow already exists with a different class name: {row[3]}, but the provided class name is: {status['class_name']}"
318
+ elif row[4] != status["config_name"]:
319
+ err_msg = f"Workflow already exists with a different config name: {row[4]}, but the provided config name is: {status['config_name']}"
320
+ elif row[5] != status["queue_name"]:
321
+ # This is a warning because a different queue name is not necessarily an error.
322
+ dbos_logger.warning(
323
+ f"Workflow already exists in queue: {row[5]}, but the provided queue name is: {status['queue_name']}. The queue is not updated."
324
+ )
325
+ if err_msg is not None:
326
+ raise DBOSConflictingWorkflowError(status["workflow_uuid"], err_msg)
327
+
328
+ if in_recovery and recovery_attempts > max_recovery_attempts:
329
+ with self.engine.begin() as c:
330
+ c.execute(
331
+ sa.delete(SystemSchema.workflow_queue).where(
332
+ SystemSchema.workflow_queue.c.workflow_uuid
333
+ == status["workflow_uuid"]
312
334
  )
313
- c.execute(
314
- sa.update(SystemSchema.workflow_status)
315
- .where(
316
- SystemSchema.workflow_status.c.workflow_uuid
317
- == status["workflow_uuid"]
318
- )
319
- .where(
320
- SystemSchema.workflow_status.c.status
321
- == WorkflowStatusString.PENDING.value
322
- )
323
- .values(
324
- status=WorkflowStatusString.RETRIES_EXCEEDED.value,
325
- queue_name=None,
326
- )
335
+ )
336
+ c.execute(
337
+ sa.update(SystemSchema.workflow_status)
338
+ .where(
339
+ SystemSchema.workflow_status.c.workflow_uuid
340
+ == status["workflow_uuid"]
341
+ )
342
+ .where(
343
+ SystemSchema.workflow_status.c.status
344
+ == WorkflowStatusString.PENDING.value
345
+ )
346
+ .values(
347
+ status=WorkflowStatusString.RETRIES_EXCEEDED.value,
348
+ queue_name=None,
327
349
  )
328
- raise DBOSDeadLetterQueueError(
329
- status["workflow_uuid"], max_recovery_attempts
330
350
  )
351
+ raise DBOSDeadLetterQueueError(
352
+ status["workflow_uuid"], max_recovery_attempts
353
+ )
331
354
 
332
355
  # Record we have exported status for this single-transaction workflow
333
356
  if status["workflow_uuid"] in self._temp_txn_wf_ids:
@@ -356,7 +379,7 @@ class SystemDatabase:
356
379
  stmt = (
357
380
  sa.update(SystemSchema.workflow_status)
358
381
  .where(
359
- SystemSchema.workflow_inputs.c.workflow_uuid == workflow_uuid
382
+ SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid
360
383
  )
361
384
  .values(recovery_attempts=reset_recovery_attempts)
362
385
  )
@@ -538,18 +561,27 @@ class SystemDatabase:
538
561
  workflow_uuid=workflow_uuid,
539
562
  inputs=inputs,
540
563
  )
541
- .on_conflict_do_nothing()
564
+ .on_conflict_do_update(
565
+ index_elements=["workflow_uuid"],
566
+ set_=dict(workflow_uuid=SystemSchema.workflow_inputs.c.workflow_uuid),
567
+ )
568
+ .returning(SystemSchema.workflow_inputs.c.inputs)
542
569
  )
543
570
  if conn is not None:
544
- conn.execute(cmd)
571
+ row = conn.execute(cmd).fetchone()
545
572
  else:
546
573
  with self.engine.begin() as c:
547
- c.execute(cmd)
548
-
574
+ row = c.execute(cmd).fetchone()
575
+ if row is not None and row[0] != inputs:
576
+ dbos_logger.warning(
577
+ f"Workflow inputs for {workflow_uuid} changed since the first call! Use the original inputs."
578
+ )
579
+ # TODO: actually changing the input
549
580
  if workflow_uuid in self._temp_txn_wf_ids:
550
581
  # Clean up the single-transaction tracking sets
551
582
  self._exported_temp_txn_wf_status.discard(workflow_uuid)
552
583
  self._temp_txn_wf_ids.discard(workflow_uuid)
584
+ return
553
585
 
554
586
  def get_workflow_inputs(
555
587
  self, workflow_uuid: str
@@ -582,12 +614,12 @@ class SystemDatabase:
582
614
  if input.start_time:
583
615
  query = query.where(
584
616
  SystemSchema.workflow_status.c.created_at
585
- >= datetime.datetime.fromisoformat(input.start_time).timestamp()
617
+ >= datetime.datetime.fromisoformat(input.start_time).timestamp() * 1000
586
618
  )
587
619
  if input.end_time:
588
620
  query = query.where(
589
621
  SystemSchema.workflow_status.c.created_at
590
- <= datetime.datetime.fromisoformat(input.end_time).timestamp()
622
+ <= datetime.datetime.fromisoformat(input.end_time).timestamp() * 1000
591
623
  )
592
624
  if input.status:
593
625
  query = query.where(SystemSchema.workflow_status.c.status == input.status)