dbos 0.19.0a4__tar.gz → 0.19.0a9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbos might be problematic. Click here for more details.
- {dbos-0.19.0a4 → dbos-0.19.0a9}/PKG-INFO +21 -16
- {dbos-0.19.0a4 → dbos-0.19.0a9}/README.md +20 -15
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_context.py +11 -2
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_core.py +5 -1
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_error.py +11 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_kafka.py +17 -1
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_queue.py +1 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_sys_db.py +69 -37
- dbos-0.19.0a9/dbos/_workflow_commands.py +172 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/cli.py +100 -1
- {dbos-0.19.0a4 → dbos-0.19.0a9}/pyproject.toml +1 -1
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/queuedworkflow.py +1 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_classdecorators.py +1 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_dbos.py +6 -3
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_kafka.py +37 -1
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_queue.py +115 -2
- dbos-0.19.0a9/tests/test_workflow_cmds.py +216 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/LICENSE +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/__init__.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_admin_server.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_app_db.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_classproperty.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_cloudutils/authentication.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_cloudutils/cloudutils.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_cloudutils/databases.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_croniter.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_db_wizard.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_dbos.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_dbos_config.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_fastapi.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_flask.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_kafka_message.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_logger.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/env.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/script.py.mako +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/04ca4f231047_workflow_queues_executor_id.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/50f3227f0b4b_fix_job_queue.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/5c361fc04708_added_system_tables.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/a3b18ad34abe_added_triggers.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/d76646551a6b_job_queue_limiter.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/d76646551a6c_workflow_queue.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_migrations/versions/eab0cc1d9a14_job_queue.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_outcome.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_recovery.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_registrations.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_request.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_roles.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_scheduler.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_schemas/__init__.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_schemas/application_database.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_schemas/system_database.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_serialization.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/README.md +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/__package/__init__.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/__package/main.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/__package/schema.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/alembic.ini +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/dbos-config.yaml.dbos +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/migrations/env.py.dbos +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/migrations/script.py.mako +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/migrations/versions/2024_07_31_180642_init.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_templates/hello/start_postgres_docker.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/_tracer.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/dbos-config.schema.json +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/dbos/py.typed +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/__init__.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/atexit_no_ctor.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/atexit_no_launch.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/classdefs.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/conftest.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/more_classdefs.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_admin_server.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_async.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_concurrency.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_config.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_croniter.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_failures.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_fastapi.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_fastapi_roles.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_flask.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_outcome.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_package.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_scheduler.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_schema_migration.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_singleton.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/tests/test_spans.py +0 -0
- {dbos-0.19.0a4 → dbos-0.19.0a9}/version/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dbos
|
|
3
|
-
Version: 0.19.
|
|
3
|
+
Version: 0.19.0a9
|
|
4
4
|
Summary: Ultra-lightweight durable execution in Python
|
|
5
5
|
Author-Email: "DBOS, Inc." <contact@dbos.dev>
|
|
6
6
|
License: MIT
|
|
@@ -28,14 +28,14 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
|
|
29
29
|
<div align="center">
|
|
30
30
|
|
|
31
|
-
# DBOS Transact:
|
|
31
|
+
# DBOS Transact: A Lightweight Durable Execution Library Built on Postgres
|
|
32
32
|
|
|
33
33
|
#### [Documentation](https://docs.dbos.dev/) • [Examples](https://docs.dbos.dev/examples) • [Github](https://github.com/dbos-inc) • [Discord](https://discord.com/invite/jsmC6pXGgX)
|
|
34
34
|
</div>
|
|
35
35
|
|
|
36
36
|
---
|
|
37
37
|
|
|
38
|
-
DBOS Transact is a Python library
|
|
38
|
+
DBOS Transact is a Python library for **ultra-lightweight durable execution**.
|
|
39
39
|
For example:
|
|
40
40
|
|
|
41
41
|
```python
|
|
@@ -55,18 +55,23 @@ def workflow()
|
|
|
55
55
|
|
|
56
56
|
Durable execution means your program is **resilient to any failure**.
|
|
57
57
|
If it is ever interrupted or crashes, all your workflows will automatically resume from the last completed step.
|
|
58
|
-
|
|
59
|
-
No matter how many times you try to crash it, it always resumes from exactly where it left off!
|
|
58
|
+
Durable execution helps solve many common problems:
|
|
60
59
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
60
|
+
- Orchestrating long-running or business-critical workflows so they seamlessly recover from any failure.
|
|
61
|
+
- Running reliable background jobs with no timeouts.
|
|
62
|
+
- Processing incoming events (e.g. from Kafka) exactly once.
|
|
63
|
+
- Running a fault-tolerant distributed task queue.
|
|
64
|
+
- Running a reliable cron scheduler.
|
|
65
|
+
- Operating an AI agent, or anything that connects to an unreliable or non-deterministic API.
|
|
64
66
|
|
|
65
|
-
|
|
67
|
+
What’s unique about DBOS's implementation of durable execution is that it’s implemented in a **lightweight library** that’s **totally backed by Postgres**.
|
|
68
|
+
To use DBOS, just `pip install` it and annotate your program with DBOS decorators.
|
|
69
|
+
Under the hood, those decorators store your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
|
|
70
|
+
If your program crashes or is interrupted, they automatically recover its workflows from their stored state.
|
|
71
|
+
So all you need to use DBOS is Postgres—there are no other dependencies you have to manage, no separate workflow server.
|
|
66
72
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
- Observability—all workflows automatically emit [OpenTelemetry](https://opentelemetry.io/) traces.
|
|
73
|
+
One big advantage of this approach is that you can add DBOS to **any** Python application—**it’s just a library**.
|
|
74
|
+
You can use DBOS to add reliable background jobs or cron scheduling or queues to your app with no external dependencies except Postgres.
|
|
70
75
|
|
|
71
76
|
## Getting Started
|
|
72
77
|
|
|
@@ -77,7 +82,7 @@ pip install dbos
|
|
|
77
82
|
dbos init --config
|
|
78
83
|
```
|
|
79
84
|
|
|
80
|
-
Then, try it out with this simple program
|
|
85
|
+
Then, try it out with this simple program:
|
|
81
86
|
|
|
82
87
|
```python
|
|
83
88
|
from fastapi import FastAPI
|
|
@@ -107,14 +112,14 @@ def fastapi_endpoint():
|
|
|
107
112
|
dbos_workflow()
|
|
108
113
|
```
|
|
109
114
|
|
|
110
|
-
Save the program into `main.py
|
|
115
|
+
Save the program into `main.py` and start it with `fastapi run`.
|
|
111
116
|
Visit `localhost:8000` in your browser to start the workflow.
|
|
112
117
|
When prompted, press `Control + \` to force quit your application.
|
|
113
118
|
It should crash midway through the workflow, having completed step one but not step two.
|
|
114
119
|
Then, restart your app with `fastapi run`.
|
|
115
120
|
It should resume the workflow from where it left off, completing step two without re-executing step one.
|
|
116
121
|
|
|
117
|
-
To learn how to build more complex workflows, see
|
|
122
|
+
To learn how to build more complex workflows, see the [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
|
|
118
123
|
|
|
119
124
|
## Documentation
|
|
120
125
|
|
|
@@ -125,7 +130,7 @@ To learn how to build more complex workflows, see our [programming guide](https:
|
|
|
125
130
|
|
|
126
131
|
- [**AI-Powered Slackbot**](https://docs.dbos.dev/python/examples/rag-slackbot) — A Slackbot that answers questions about previous Slack conversations, using DBOS to durably orchestrate its RAG pipeline.
|
|
127
132
|
- [**Widget Store**](https://docs.dbos.dev/python/examples/widget-store) — An online storefront that uses DBOS durable workflows to be resilient to any failure.
|
|
128
|
-
- [**
|
|
133
|
+
- [**Scheduled Reminders**](https://docs.dbos.dev/python/examples/scheduled-reminders) — In just three lines of code, schedule an email to send days, weeks, or months in the future.
|
|
129
134
|
|
|
130
135
|
More examples [here](https://docs.dbos.dev/examples)!
|
|
131
136
|
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
|
|
2
2
|
<div align="center">
|
|
3
3
|
|
|
4
|
-
# DBOS Transact:
|
|
4
|
+
# DBOS Transact: A Lightweight Durable Execution Library Built on Postgres
|
|
5
5
|
|
|
6
6
|
#### [Documentation](https://docs.dbos.dev/) • [Examples](https://docs.dbos.dev/examples) • [Github](https://github.com/dbos-inc) • [Discord](https://discord.com/invite/jsmC6pXGgX)
|
|
7
7
|
</div>
|
|
8
8
|
|
|
9
9
|
---
|
|
10
10
|
|
|
11
|
-
DBOS Transact is a Python library
|
|
11
|
+
DBOS Transact is a Python library for **ultra-lightweight durable execution**.
|
|
12
12
|
For example:
|
|
13
13
|
|
|
14
14
|
```python
|
|
@@ -28,18 +28,23 @@ def workflow()
|
|
|
28
28
|
|
|
29
29
|
Durable execution means your program is **resilient to any failure**.
|
|
30
30
|
If it is ever interrupted or crashes, all your workflows will automatically resume from the last completed step.
|
|
31
|
-
|
|
32
|
-
No matter how many times you try to crash it, it always resumes from exactly where it left off!
|
|
31
|
+
Durable execution helps solve many common problems:
|
|
33
32
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
- Orchestrating long-running or business-critical workflows so they seamlessly recover from any failure.
|
|
34
|
+
- Running reliable background jobs with no timeouts.
|
|
35
|
+
- Processing incoming events (e.g. from Kafka) exactly once.
|
|
36
|
+
- Running a fault-tolerant distributed task queue.
|
|
37
|
+
- Running a reliable cron scheduler.
|
|
38
|
+
- Operating an AI agent, or anything that connects to an unreliable or non-deterministic API.
|
|
37
39
|
|
|
38
|
-
|
|
40
|
+
What’s unique about DBOS's implementation of durable execution is that it’s implemented in a **lightweight library** that’s **totally backed by Postgres**.
|
|
41
|
+
To use DBOS, just `pip install` it and annotate your program with DBOS decorators.
|
|
42
|
+
Under the hood, those decorators store your program's execution state (which workflows are currently executing and which steps they've completed) in a Postgres database.
|
|
43
|
+
If your program crashes or is interrupted, they automatically recover its workflows from their stored state.
|
|
44
|
+
So all you need to use DBOS is Postgres—there are no other dependencies you have to manage, no separate workflow server.
|
|
39
45
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
- Observability—all workflows automatically emit [OpenTelemetry](https://opentelemetry.io/) traces.
|
|
46
|
+
One big advantage of this approach is that you can add DBOS to **any** Python application—**it’s just a library**.
|
|
47
|
+
You can use DBOS to add reliable background jobs or cron scheduling or queues to your app with no external dependencies except Postgres.
|
|
43
48
|
|
|
44
49
|
## Getting Started
|
|
45
50
|
|
|
@@ -50,7 +55,7 @@ pip install dbos
|
|
|
50
55
|
dbos init --config
|
|
51
56
|
```
|
|
52
57
|
|
|
53
|
-
Then, try it out with this simple program
|
|
58
|
+
Then, try it out with this simple program:
|
|
54
59
|
|
|
55
60
|
```python
|
|
56
61
|
from fastapi import FastAPI
|
|
@@ -80,14 +85,14 @@ def fastapi_endpoint():
|
|
|
80
85
|
dbos_workflow()
|
|
81
86
|
```
|
|
82
87
|
|
|
83
|
-
Save the program into `main.py
|
|
88
|
+
Save the program into `main.py` and start it with `fastapi run`.
|
|
84
89
|
Visit `localhost:8000` in your browser to start the workflow.
|
|
85
90
|
When prompted, press `Control + \` to force quit your application.
|
|
86
91
|
It should crash midway through the workflow, having completed step one but not step two.
|
|
87
92
|
Then, restart your app with `fastapi run`.
|
|
88
93
|
It should resume the workflow from where it left off, completing step two without re-executing step one.
|
|
89
94
|
|
|
90
|
-
To learn how to build more complex workflows, see
|
|
95
|
+
To learn how to build more complex workflows, see the [programming guide](https://docs.dbos.dev/python/programming-guide) or [examples](https://docs.dbos.dev/examples).
|
|
91
96
|
|
|
92
97
|
## Documentation
|
|
93
98
|
|
|
@@ -98,7 +103,7 @@ To learn how to build more complex workflows, see our [programming guide](https:
|
|
|
98
103
|
|
|
99
104
|
- [**AI-Powered Slackbot**](https://docs.dbos.dev/python/examples/rag-slackbot) — A Slackbot that answers questions about previous Slack conversations, using DBOS to durably orchestrate its RAG pipeline.
|
|
100
105
|
- [**Widget Store**](https://docs.dbos.dev/python/examples/widget-store) — An online storefront that uses DBOS durable workflows to be resilient to any failure.
|
|
101
|
-
- [**
|
|
106
|
+
- [**Scheduled Reminders**](https://docs.dbos.dev/python/examples/scheduled-reminders) — In just three lines of code, schedule an email to send days, weeks, or months in the future.
|
|
102
107
|
|
|
103
108
|
More examples [here](https://docs.dbos.dev/examples)!
|
|
104
109
|
|
|
@@ -57,6 +57,7 @@ class DBOSContext:
|
|
|
57
57
|
self.request: Optional["Request"] = None
|
|
58
58
|
|
|
59
59
|
self.id_assigned_for_next_workflow: str = ""
|
|
60
|
+
self.is_within_set_workflow_id_block: bool = False
|
|
60
61
|
|
|
61
62
|
self.parent_workflow_id: str = ""
|
|
62
63
|
self.parent_workflow_fid: int = -1
|
|
@@ -78,6 +79,7 @@ class DBOSContext:
|
|
|
78
79
|
rv.logger = self.logger
|
|
79
80
|
rv.id_assigned_for_next_workflow = self.id_assigned_for_next_workflow
|
|
80
81
|
self.id_assigned_for_next_workflow = ""
|
|
82
|
+
rv.is_within_set_workflow_id_block = self.is_within_set_workflow_id_block
|
|
81
83
|
rv.parent_workflow_id = self.workflow_id
|
|
82
84
|
rv.parent_workflow_fid = self.function_id
|
|
83
85
|
rv.in_recovery = self.in_recovery
|
|
@@ -95,6 +97,10 @@ class DBOSContext:
|
|
|
95
97
|
if len(self.id_assigned_for_next_workflow) > 0:
|
|
96
98
|
wfid = self.id_assigned_for_next_workflow
|
|
97
99
|
else:
|
|
100
|
+
if self.is_within_set_workflow_id_block:
|
|
101
|
+
self.logger.warning(
|
|
102
|
+
f"Multiple workflows started in the same SetWorkflowID block. Only the first workflow is assigned the specified workflow ID; subsequent workflows will use a generated workflow ID."
|
|
103
|
+
)
|
|
98
104
|
wfid = str(uuid.uuid4())
|
|
99
105
|
return wfid
|
|
100
106
|
|
|
@@ -286,7 +292,7 @@ class DBOSContextSwap:
|
|
|
286
292
|
|
|
287
293
|
class SetWorkflowID:
|
|
288
294
|
"""
|
|
289
|
-
Set the workflow ID to be used for the enclosed workflow invocation.
|
|
295
|
+
Set the workflow ID to be used for the enclosed workflow invocation. Note: Only the first workflow will be started with the specified workflow ID within a `with SetWorkflowID` block.
|
|
290
296
|
|
|
291
297
|
Typical Usage
|
|
292
298
|
```
|
|
@@ -311,7 +317,9 @@ class SetWorkflowID:
|
|
|
311
317
|
if ctx is None:
|
|
312
318
|
self.created_ctx = True
|
|
313
319
|
_set_local_dbos_context(DBOSContext())
|
|
314
|
-
assert_current_dbos_context()
|
|
320
|
+
ctx = assert_current_dbos_context()
|
|
321
|
+
ctx.id_assigned_for_next_workflow = self.wfid
|
|
322
|
+
ctx.is_within_set_workflow_id_block = True
|
|
315
323
|
return self
|
|
316
324
|
|
|
317
325
|
def __exit__(
|
|
@@ -321,6 +329,7 @@ class SetWorkflowID:
|
|
|
321
329
|
traceback: Optional[TracebackType],
|
|
322
330
|
) -> Literal[False]:
|
|
323
331
|
# Code to clean up the basic context if we created it
|
|
332
|
+
assert_current_dbos_context().is_within_set_workflow_id_block = False
|
|
324
333
|
if self.created_ctx:
|
|
325
334
|
_clear_local_dbos_context()
|
|
326
335
|
return False # Did not handle
|
|
@@ -188,6 +188,7 @@ def _init_workflow(
|
|
|
188
188
|
wf_status = dbos._sys_db.update_workflow_status(
|
|
189
189
|
status, False, ctx.in_recovery, max_recovery_attempts=max_recovery_attempts
|
|
190
190
|
)
|
|
191
|
+
# TODO: Modify the inputs if they were changed by `update_workflow_inputs`
|
|
191
192
|
dbos._sys_db.update_workflow_inputs(wfid, _serialization.serialize_args(inputs))
|
|
192
193
|
else:
|
|
193
194
|
# Buffer the inputs for single-transaction workflows, but don't buffer the status
|
|
@@ -422,6 +423,9 @@ def start_workflow(
|
|
|
422
423
|
or wf_status == WorkflowStatusString.ERROR.value
|
|
423
424
|
or wf_status == WorkflowStatusString.SUCCESS.value
|
|
424
425
|
):
|
|
426
|
+
dbos.logger.debug(
|
|
427
|
+
f"Workflow {new_wf_id} already completed with status {wf_status}. Directly returning a workflow handle."
|
|
428
|
+
)
|
|
425
429
|
return WorkflowHandlePolling(new_wf_id, dbos)
|
|
426
430
|
|
|
427
431
|
if fself is not None:
|
|
@@ -494,7 +498,7 @@ def workflow_wrapper(
|
|
|
494
498
|
temp_wf_type=get_temp_workflow_type(func),
|
|
495
499
|
max_recovery_attempts=max_recovery_attempts,
|
|
496
500
|
)
|
|
497
|
-
|
|
501
|
+
# TODO: maybe modify the parameters if they've been changed by `_init_workflow`
|
|
498
502
|
dbos.logger.debug(
|
|
499
503
|
f"Running workflow, id: {ctx.workflow_id}, name: {get_dbos_func_name(func)}"
|
|
500
504
|
)
|
|
@@ -35,6 +35,7 @@ class DBOSErrorCode(Enum):
|
|
|
35
35
|
DeadLetterQueueError = 6
|
|
36
36
|
MaxStepRetriesExceeded = 7
|
|
37
37
|
NotAuthorized = 8
|
|
38
|
+
ConflictingWorkflowError = 9
|
|
38
39
|
|
|
39
40
|
|
|
40
41
|
class DBOSWorkflowConflictIDError(DBOSException):
|
|
@@ -47,6 +48,16 @@ class DBOSWorkflowConflictIDError(DBOSException):
|
|
|
47
48
|
)
|
|
48
49
|
|
|
49
50
|
|
|
51
|
+
class DBOSConflictingWorkflowError(DBOSException):
|
|
52
|
+
"""Exception raised different workflows started with the same workflow ID."""
|
|
53
|
+
|
|
54
|
+
def __init__(self, workflow_id: str, message: Optional[str] = None):
|
|
55
|
+
super().__init__(
|
|
56
|
+
f"Conflicting workflow invocation with the same ID ({workflow_id}): {message}",
|
|
57
|
+
dbos_error_code=DBOSErrorCode.ConflictingWorkflowError.value,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
50
61
|
class DBOSRecoveryError(DBOSException):
|
|
51
62
|
"""Exception raised when a workflow recovery fails."""
|
|
52
63
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import re
|
|
1
2
|
import threading
|
|
2
3
|
from typing import TYPE_CHECKING, Any, Callable, NoReturn
|
|
3
4
|
|
|
@@ -19,6 +20,14 @@ _kafka_queue: Queue
|
|
|
19
20
|
_in_order_kafka_queues: dict[str, Queue] = {}
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
def safe_group_name(method_name: str, topics: list[str]) -> str:
|
|
24
|
+
safe_group_id = "-".join(
|
|
25
|
+
re.sub(r"[^a-zA-Z0-9\-]", "", str(r)) for r in [method_name, *topics]
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
return f"dbos-kafka-group-{safe_group_id}"[:255]
|
|
29
|
+
|
|
30
|
+
|
|
22
31
|
def _kafka_consumer_loop(
|
|
23
32
|
func: _KafkaConsumerWorkflow,
|
|
24
33
|
config: dict[str, Any],
|
|
@@ -34,6 +43,12 @@ def _kafka_consumer_loop(
|
|
|
34
43
|
if "auto.offset.reset" not in config:
|
|
35
44
|
config["auto.offset.reset"] = "earliest"
|
|
36
45
|
|
|
46
|
+
if config.get("group.id") is None:
|
|
47
|
+
config["group.id"] = safe_group_name(func.__qualname__, topics)
|
|
48
|
+
dbos_logger.warning(
|
|
49
|
+
f"Consumer group ID not found. Using generated group.id {config['group.id']}"
|
|
50
|
+
)
|
|
51
|
+
|
|
37
52
|
consumer = Consumer(config)
|
|
38
53
|
try:
|
|
39
54
|
consumer.subscribe(topics)
|
|
@@ -71,8 +86,9 @@ def _kafka_consumer_loop(
|
|
|
71
86
|
topic=cmsg.topic(),
|
|
72
87
|
value=cmsg.value(),
|
|
73
88
|
)
|
|
89
|
+
groupID = config.get("group.id")
|
|
74
90
|
with SetWorkflowID(
|
|
75
|
-
f"kafka-unique-id-{msg.topic}-{msg.partition}-{msg.offset}"
|
|
91
|
+
f"kafka-unique-id-{msg.topic}-{msg.partition}-{groupID}-{msg.offset}"
|
|
76
92
|
):
|
|
77
93
|
if in_order:
|
|
78
94
|
assert msg.topic is not None
|
|
@@ -28,6 +28,7 @@ from sqlalchemy.exc import DBAPIError
|
|
|
28
28
|
from . import _serialization
|
|
29
29
|
from ._dbos_config import ConfigFile
|
|
30
30
|
from ._error import (
|
|
31
|
+
DBOSConflictingWorkflowError,
|
|
31
32
|
DBOSDeadLetterQueueError,
|
|
32
33
|
DBOSException,
|
|
33
34
|
DBOSNonExistentWorkflowError,
|
|
@@ -288,8 +289,14 @@ class SystemDatabase:
|
|
|
288
289
|
),
|
|
289
290
|
)
|
|
290
291
|
else:
|
|
291
|
-
|
|
292
|
-
|
|
292
|
+
# A blank update so that we can return the existing status
|
|
293
|
+
cmd = cmd.on_conflict_do_update(
|
|
294
|
+
index_elements=["workflow_uuid"],
|
|
295
|
+
set_=dict(
|
|
296
|
+
recovery_attempts=SystemSchema.workflow_status.c.recovery_attempts
|
|
297
|
+
),
|
|
298
|
+
)
|
|
299
|
+
cmd = cmd.returning(SystemSchema.workflow_status.c.recovery_attempts, SystemSchema.workflow_status.c.status, SystemSchema.workflow_status.c.name, SystemSchema.workflow_status.c.class_name, SystemSchema.workflow_status.c.config_name, SystemSchema.workflow_status.c.queue_name) # type: ignore
|
|
293
300
|
|
|
294
301
|
if conn is not None:
|
|
295
302
|
results = conn.execute(cmd)
|
|
@@ -297,37 +304,53 @@ class SystemDatabase:
|
|
|
297
304
|
with self.engine.begin() as c:
|
|
298
305
|
results = c.execute(cmd)
|
|
299
306
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
307
|
+
row = results.fetchone()
|
|
308
|
+
if row is not None:
|
|
309
|
+
# Check the started workflow matches the expected name, class_name, config_name, and queue_name
|
|
310
|
+
# A mismatch indicates a workflow starting with the same UUID but different functions, which would throw an exception.
|
|
311
|
+
recovery_attempts: int = row[0]
|
|
312
|
+
wf_status = row[1]
|
|
313
|
+
err_msg: Optional[str] = None
|
|
314
|
+
if row[2] != status["name"]:
|
|
315
|
+
err_msg = f"Workflow already exists with a different function name: {row[2]}, but the provided function name is: {status['name']}"
|
|
316
|
+
elif row[3] != status["class_name"]:
|
|
317
|
+
err_msg = f"Workflow already exists with a different class name: {row[3]}, but the provided class name is: {status['class_name']}"
|
|
318
|
+
elif row[4] != status["config_name"]:
|
|
319
|
+
err_msg = f"Workflow already exists with a different config name: {row[4]}, but the provided config name is: {status['config_name']}"
|
|
320
|
+
elif row[5] != status["queue_name"]:
|
|
321
|
+
# This is a warning because a different queue name is not necessarily an error.
|
|
322
|
+
dbos_logger.warning(
|
|
323
|
+
f"Workflow already exists in queue: {row[5]}, but the provided queue name is: {status['queue_name']}. The queue is not updated."
|
|
324
|
+
)
|
|
325
|
+
if err_msg is not None:
|
|
326
|
+
raise DBOSConflictingWorkflowError(status["workflow_uuid"], err_msg)
|
|
327
|
+
|
|
328
|
+
if in_recovery and recovery_attempts > max_recovery_attempts:
|
|
329
|
+
with self.engine.begin() as c:
|
|
330
|
+
c.execute(
|
|
331
|
+
sa.delete(SystemSchema.workflow_queue).where(
|
|
332
|
+
SystemSchema.workflow_queue.c.workflow_uuid
|
|
333
|
+
== status["workflow_uuid"]
|
|
312
334
|
)
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
335
|
+
)
|
|
336
|
+
c.execute(
|
|
337
|
+
sa.update(SystemSchema.workflow_status)
|
|
338
|
+
.where(
|
|
339
|
+
SystemSchema.workflow_status.c.workflow_uuid
|
|
340
|
+
== status["workflow_uuid"]
|
|
341
|
+
)
|
|
342
|
+
.where(
|
|
343
|
+
SystemSchema.workflow_status.c.status
|
|
344
|
+
== WorkflowStatusString.PENDING.value
|
|
345
|
+
)
|
|
346
|
+
.values(
|
|
347
|
+
status=WorkflowStatusString.RETRIES_EXCEEDED.value,
|
|
348
|
+
queue_name=None,
|
|
327
349
|
)
|
|
328
|
-
raise DBOSDeadLetterQueueError(
|
|
329
|
-
status["workflow_uuid"], max_recovery_attempts
|
|
330
350
|
)
|
|
351
|
+
raise DBOSDeadLetterQueueError(
|
|
352
|
+
status["workflow_uuid"], max_recovery_attempts
|
|
353
|
+
)
|
|
331
354
|
|
|
332
355
|
# Record we have exported status for this single-transaction workflow
|
|
333
356
|
if status["workflow_uuid"] in self._temp_txn_wf_ids:
|
|
@@ -356,7 +379,7 @@ class SystemDatabase:
|
|
|
356
379
|
stmt = (
|
|
357
380
|
sa.update(SystemSchema.workflow_status)
|
|
358
381
|
.where(
|
|
359
|
-
SystemSchema.
|
|
382
|
+
SystemSchema.workflow_status.c.workflow_uuid == workflow_uuid
|
|
360
383
|
)
|
|
361
384
|
.values(recovery_attempts=reset_recovery_attempts)
|
|
362
385
|
)
|
|
@@ -538,18 +561,27 @@ class SystemDatabase:
|
|
|
538
561
|
workflow_uuid=workflow_uuid,
|
|
539
562
|
inputs=inputs,
|
|
540
563
|
)
|
|
541
|
-
.
|
|
564
|
+
.on_conflict_do_update(
|
|
565
|
+
index_elements=["workflow_uuid"],
|
|
566
|
+
set_=dict(workflow_uuid=SystemSchema.workflow_inputs.c.workflow_uuid),
|
|
567
|
+
)
|
|
568
|
+
.returning(SystemSchema.workflow_inputs.c.inputs)
|
|
542
569
|
)
|
|
543
570
|
if conn is not None:
|
|
544
|
-
conn.execute(cmd)
|
|
571
|
+
row = conn.execute(cmd).fetchone()
|
|
545
572
|
else:
|
|
546
573
|
with self.engine.begin() as c:
|
|
547
|
-
c.execute(cmd)
|
|
548
|
-
|
|
574
|
+
row = c.execute(cmd).fetchone()
|
|
575
|
+
if row is not None and row[0] != inputs:
|
|
576
|
+
dbos_logger.warning(
|
|
577
|
+
f"Workflow inputs for {workflow_uuid} changed since the first call! Use the original inputs."
|
|
578
|
+
)
|
|
579
|
+
# TODO: actually changing the input
|
|
549
580
|
if workflow_uuid in self._temp_txn_wf_ids:
|
|
550
581
|
# Clean up the single-transaction tracking sets
|
|
551
582
|
self._exported_temp_txn_wf_status.discard(workflow_uuid)
|
|
552
583
|
self._temp_txn_wf_ids.discard(workflow_uuid)
|
|
584
|
+
return
|
|
553
585
|
|
|
554
586
|
def get_workflow_inputs(
|
|
555
587
|
self, workflow_uuid: str
|
|
@@ -582,12 +614,12 @@ class SystemDatabase:
|
|
|
582
614
|
if input.start_time:
|
|
583
615
|
query = query.where(
|
|
584
616
|
SystemSchema.workflow_status.c.created_at
|
|
585
|
-
>= datetime.datetime.fromisoformat(input.start_time).timestamp()
|
|
617
|
+
>= datetime.datetime.fromisoformat(input.start_time).timestamp() * 1000
|
|
586
618
|
)
|
|
587
619
|
if input.end_time:
|
|
588
620
|
query = query.where(
|
|
589
621
|
SystemSchema.workflow_status.c.created_at
|
|
590
|
-
<= datetime.datetime.fromisoformat(input.end_time).timestamp()
|
|
622
|
+
<= datetime.datetime.fromisoformat(input.end_time).timestamp() * 1000
|
|
591
623
|
)
|
|
592
624
|
if input.status:
|
|
593
625
|
query = query.where(SystemSchema.workflow_status.c.status == input.status)
|