dbos 0.22.0a10__py3-none-any.whl → 0.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbos might be problematic. Click here for more details.
- dbos/__main__.py +26 -0
- dbos/_app_db.py +29 -24
- dbos/_cloudutils/cloudutils.py +4 -2
- dbos/_cloudutils/databases.py +4 -0
- dbos/_conductor/conductor.py +213 -0
- dbos/_conductor/protocol.py +197 -0
- dbos/_context.py +3 -1
- dbos/_core.py +73 -26
- dbos/_croniter.py +2 -2
- dbos/_dbos.py +74 -16
- dbos/_dbos_config.py +45 -11
- dbos/_debug.py +45 -0
- dbos/_error.py +11 -0
- dbos/_logger.py +5 -6
- dbos/_migrations/versions/5c361fc04708_added_system_tables.py +1 -1
- dbos/_queue.py +5 -1
- dbos/_recovery.py +23 -24
- dbos/_schemas/system_database.py +1 -1
- dbos/_sys_db.py +212 -187
- dbos/_templates/dbos-db-starter/migrations/versions/2024_07_31_180642_init.py +1 -1
- dbos/_tracer.py +4 -4
- dbos/_utils.py +6 -0
- dbos/_workflow_commands.py +76 -111
- dbos/cli/cli.py +63 -21
- {dbos-0.22.0a10.dist-info → dbos-0.23.0.dist-info}/METADATA +7 -3
- {dbos-0.22.0a10.dist-info → dbos-0.23.0.dist-info}/RECORD +29 -24
- {dbos-0.22.0a10.dist-info → dbos-0.23.0.dist-info}/WHEEL +0 -0
- {dbos-0.22.0a10.dist-info → dbos-0.23.0.dist-info}/entry_points.txt +0 -0
- {dbos-0.22.0a10.dist-info → dbos-0.23.0.dist-info}/licenses/LICENSE +0 -0
dbos/_core.py
CHANGED
|
@@ -22,6 +22,7 @@ from typing import (
|
|
|
22
22
|
)
|
|
23
23
|
|
|
24
24
|
from dbos._outcome import Immediate, NoResult, Outcome, Pending
|
|
25
|
+
from dbos._utils import GlobalParams
|
|
25
26
|
|
|
26
27
|
from ._app_db import ApplicationDatabase, TransactionResultInternal
|
|
27
28
|
|
|
@@ -51,6 +52,7 @@ from ._error import (
|
|
|
51
52
|
DBOSMaxStepRetriesExceeded,
|
|
52
53
|
DBOSNonExistentWorkflowError,
|
|
53
54
|
DBOSRecoveryError,
|
|
55
|
+
DBOSWorkflowCancelledError,
|
|
54
56
|
DBOSWorkflowConflictIDError,
|
|
55
57
|
DBOSWorkflowFunctionNotFoundError,
|
|
56
58
|
)
|
|
@@ -163,7 +165,7 @@ def _init_workflow(
|
|
|
163
165
|
"output": None,
|
|
164
166
|
"error": None,
|
|
165
167
|
"app_id": ctx.app_id,
|
|
166
|
-
"app_version":
|
|
168
|
+
"app_version": GlobalParams.app_version,
|
|
167
169
|
"executor_id": ctx.executor_id,
|
|
168
170
|
"request": (
|
|
169
171
|
_serialization.serialize(ctx.request) if ctx.request is not None else None
|
|
@@ -175,6 +177,8 @@ def _init_workflow(
|
|
|
175
177
|
),
|
|
176
178
|
"assumed_role": ctx.assumed_role,
|
|
177
179
|
"queue_name": queue,
|
|
180
|
+
"created_at": None,
|
|
181
|
+
"updated_at": None,
|
|
178
182
|
}
|
|
179
183
|
|
|
180
184
|
# If we have a class name, the first arg is the instance and do not serialize
|
|
@@ -182,21 +186,31 @@ def _init_workflow(
|
|
|
182
186
|
inputs = {"args": inputs["args"][1:], "kwargs": inputs["kwargs"]}
|
|
183
187
|
|
|
184
188
|
wf_status = status["status"]
|
|
185
|
-
if
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
wf_status =
|
|
190
|
-
status, max_recovery_attempts=max_recovery_attempts
|
|
191
|
-
)
|
|
192
|
-
# TODO: Modify the inputs if they were changed by `update_workflow_inputs`
|
|
193
|
-
dbos._sys_db.update_workflow_inputs(wfid, _serialization.serialize_args(inputs))
|
|
189
|
+
if dbos.debug_mode:
|
|
190
|
+
get_status_result = dbos._sys_db.get_workflow_status(wfid)
|
|
191
|
+
if get_status_result is None:
|
|
192
|
+
raise DBOSNonExistentWorkflowError(wfid)
|
|
193
|
+
wf_status = get_status_result["status"]
|
|
194
194
|
else:
|
|
195
|
-
|
|
196
|
-
|
|
195
|
+
if temp_wf_type != "transaction" or queue is not None:
|
|
196
|
+
# Synchronously record the status and inputs for workflows and single-step workflows
|
|
197
|
+
# We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
|
|
198
|
+
# TODO: Make this transactional (and with the queue step below)
|
|
199
|
+
wf_status = dbos._sys_db.insert_workflow_status(
|
|
200
|
+
status, max_recovery_attempts=max_recovery_attempts
|
|
201
|
+
)
|
|
202
|
+
# TODO: Modify the inputs if they were changed by `update_workflow_inputs`
|
|
203
|
+
dbos._sys_db.update_workflow_inputs(
|
|
204
|
+
wfid, _serialization.serialize_args(inputs)
|
|
205
|
+
)
|
|
206
|
+
else:
|
|
207
|
+
# Buffer the inputs for single-transaction workflows, but don't buffer the status
|
|
208
|
+
dbos._sys_db.buffer_workflow_inputs(
|
|
209
|
+
wfid, _serialization.serialize_args(inputs)
|
|
210
|
+
)
|
|
197
211
|
|
|
198
|
-
|
|
199
|
-
|
|
212
|
+
if queue is not None and wf_status == WorkflowStatusString.ENQUEUED.value:
|
|
213
|
+
dbos._sys_db.enqueue(wfid, queue)
|
|
200
214
|
|
|
201
215
|
status["status"] = wf_status
|
|
202
216
|
return status
|
|
@@ -211,10 +225,11 @@ def _get_wf_invoke_func(
|
|
|
211
225
|
output = func()
|
|
212
226
|
status["status"] = "SUCCESS"
|
|
213
227
|
status["output"] = _serialization.serialize(output)
|
|
214
|
-
if
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
228
|
+
if not dbos.debug_mode:
|
|
229
|
+
if status["queue_name"] is not None:
|
|
230
|
+
queue = dbos._registry.queue_info_map[status["queue_name"]]
|
|
231
|
+
dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
|
|
232
|
+
dbos._sys_db.buffer_workflow_status(status)
|
|
218
233
|
return output
|
|
219
234
|
except DBOSWorkflowConflictIDError:
|
|
220
235
|
# Retrieve the workflow handle and wait for the result.
|
|
@@ -224,13 +239,16 @@ def _get_wf_invoke_func(
|
|
|
224
239
|
)
|
|
225
240
|
output = wf_handle.get_result()
|
|
226
241
|
return output
|
|
242
|
+
except DBOSWorkflowCancelledError as error:
|
|
243
|
+
raise
|
|
227
244
|
except Exception as error:
|
|
228
245
|
status["status"] = "ERROR"
|
|
229
246
|
status["error"] = _serialization.serialize_exception(error)
|
|
230
|
-
if
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
247
|
+
if not dbos.debug_mode:
|
|
248
|
+
if status["queue_name"] is not None:
|
|
249
|
+
queue = dbos._registry.queue_info_map[status["queue_name"]]
|
|
250
|
+
dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
|
|
251
|
+
dbos._sys_db.update_workflow_status(status)
|
|
234
252
|
raise
|
|
235
253
|
|
|
236
254
|
return persist
|
|
@@ -416,10 +434,12 @@ def start_workflow(
|
|
|
416
434
|
|
|
417
435
|
wf_status = status["status"]
|
|
418
436
|
|
|
419
|
-
if (
|
|
420
|
-
not
|
|
421
|
-
|
|
422
|
-
|
|
437
|
+
if not execute_workflow or (
|
|
438
|
+
not dbos.debug_mode
|
|
439
|
+
and (
|
|
440
|
+
wf_status == WorkflowStatusString.ERROR.value
|
|
441
|
+
or wf_status == WorkflowStatusString.SUCCESS.value
|
|
442
|
+
)
|
|
423
443
|
):
|
|
424
444
|
dbos.logger.debug(
|
|
425
445
|
f"Workflow {new_wf_id} already completed with status {wf_status}. Directly returning a workflow handle."
|
|
@@ -539,6 +559,13 @@ def decorate_transaction(
|
|
|
539
559
|
raise DBOSException(
|
|
540
560
|
f"Function {func.__name__} invoked before DBOS initialized"
|
|
541
561
|
)
|
|
562
|
+
|
|
563
|
+
ctx = assert_current_dbos_context()
|
|
564
|
+
if dbosreg.is_workflow_cancelled(ctx.workflow_id):
|
|
565
|
+
raise DBOSWorkflowCancelledError(
|
|
566
|
+
f"Workflow {ctx.workflow_id} is cancelled. Aborting transaction {func.__name__}."
|
|
567
|
+
)
|
|
568
|
+
|
|
542
569
|
dbos = dbosreg.dbos
|
|
543
570
|
with dbos._app_db.sessionmaker() as session:
|
|
544
571
|
attributes: TracedAttributes = {
|
|
@@ -560,6 +587,12 @@ def decorate_transaction(
|
|
|
560
587
|
backoff_factor = 1.5
|
|
561
588
|
max_retry_wait_seconds = 2.0
|
|
562
589
|
while True:
|
|
590
|
+
|
|
591
|
+
if dbosreg.is_workflow_cancelled(ctx.workflow_id):
|
|
592
|
+
raise DBOSWorkflowCancelledError(
|
|
593
|
+
f"Workflow {ctx.workflow_id} is cancelled. Aborting transaction {func.__name__}."
|
|
594
|
+
)
|
|
595
|
+
|
|
563
596
|
has_recorded_error = False
|
|
564
597
|
txn_error: Optional[Exception] = None
|
|
565
598
|
try:
|
|
@@ -578,6 +611,10 @@ def decorate_transaction(
|
|
|
578
611
|
ctx.function_id,
|
|
579
612
|
)
|
|
580
613
|
)
|
|
614
|
+
if dbos.debug_mode and recorded_output is None:
|
|
615
|
+
raise DBOSException(
|
|
616
|
+
"Transaction output not found in debug mode"
|
|
617
|
+
)
|
|
581
618
|
if recorded_output:
|
|
582
619
|
dbos.logger.debug(
|
|
583
620
|
f"Replaying transaction, id: {ctx.function_id}, name: {attributes['name']}"
|
|
@@ -710,6 +747,13 @@ def decorate_step(
|
|
|
710
747
|
"operationType": OperationType.STEP.value,
|
|
711
748
|
}
|
|
712
749
|
|
|
750
|
+
# Check if the workflow is cancelled
|
|
751
|
+
ctx = assert_current_dbos_context()
|
|
752
|
+
if dbosreg.is_workflow_cancelled(ctx.workflow_id):
|
|
753
|
+
raise DBOSWorkflowCancelledError(
|
|
754
|
+
f"Workflow {ctx.workflow_id} is cancelled. Aborting step {func.__name__}."
|
|
755
|
+
)
|
|
756
|
+
|
|
713
757
|
attempts = max_attempts if retries_allowed else 1
|
|
714
758
|
max_retry_interval_seconds: float = 3600 # 1 Hour
|
|
715
759
|
|
|
@@ -754,6 +798,8 @@ def decorate_step(
|
|
|
754
798
|
recorded_output = dbos._sys_db.check_operation_execution(
|
|
755
799
|
ctx.workflow_id, ctx.function_id
|
|
756
800
|
)
|
|
801
|
+
if dbos.debug_mode and recorded_output is None:
|
|
802
|
+
raise DBOSException("Step output not found in debug mode")
|
|
757
803
|
if recorded_output:
|
|
758
804
|
dbos.logger.debug(
|
|
759
805
|
f"Replaying step, id: {ctx.function_id}, name: {attributes['name']}"
|
|
@@ -800,6 +846,7 @@ def decorate_step(
|
|
|
800
846
|
ctx = get_local_dbos_context()
|
|
801
847
|
if ctx and ctx.is_step():
|
|
802
848
|
# Call the original function directly
|
|
849
|
+
|
|
803
850
|
return func(*args, **kwargs)
|
|
804
851
|
if ctx and ctx.is_within_workflow():
|
|
805
852
|
assert ctx.is_workflow(), "Steps must be called from within workflows"
|
dbos/_croniter.py
CHANGED
|
@@ -5,14 +5,14 @@ Copyright (C) 2010-2012 Matsumoto Taichi.
|
|
|
5
5
|
|
|
6
6
|
Permission is hereby granted, free of charge, to any person obtaining a copy of this
|
|
7
7
|
software and associated documentation files (the "Software"), to deal in the Software
|
|
8
|
-
without restriction, including without limitation the rights to use, copy, modify,
|
|
8
|
+
without restriction, including without limitation the rights to use, copy, modify,
|
|
9
9
|
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
|
|
10
10
|
persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
11
11
|
|
|
12
12
|
The above copyright notice and this permission notice shall be included in all
|
|
13
13
|
copies or substantial portions of the Software.
|
|
14
14
|
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
16
16
|
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
|
17
17
|
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
|
|
18
18
|
FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
dbos/_dbos.py
CHANGED
|
@@ -9,6 +9,7 @@ import os
|
|
|
9
9
|
import sys
|
|
10
10
|
import threading
|
|
11
11
|
import traceback
|
|
12
|
+
import uuid
|
|
12
13
|
from concurrent.futures import ThreadPoolExecutor
|
|
13
14
|
from dataclasses import dataclass
|
|
14
15
|
from logging import Logger
|
|
@@ -32,6 +33,9 @@ from typing import (
|
|
|
32
33
|
|
|
33
34
|
from opentelemetry.trace import Span
|
|
34
35
|
|
|
36
|
+
from dbos._conductor.conductor import ConductorWebsocket
|
|
37
|
+
from dbos._utils import GlobalParams
|
|
38
|
+
|
|
35
39
|
from ._classproperty import classproperty
|
|
36
40
|
from ._core import (
|
|
37
41
|
TEMP_SEND_WF_NAME,
|
|
@@ -155,6 +159,7 @@ class DBOSRegistry:
|
|
|
155
159
|
self.pollers: list[RegisteredJob] = []
|
|
156
160
|
self.dbos: Optional[DBOS] = None
|
|
157
161
|
self.config: Optional[ConfigFile] = None
|
|
162
|
+
self.workflow_cancelled_map: dict[str, bool] = {}
|
|
158
163
|
|
|
159
164
|
def register_wf_function(self, name: str, wrapped_func: F, functype: str) -> None:
|
|
160
165
|
if name in self.function_type_map:
|
|
@@ -197,6 +202,15 @@ class DBOSRegistry:
|
|
|
197
202
|
else:
|
|
198
203
|
self.instance_info_map[fn] = inst
|
|
199
204
|
|
|
205
|
+
def cancel_workflow(self, workflow_id: str) -> None:
|
|
206
|
+
self.workflow_cancelled_map[workflow_id] = True
|
|
207
|
+
|
|
208
|
+
def is_workflow_cancelled(self, workflow_id: str) -> bool:
|
|
209
|
+
return self.workflow_cancelled_map.get(workflow_id, False)
|
|
210
|
+
|
|
211
|
+
def clear_workflow_cancelled(self, workflow_id: str) -> None:
|
|
212
|
+
self.workflow_cancelled_map.pop(workflow_id, None)
|
|
213
|
+
|
|
200
214
|
def compute_app_version(self) -> str:
|
|
201
215
|
"""
|
|
202
216
|
An application's version is computed from a hash of the source of its workflows.
|
|
@@ -246,6 +260,8 @@ class DBOS:
|
|
|
246
260
|
config: Optional[ConfigFile] = None,
|
|
247
261
|
fastapi: Optional["FastAPI"] = None,
|
|
248
262
|
flask: Optional["Flask"] = None,
|
|
263
|
+
conductor_url: Optional[str] = None,
|
|
264
|
+
conductor_key: Optional[str] = None,
|
|
249
265
|
) -> DBOS:
|
|
250
266
|
global _dbos_global_instance
|
|
251
267
|
global _dbos_global_registry
|
|
@@ -261,7 +277,7 @@ class DBOS:
|
|
|
261
277
|
config = _dbos_global_registry.config
|
|
262
278
|
|
|
263
279
|
_dbos_global_instance = super().__new__(cls)
|
|
264
|
-
_dbos_global_instance.__init__(fastapi=fastapi, config=config, flask=flask) # type: ignore
|
|
280
|
+
_dbos_global_instance.__init__(fastapi=fastapi, config=config, flask=flask, conductor_url=conductor_url, conductor_key=conductor_key) # type: ignore
|
|
265
281
|
else:
|
|
266
282
|
if (config is not None and _dbos_global_instance.config is not config) or (
|
|
267
283
|
_dbos_global_instance.fastapi is not fastapi
|
|
@@ -280,6 +296,8 @@ class DBOS:
|
|
|
280
296
|
if destroy_registry:
|
|
281
297
|
global _dbos_global_registry
|
|
282
298
|
_dbos_global_registry = None
|
|
299
|
+
GlobalParams.app_version = os.environ.get("DBOS__APPVERSION", "")
|
|
300
|
+
GlobalParams.executor_id = os.environ.get("DBOS__VMID", "local")
|
|
283
301
|
|
|
284
302
|
def __init__(
|
|
285
303
|
self,
|
|
@@ -287,6 +305,8 @@ class DBOS:
|
|
|
287
305
|
config: Optional[ConfigFile] = None,
|
|
288
306
|
fastapi: Optional["FastAPI"] = None,
|
|
289
307
|
flask: Optional["Flask"] = None,
|
|
308
|
+
conductor_url: Optional[str] = None,
|
|
309
|
+
conductor_key: Optional[str] = None,
|
|
290
310
|
) -> None:
|
|
291
311
|
if hasattr(self, "_initialized") and self._initialized:
|
|
292
312
|
return
|
|
@@ -299,6 +319,7 @@ class DBOS:
|
|
|
299
319
|
dbos_logger.info("Initializing DBOS")
|
|
300
320
|
self.config: ConfigFile = config
|
|
301
321
|
self._launched: bool = False
|
|
322
|
+
self._debug_mode: bool = False
|
|
302
323
|
self._sys_db_field: Optional[SystemDatabase] = None
|
|
303
324
|
self._app_db_field: Optional[ApplicationDatabase] = None
|
|
304
325
|
self._registry: DBOSRegistry = _get_or_create_dbos_registry()
|
|
@@ -309,8 +330,9 @@ class DBOS:
|
|
|
309
330
|
self.flask: Optional["Flask"] = flask
|
|
310
331
|
self._executor_field: Optional[ThreadPoolExecutor] = None
|
|
311
332
|
self._background_threads: List[threading.Thread] = []
|
|
312
|
-
self.
|
|
313
|
-
self.
|
|
333
|
+
self.conductor_url: Optional[str] = conductor_url
|
|
334
|
+
self.conductor_key: Optional[str] = conductor_key
|
|
335
|
+
self.conductor_websocket: Optional[ConductorWebsocket] = None
|
|
314
336
|
|
|
315
337
|
# If using FastAPI, set up middleware and lifecycle events
|
|
316
338
|
if self.fastapi is not None:
|
|
@@ -368,39 +390,50 @@ class DBOS:
|
|
|
368
390
|
rv: AdminServer = self._admin_server_field
|
|
369
391
|
return rv
|
|
370
392
|
|
|
393
|
+
@property
|
|
394
|
+
def debug_mode(self) -> bool:
|
|
395
|
+
return self._debug_mode
|
|
396
|
+
|
|
371
397
|
@classmethod
|
|
372
|
-
def launch(cls) -> None:
|
|
398
|
+
def launch(cls, *, debug_mode: bool = False) -> None:
|
|
373
399
|
if _dbos_global_instance is not None:
|
|
374
|
-
_dbos_global_instance._launch()
|
|
400
|
+
_dbos_global_instance._launch(debug_mode=debug_mode)
|
|
375
401
|
|
|
376
|
-
def _launch(self) -> None:
|
|
402
|
+
def _launch(self, *, debug_mode: bool = False) -> None:
|
|
377
403
|
try:
|
|
378
404
|
if self._launched:
|
|
379
405
|
dbos_logger.warning(f"DBOS was already launched")
|
|
380
406
|
return
|
|
381
407
|
self._launched = True
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
408
|
+
self._debug_mode = debug_mode
|
|
409
|
+
if GlobalParams.app_version == "":
|
|
410
|
+
GlobalParams.app_version = self._registry.compute_app_version()
|
|
411
|
+
if self.conductor_key is not None:
|
|
412
|
+
GlobalParams.executor_id = str(uuid.uuid4())
|
|
413
|
+
dbos_logger.info(f"Executor ID: {GlobalParams.executor_id}")
|
|
414
|
+
dbos_logger.info(f"Application version: {GlobalParams.app_version}")
|
|
386
415
|
self._executor_field = ThreadPoolExecutor(max_workers=64)
|
|
387
|
-
self._sys_db_field = SystemDatabase(self.config)
|
|
388
|
-
self._app_db_field = ApplicationDatabase(self.config)
|
|
416
|
+
self._sys_db_field = SystemDatabase(self.config, debug_mode=debug_mode)
|
|
417
|
+
self._app_db_field = ApplicationDatabase(self.config, debug_mode=debug_mode)
|
|
418
|
+
|
|
419
|
+
if debug_mode:
|
|
420
|
+
return
|
|
421
|
+
|
|
389
422
|
admin_port = self.config["runtimeConfig"].get("admin_port")
|
|
390
423
|
if admin_port is None:
|
|
391
424
|
admin_port = 3001
|
|
392
425
|
self._admin_server_field = AdminServer(dbos=self, port=admin_port)
|
|
393
426
|
|
|
394
427
|
workflow_ids = self._sys_db.get_pending_workflows(
|
|
395
|
-
|
|
428
|
+
GlobalParams.executor_id, GlobalParams.app_version
|
|
396
429
|
)
|
|
397
430
|
if (len(workflow_ids)) > 0:
|
|
398
431
|
self.logger.info(
|
|
399
|
-
f"Recovering {len(workflow_ids)} workflows from application version {
|
|
432
|
+
f"Recovering {len(workflow_ids)} workflows from application version {GlobalParams.app_version}"
|
|
400
433
|
)
|
|
401
434
|
else:
|
|
402
435
|
self.logger.info(
|
|
403
|
-
f"No workflows to recover from application version {
|
|
436
|
+
f"No workflows to recover from application version {GlobalParams.app_version}"
|
|
404
437
|
)
|
|
405
438
|
|
|
406
439
|
self._executor.submit(startup_recovery_thread, self, workflow_ids)
|
|
@@ -430,6 +463,22 @@ class DBOS:
|
|
|
430
463
|
bg_queue_thread.start()
|
|
431
464
|
self._background_threads.append(bg_queue_thread)
|
|
432
465
|
|
|
466
|
+
# Start the conductor thread if requested
|
|
467
|
+
if self.conductor_key is not None:
|
|
468
|
+
if self.conductor_url is None:
|
|
469
|
+
dbos_domain = os.environ.get("DBOS_DOMAIN", "cloud.dbos.dev")
|
|
470
|
+
self.conductor_url = f"wss://{dbos_domain}/conductor/v1alpha1"
|
|
471
|
+
evt = threading.Event()
|
|
472
|
+
self.stop_events.append(evt)
|
|
473
|
+
self.conductor_websocket = ConductorWebsocket(
|
|
474
|
+
self,
|
|
475
|
+
conductor_url=self.conductor_url,
|
|
476
|
+
conductor_key=self.conductor_key,
|
|
477
|
+
evt=evt,
|
|
478
|
+
)
|
|
479
|
+
self.conductor_websocket.start()
|
|
480
|
+
self._background_threads.append(self.conductor_websocket)
|
|
481
|
+
|
|
433
482
|
# Grab any pollers that were deferred and start them
|
|
434
483
|
for evt, func, args, kwargs in self._registry.pollers:
|
|
435
484
|
self.stop_events.append(evt)
|
|
@@ -446,7 +495,7 @@ class DBOS:
|
|
|
446
495
|
# to enable their export in DBOS Cloud
|
|
447
496
|
for handler in dbos_logger.handlers:
|
|
448
497
|
handler.flush()
|
|
449
|
-
add_otlp_to_all_loggers(
|
|
498
|
+
add_otlp_to_all_loggers()
|
|
450
499
|
except Exception:
|
|
451
500
|
dbos_logger.error(f"DBOS failed to launch: {traceback.format_exc()}")
|
|
452
501
|
raise
|
|
@@ -480,6 +529,11 @@ class DBOS:
|
|
|
480
529
|
if self._admin_server_field is not None:
|
|
481
530
|
self._admin_server_field.stop()
|
|
482
531
|
self._admin_server_field = None
|
|
532
|
+
if (
|
|
533
|
+
self.conductor_websocket is not None
|
|
534
|
+
and self.conductor_websocket.websocket is not None
|
|
535
|
+
):
|
|
536
|
+
self.conductor_websocket.websocket.close()
|
|
483
537
|
# CB - This needs work, some things ought to stop before DBs are tossed out,
|
|
484
538
|
# on the other hand it hangs to move it
|
|
485
539
|
if self._executor_field is not None:
|
|
@@ -843,12 +897,16 @@ class DBOS:
|
|
|
843
897
|
@classmethod
|
|
844
898
|
def cancel_workflow(cls, workflow_id: str) -> None:
|
|
845
899
|
"""Cancel a workflow by ID."""
|
|
900
|
+
dbos_logger.info(f"Cancelling workflow: {workflow_id}")
|
|
846
901
|
_get_dbos_instance()._sys_db.cancel_workflow(workflow_id)
|
|
902
|
+
_get_or_create_dbos_registry().cancel_workflow(workflow_id)
|
|
847
903
|
|
|
848
904
|
@classmethod
|
|
849
905
|
def resume_workflow(cls, workflow_id: str) -> WorkflowHandle[Any]:
|
|
850
906
|
"""Resume a workflow by ID."""
|
|
907
|
+
dbos_logger.info(f"Resuming workflow: {workflow_id}")
|
|
851
908
|
_get_dbos_instance()._sys_db.resume_workflow(workflow_id)
|
|
909
|
+
_get_or_create_dbos_registry().clear_workflow_cancelled(workflow_id)
|
|
852
910
|
return execute_workflow_by_id(_get_dbos_instance(), workflow_id, False)
|
|
853
911
|
|
|
854
912
|
@classproperty
|
dbos/_dbos_config.py
CHANGED
|
@@ -192,7 +192,11 @@ def load_config(
|
|
|
192
192
|
data = cast(ConfigFile, data)
|
|
193
193
|
db_connection = load_db_connection()
|
|
194
194
|
if not silent:
|
|
195
|
-
if
|
|
195
|
+
if os.getenv("DBOS_DBHOST"):
|
|
196
|
+
print(
|
|
197
|
+
"[bold blue]Loading database connection parameters from debug environment variables[/bold blue]"
|
|
198
|
+
)
|
|
199
|
+
elif data["database"].get("hostname"):
|
|
196
200
|
print(
|
|
197
201
|
"[bold blue]Loading database connection parameters from dbos-config.yaml[/bold blue]"
|
|
198
202
|
)
|
|
@@ -205,32 +209,62 @@ def load_config(
|
|
|
205
209
|
"[bold blue]Using default database connection parameters (localhost)[/bold blue]"
|
|
206
210
|
)
|
|
207
211
|
|
|
212
|
+
dbos_dbport: Optional[int] = None
|
|
213
|
+
dbport_env = os.getenv("DBOS_DBPORT")
|
|
214
|
+
if dbport_env:
|
|
215
|
+
try:
|
|
216
|
+
dbos_dbport = int(dbport_env)
|
|
217
|
+
except ValueError:
|
|
218
|
+
pass
|
|
219
|
+
dbos_dblocalsuffix: Optional[bool] = None
|
|
220
|
+
dblocalsuffix_env = os.getenv("DBOS_DBLOCALSUFFIX")
|
|
221
|
+
if dblocalsuffix_env:
|
|
222
|
+
try:
|
|
223
|
+
dbos_dblocalsuffix = dblocalsuffix_env.casefold() == "true".casefold()
|
|
224
|
+
except ValueError:
|
|
225
|
+
pass
|
|
226
|
+
|
|
208
227
|
data["database"]["hostname"] = (
|
|
209
|
-
|
|
228
|
+
os.getenv("DBOS_DBHOST")
|
|
229
|
+
or data["database"].get("hostname")
|
|
230
|
+
or db_connection.get("hostname")
|
|
231
|
+
or "localhost"
|
|
210
232
|
)
|
|
233
|
+
|
|
211
234
|
data["database"]["port"] = (
|
|
212
|
-
data["database"].get("port") or db_connection.get("port") or 5432
|
|
235
|
+
dbos_dbport or data["database"].get("port") or db_connection.get("port") or 5432
|
|
213
236
|
)
|
|
214
237
|
data["database"]["username"] = (
|
|
215
|
-
|
|
238
|
+
os.getenv("DBOS_DBUSER")
|
|
239
|
+
or data["database"].get("username")
|
|
240
|
+
or db_connection.get("username")
|
|
241
|
+
or "postgres"
|
|
216
242
|
)
|
|
217
243
|
data["database"]["password"] = (
|
|
218
|
-
|
|
244
|
+
os.getenv("DBOS_DBPASSWORD")
|
|
245
|
+
or data["database"].get("password")
|
|
219
246
|
or db_connection.get("password")
|
|
220
247
|
or os.environ.get("PGPASSWORD")
|
|
221
248
|
or "dbos"
|
|
222
249
|
)
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
250
|
+
|
|
251
|
+
local_suffix = False
|
|
252
|
+
dbcon_local_suffix = db_connection.get("local_suffix")
|
|
253
|
+
if dbcon_local_suffix is not None:
|
|
254
|
+
local_suffix = dbcon_local_suffix
|
|
255
|
+
if data["database"].get("local_suffix") is not None:
|
|
256
|
+
local_suffix = data["database"].get("local_suffix")
|
|
257
|
+
if dbos_dblocalsuffix is not None:
|
|
258
|
+
local_suffix = dbos_dblocalsuffix
|
|
259
|
+
data["database"]["local_suffix"] = local_suffix
|
|
228
260
|
|
|
229
261
|
# Configure the DBOS logger
|
|
230
262
|
config_logger(data)
|
|
231
263
|
|
|
232
264
|
# Check the connectivity to the database and make sure it's properly configured
|
|
233
|
-
if
|
|
265
|
+
# Note, never use db wizard if the DBOS is running in debug mode (i.e. DBOS_DEBUG_WORKFLOW_ID env var is set)
|
|
266
|
+
debugWorkflowId = os.getenv("DBOS_DEBUG_WORKFLOW_ID")
|
|
267
|
+
if use_db_wizard and debugWorkflowId is None:
|
|
234
268
|
data = db_wizard(data, config_file_path)
|
|
235
269
|
|
|
236
270
|
if "local_suffix" in data["database"] and data["database"]["local_suffix"]:
|
dbos/_debug.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import runpy
|
|
3
|
+
import sys
|
|
4
|
+
from typing import Union
|
|
5
|
+
|
|
6
|
+
from dbos import DBOS
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PythonModule:
|
|
10
|
+
def __init__(self, module_name: str):
|
|
11
|
+
self.module_name = module_name
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def debug_workflow(workflow_id: str, entrypoint: Union[str, PythonModule]) -> None:
|
|
15
|
+
# include the current directory (represented by empty string) in the search path
|
|
16
|
+
# if it not already included
|
|
17
|
+
if "" not in sys.path:
|
|
18
|
+
sys.path.insert(0, "")
|
|
19
|
+
if isinstance(entrypoint, str):
|
|
20
|
+
runpy.run_path(entrypoint)
|
|
21
|
+
elif isinstance(entrypoint, PythonModule):
|
|
22
|
+
runpy.run_module(entrypoint.module_name)
|
|
23
|
+
else:
|
|
24
|
+
raise ValueError("Invalid entrypoint type. Must be a string or PythonModule.")
|
|
25
|
+
|
|
26
|
+
DBOS.logger.info(f"Debugging workflow {workflow_id}...")
|
|
27
|
+
DBOS.launch(debug_mode=True)
|
|
28
|
+
handle = DBOS.execute_workflow_id(workflow_id)
|
|
29
|
+
handle.get_result()
|
|
30
|
+
DBOS.logger.info("Workflow Debugging complete. Exiting process.")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def parse_start_command(command: str) -> Union[str, PythonModule]:
|
|
34
|
+
match = re.match(r"fastapi\s+run\s+(\.?[\w/]+\.py)", command)
|
|
35
|
+
if match:
|
|
36
|
+
return match.group(1)
|
|
37
|
+
match = re.match(r"python3?\s+(\.?[\w/]+\.py)", command)
|
|
38
|
+
if match:
|
|
39
|
+
return match.group(1)
|
|
40
|
+
match = re.match(r"python3?\s+-m\s+([\w\.]+)", command)
|
|
41
|
+
if match:
|
|
42
|
+
return PythonModule(match.group(1))
|
|
43
|
+
raise ValueError(
|
|
44
|
+
"Invalid command format. Must be 'fastapi run <script>' or 'python <script>' or 'python -m <module>'"
|
|
45
|
+
)
|
dbos/_error.py
CHANGED
|
@@ -36,6 +36,7 @@ class DBOSErrorCode(Enum):
|
|
|
36
36
|
MaxStepRetriesExceeded = 7
|
|
37
37
|
NotAuthorized = 8
|
|
38
38
|
ConflictingWorkflowError = 9
|
|
39
|
+
WorkflowCancelled = 10
|
|
39
40
|
ConflictingRegistrationError = 25
|
|
40
41
|
|
|
41
42
|
|
|
@@ -130,6 +131,16 @@ class DBOSMaxStepRetriesExceeded(DBOSException):
|
|
|
130
131
|
)
|
|
131
132
|
|
|
132
133
|
|
|
134
|
+
class DBOSWorkflowCancelledError(DBOSException):
|
|
135
|
+
"""Exception raised when the workflow has already been cancelled."""
|
|
136
|
+
|
|
137
|
+
def __init__(self, msg: str) -> None:
|
|
138
|
+
super().__init__(
|
|
139
|
+
msg,
|
|
140
|
+
dbos_error_code=DBOSErrorCode.WorkflowCancelled.value,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
133
144
|
class DBOSConflictingRegistrationError(DBOSException):
|
|
134
145
|
"""Exception raised when conflicting decorators are applied to the same function."""
|
|
135
146
|
|
dbos/_logger.py
CHANGED
|
@@ -8,6 +8,8 @@ from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
|
8
8
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
9
9
|
from opentelemetry.sdk.resources import Resource
|
|
10
10
|
|
|
11
|
+
from dbos._utils import GlobalParams
|
|
12
|
+
|
|
11
13
|
if TYPE_CHECKING:
|
|
12
14
|
from ._dbos_config import ConfigFile
|
|
13
15
|
|
|
@@ -19,13 +21,11 @@ class DBOSLogTransformer(logging.Filter):
|
|
|
19
21
|
def __init__(self) -> None:
|
|
20
22
|
super().__init__()
|
|
21
23
|
self.app_id = os.environ.get("DBOS__APPID", "")
|
|
22
|
-
self.app_version = os.environ.get("DBOS__APPVERSION", "")
|
|
23
|
-
self.executor_id = os.environ.get("DBOS__VMID", "local")
|
|
24
24
|
|
|
25
25
|
def filter(self, record: Any) -> bool:
|
|
26
26
|
record.applicationID = self.app_id
|
|
27
|
-
record.applicationVersion =
|
|
28
|
-
record.executorID =
|
|
27
|
+
record.applicationVersion = GlobalParams.app_version
|
|
28
|
+
record.executorID = GlobalParams.executor_id
|
|
29
29
|
return True
|
|
30
30
|
|
|
31
31
|
|
|
@@ -86,9 +86,8 @@ def config_logger(config: "ConfigFile") -> None:
|
|
|
86
86
|
dbos_logger.addFilter(_otlp_transformer)
|
|
87
87
|
|
|
88
88
|
|
|
89
|
-
def add_otlp_to_all_loggers(
|
|
89
|
+
def add_otlp_to_all_loggers() -> None:
|
|
90
90
|
if _otlp_handler is not None and _otlp_transformer is not None:
|
|
91
|
-
_otlp_transformer.app_version = app_version
|
|
92
91
|
root = logging.root
|
|
93
92
|
|
|
94
93
|
root.addHandler(_otlp_handler)
|
dbos/_queue.py
CHANGED
|
@@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Optional, TypedDict
|
|
|
5
5
|
from psycopg import errors
|
|
6
6
|
from sqlalchemy.exc import OperationalError
|
|
7
7
|
|
|
8
|
+
from dbos._utils import GlobalParams
|
|
9
|
+
|
|
8
10
|
from ._core import P, R, execute_workflow_by_id, start_workflow
|
|
9
11
|
|
|
10
12
|
if TYPE_CHECKING:
|
|
@@ -71,7 +73,9 @@ def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
|
|
|
71
73
|
return
|
|
72
74
|
for _, queue in dbos._registry.queue_info_map.items():
|
|
73
75
|
try:
|
|
74
|
-
wf_ids = dbos._sys_db.start_queued_workflows(
|
|
76
|
+
wf_ids = dbos._sys_db.start_queued_workflows(
|
|
77
|
+
queue, GlobalParams.executor_id
|
|
78
|
+
)
|
|
75
79
|
for id in wf_ids:
|
|
76
80
|
execute_workflow_by_id(dbos, id)
|
|
77
81
|
except OperationalError as e:
|