dbos 0.24.0a4__py3-none-any.whl → 0.24.0a6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbos might be problematic. Click here for more details.
- dbos/__init__.py +2 -1
- dbos/_core.py +3 -1
- dbos/_db_wizard.py +18 -14
- dbos/_dbos.py +49 -12
- dbos/_dbos_config.py +256 -36
- dbos/_error.py +9 -3
- dbos/_templates/dbos-db-starter/dbos-config.yaml.dbos +2 -4
- dbos/dbos-config.schema.json +4 -1
- {dbos-0.24.0a4.dist-info → dbos-0.24.0a6.dist-info}/METADATA +1 -1
- {dbos-0.24.0a4.dist-info → dbos-0.24.0a6.dist-info}/RECORD +13 -13
- {dbos-0.24.0a4.dist-info → dbos-0.24.0a6.dist-info}/WHEEL +0 -0
- {dbos-0.24.0a4.dist-info → dbos-0.24.0a6.dist-info}/entry_points.txt +0 -0
- {dbos-0.24.0a4.dist-info → dbos-0.24.0a6.dist-info}/licenses/LICENSE +0 -0
dbos/__init__.py
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
from . import _error as error
|
|
2
2
|
from ._context import DBOSContextEnsure, DBOSContextSetAuth, SetWorkflowID
|
|
3
3
|
from ._dbos import DBOS, DBOSConfiguredInstance, WorkflowHandle, WorkflowStatus
|
|
4
|
-
from ._dbos_config import ConfigFile, get_dbos_database_url, load_config
|
|
4
|
+
from ._dbos_config import ConfigFile, DBOSConfig, get_dbos_database_url, load_config
|
|
5
5
|
from ._kafka_message import KafkaMessage
|
|
6
6
|
from ._queue import Queue
|
|
7
7
|
from ._sys_db import GetWorkflowsInput, WorkflowStatusString
|
|
8
8
|
|
|
9
9
|
__all__ = [
|
|
10
10
|
"ConfigFile",
|
|
11
|
+
"DBOSConfig",
|
|
11
12
|
"DBOS",
|
|
12
13
|
"DBOSConfiguredInstance",
|
|
13
14
|
"DBOSContextEnsure",
|
dbos/_core.py
CHANGED
|
@@ -824,7 +824,9 @@ def decorate_step(
|
|
|
824
824
|
stepOutcome = Outcome[R].make(functools.partial(func, *args, **kwargs))
|
|
825
825
|
if retries_allowed:
|
|
826
826
|
stepOutcome = stepOutcome.retry(
|
|
827
|
-
max_attempts,
|
|
827
|
+
max_attempts,
|
|
828
|
+
on_exception,
|
|
829
|
+
lambda i: DBOSMaxStepRetriesExceeded(func.__name__, i),
|
|
828
830
|
)
|
|
829
831
|
|
|
830
832
|
outcome = (
|
dbos/_db_wizard.py
CHANGED
|
@@ -28,7 +28,20 @@ class DatabaseConnection(TypedDict):
|
|
|
28
28
|
local_suffix: Optional[bool]
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def db_wizard(config: "ConfigFile"
|
|
31
|
+
def db_wizard(config: "ConfigFile") -> "ConfigFile":
|
|
32
|
+
"""Checks database connectivity and helps the user start a database if needed
|
|
33
|
+
|
|
34
|
+
First, check connectivity to the database configured in the provided `config` object.
|
|
35
|
+
If it fails:
|
|
36
|
+
- Return an error if the connection failed due to incorrect credentials.
|
|
37
|
+
- Return an error if it detects a non-default configuration.
|
|
38
|
+
- Otherwise assume the configured database is not running and guide the user through setting it up.
|
|
39
|
+
|
|
40
|
+
The wizard will first attempt to start a local Postgres instance using Docker.
|
|
41
|
+
If Docker is not available, it will prompt the user to connect to a DBOS Cloud database.
|
|
42
|
+
|
|
43
|
+
Finally, if a database was configured, its connection details will be saved in the local `.dbos/db_connection` file.
|
|
44
|
+
"""
|
|
32
45
|
# 1. Check the connectivity to the database. Return if successful. If cannot connect, continue to the following steps.
|
|
33
46
|
db_connection_error = _check_db_connectivity(config)
|
|
34
47
|
if db_connection_error is None:
|
|
@@ -44,27 +57,18 @@ def db_wizard(config: "ConfigFile", config_file_path: str) -> "ConfigFile":
|
|
|
44
57
|
raise DBOSInitializationError(
|
|
45
58
|
f"Could not connect to Postgres: password authentication failed: {db_connection_error}"
|
|
46
59
|
)
|
|
47
|
-
db_config = config["database"]
|
|
48
|
-
|
|
49
|
-
# Read the config file and check if the database hostname/port/username are set. If so, skip the wizard.
|
|
50
|
-
with open(config_file_path, "r") as file:
|
|
51
|
-
content = file.read()
|
|
52
|
-
local_config = yaml.safe_load(content)
|
|
53
|
-
if "database" not in local_config:
|
|
54
|
-
local_config["database"] = {}
|
|
55
|
-
local_config = cast("ConfigFile", local_config)
|
|
56
60
|
|
|
61
|
+
# If the database config is not the default one, surface the error and exit.
|
|
62
|
+
db_config = config["database"] # FIXME: what if database is not in config?
|
|
57
63
|
if (
|
|
58
|
-
|
|
59
|
-
or local_config["database"].get("port")
|
|
60
|
-
or local_config["database"].get("username")
|
|
61
|
-
or db_config["hostname"] != "localhost"
|
|
64
|
+
db_config["hostname"] != "localhost"
|
|
62
65
|
or db_config["port"] != 5432
|
|
63
66
|
or db_config["username"] != "postgres"
|
|
64
67
|
):
|
|
65
68
|
raise DBOSInitializationError(
|
|
66
69
|
f"Could not connect to the database. Exception: {db_connection_error}"
|
|
67
70
|
)
|
|
71
|
+
|
|
68
72
|
print("[yellow]Postgres not detected locally[/yellow]")
|
|
69
73
|
|
|
70
74
|
# 3. If the database config is the default one, check if the user has Docker properly installed.
|
dbos/_dbos.py
CHANGED
|
@@ -88,13 +88,23 @@ from ._context import (
|
|
|
88
88
|
assert_current_dbos_context,
|
|
89
89
|
get_local_dbos_context,
|
|
90
90
|
)
|
|
91
|
-
from ._dbos_config import
|
|
91
|
+
from ._dbos_config import (
|
|
92
|
+
ConfigFile,
|
|
93
|
+
DBOSConfig,
|
|
94
|
+
check_config_consistency,
|
|
95
|
+
is_dbos_configfile,
|
|
96
|
+
load_config,
|
|
97
|
+
overwrite_config,
|
|
98
|
+
process_config,
|
|
99
|
+
set_env_vars,
|
|
100
|
+
translate_dbos_config_to_config_file,
|
|
101
|
+
)
|
|
92
102
|
from ._error import (
|
|
93
103
|
DBOSConflictingRegistrationError,
|
|
94
104
|
DBOSException,
|
|
95
105
|
DBOSNonExistentWorkflowError,
|
|
96
106
|
)
|
|
97
|
-
from ._logger import add_otlp_to_all_loggers, dbos_logger
|
|
107
|
+
from ._logger import add_otlp_to_all_loggers, config_logger, dbos_logger, init_logger
|
|
98
108
|
from ._sys_db import SystemDatabase
|
|
99
109
|
|
|
100
110
|
# Most DBOS functions are just any callable F, so decorators / wrappers work on F
|
|
@@ -257,7 +267,7 @@ class DBOS:
|
|
|
257
267
|
def __new__(
|
|
258
268
|
cls: Type[DBOS],
|
|
259
269
|
*,
|
|
260
|
-
config: Optional[ConfigFile] = None,
|
|
270
|
+
config: Optional[Union[ConfigFile, DBOSConfig]] = None,
|
|
261
271
|
fastapi: Optional["FastAPI"] = None,
|
|
262
272
|
flask: Optional["Flask"] = None,
|
|
263
273
|
conductor_url: Optional[str] = None,
|
|
@@ -302,7 +312,7 @@ class DBOS:
|
|
|
302
312
|
def __init__(
|
|
303
313
|
self,
|
|
304
314
|
*,
|
|
305
|
-
config: Optional[ConfigFile] = None,
|
|
315
|
+
config: Optional[Union[ConfigFile, DBOSConfig]] = None,
|
|
306
316
|
fastapi: Optional["FastAPI"] = None,
|
|
307
317
|
flask: Optional["Flask"] = None,
|
|
308
318
|
conductor_url: Optional[str] = None,
|
|
@@ -312,12 +322,7 @@ class DBOS:
|
|
|
312
322
|
return
|
|
313
323
|
|
|
314
324
|
self._initialized: bool = True
|
|
315
|
-
|
|
316
|
-
config = load_config()
|
|
317
|
-
set_env_vars(config)
|
|
318
|
-
dbos_tracer.config(config)
|
|
319
|
-
dbos_logger.info("Initializing DBOS")
|
|
320
|
-
self.config: ConfigFile = config
|
|
325
|
+
|
|
321
326
|
self._launched: bool = False
|
|
322
327
|
self._debug_mode: bool = False
|
|
323
328
|
self._sys_db_field: Optional[SystemDatabase] = None
|
|
@@ -334,6 +339,36 @@ class DBOS:
|
|
|
334
339
|
self.conductor_key: Optional[str] = conductor_key
|
|
335
340
|
self.conductor_websocket: Optional[ConductorWebsocket] = None
|
|
336
341
|
|
|
342
|
+
init_logger()
|
|
343
|
+
|
|
344
|
+
unvalidated_config: Optional[ConfigFile] = None
|
|
345
|
+
|
|
346
|
+
if config is None:
|
|
347
|
+
# If no config is provided, load it from dbos-config.yaml
|
|
348
|
+
unvalidated_config = load_config(run_process_config=False)
|
|
349
|
+
elif is_dbos_configfile(config):
|
|
350
|
+
unvalidated_config = cast(ConfigFile, config)
|
|
351
|
+
if os.environ.get("DBOS__CLOUD") == "true":
|
|
352
|
+
unvalidated_config = overwrite_config(unvalidated_config)
|
|
353
|
+
check_config_consistency(name=unvalidated_config["name"])
|
|
354
|
+
else:
|
|
355
|
+
unvalidated_config = translate_dbos_config_to_config_file(
|
|
356
|
+
cast(DBOSConfig, config)
|
|
357
|
+
)
|
|
358
|
+
if os.environ.get("DBOS__CLOUD") == "true":
|
|
359
|
+
unvalidated_config = overwrite_config(unvalidated_config)
|
|
360
|
+
check_config_consistency(name=unvalidated_config["name"])
|
|
361
|
+
|
|
362
|
+
if unvalidated_config is not None:
|
|
363
|
+
self.config: ConfigFile = process_config(data=unvalidated_config)
|
|
364
|
+
else:
|
|
365
|
+
raise ValueError("No valid configuration was loaded.")
|
|
366
|
+
|
|
367
|
+
set_env_vars(self.config)
|
|
368
|
+
config_logger(self.config)
|
|
369
|
+
dbos_tracer.config(self.config)
|
|
370
|
+
dbos_logger.info("Initializing DBOS")
|
|
371
|
+
|
|
337
372
|
# If using FastAPI, set up middleware and lifecycle events
|
|
338
373
|
if self.fastapi is not None:
|
|
339
374
|
from ._fastapi import setup_fastapi_middleware
|
|
@@ -419,7 +454,7 @@ class DBOS:
|
|
|
419
454
|
if debug_mode:
|
|
420
455
|
return
|
|
421
456
|
|
|
422
|
-
admin_port = self.config
|
|
457
|
+
admin_port = self.config.get("runtimeConfig", {}).get("admin_port")
|
|
423
458
|
if admin_port is None:
|
|
424
459
|
admin_port = 3001
|
|
425
460
|
self._admin_server_field = AdminServer(dbos=self, port=admin_port)
|
|
@@ -923,7 +958,9 @@ class DBOS:
|
|
|
923
958
|
reg = _get_or_create_dbos_registry()
|
|
924
959
|
if reg.config is not None:
|
|
925
960
|
return reg.config
|
|
926
|
-
config =
|
|
961
|
+
config = (
|
|
962
|
+
load_config()
|
|
963
|
+
) # This will return the processed & validated config (with defaults)
|
|
927
964
|
reg.config = config
|
|
928
965
|
return config
|
|
929
966
|
|
dbos/_dbos_config.py
CHANGED
|
@@ -1,21 +1,47 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
+
import sys
|
|
4
5
|
from importlib import resources
|
|
5
|
-
from typing import Any, Dict, List, Optional, TypedDict, cast
|
|
6
|
+
from typing import Any, Dict, List, Optional, TypedDict, Union, cast
|
|
7
|
+
|
|
8
|
+
if sys.version_info < (3, 10):
|
|
9
|
+
from typing_extensions import TypeGuard
|
|
10
|
+
else:
|
|
11
|
+
from typing import TypeGuard
|
|
6
12
|
|
|
7
13
|
import yaml
|
|
8
14
|
from jsonschema import ValidationError, validate
|
|
9
15
|
from rich import print
|
|
10
|
-
from sqlalchemy import URL
|
|
16
|
+
from sqlalchemy import URL, make_url
|
|
11
17
|
|
|
12
18
|
from ._db_wizard import db_wizard, load_db_connection
|
|
13
19
|
from ._error import DBOSInitializationError
|
|
14
|
-
from ._logger import
|
|
20
|
+
from ._logger import dbos_logger
|
|
15
21
|
|
|
16
22
|
DBOS_CONFIG_PATH = "dbos-config.yaml"
|
|
17
23
|
|
|
18
24
|
|
|
25
|
+
class DBOSConfig(TypedDict):
|
|
26
|
+
"""
|
|
27
|
+
Data structure containing the DBOS library configuration.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
name (str): Application name
|
|
31
|
+
database_url (str): Database connection string
|
|
32
|
+
sys_db_name (str): System database name
|
|
33
|
+
log_level (str): Log level
|
|
34
|
+
otlp_traces_endpoints: List[str]: OTLP traces endpoints
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
name: str
|
|
38
|
+
database_url: Optional[str]
|
|
39
|
+
sys_db_name: Optional[str]
|
|
40
|
+
log_level: Optional[str]
|
|
41
|
+
otlp_traces_endpoints: Optional[List[str]]
|
|
42
|
+
admin_port: Optional[int]
|
|
43
|
+
|
|
44
|
+
|
|
19
45
|
class RuntimeConfig(TypedDict, total=False):
|
|
20
46
|
start: List[str]
|
|
21
47
|
setup: Optional[List[str]]
|
|
@@ -33,11 +59,30 @@ class DatabaseConfig(TypedDict, total=False):
|
|
|
33
59
|
ssl: Optional[bool]
|
|
34
60
|
ssl_ca: Optional[str]
|
|
35
61
|
local_suffix: Optional[bool]
|
|
36
|
-
app_db_client: Optional[str]
|
|
37
62
|
migrate: Optional[List[str]]
|
|
38
63
|
rollback: Optional[List[str]]
|
|
39
64
|
|
|
40
65
|
|
|
66
|
+
def parse_database_url_to_dbconfig(database_url: str) -> DatabaseConfig:
|
|
67
|
+
db_url = make_url(database_url)
|
|
68
|
+
db_config = {
|
|
69
|
+
"hostname": db_url.host,
|
|
70
|
+
"port": db_url.port or 5432,
|
|
71
|
+
"username": db_url.username,
|
|
72
|
+
"password": db_url.password,
|
|
73
|
+
"app_db_name": db_url.database,
|
|
74
|
+
}
|
|
75
|
+
for key, value in db_url.query.items():
|
|
76
|
+
str_value = value[0] if isinstance(value, tuple) else value
|
|
77
|
+
if key == "connect_timeout":
|
|
78
|
+
db_config["connectionTimeoutMillis"] = int(str_value) * 1000
|
|
79
|
+
elif key == "sslmode":
|
|
80
|
+
db_config["ssl"] = str_value == "require"
|
|
81
|
+
elif key == "sslrootcert":
|
|
82
|
+
db_config["ssl_ca"] = str_value
|
|
83
|
+
return cast(DatabaseConfig, db_config)
|
|
84
|
+
|
|
85
|
+
|
|
41
86
|
class OTLPExporterConfig(TypedDict, total=False):
|
|
42
87
|
logsEndpoint: Optional[str]
|
|
43
88
|
tracesEndpoint: Optional[str]
|
|
@@ -61,9 +106,9 @@ class ConfigFile(TypedDict, total=False):
|
|
|
61
106
|
|
|
62
107
|
Attributes:
|
|
63
108
|
name (str): Application name
|
|
64
|
-
language (str): The app language (probably `python`)
|
|
65
109
|
runtimeConfig (RuntimeConfig): Configuration for request serving
|
|
66
110
|
database (DatabaseConfig): Configuration for the application and system databases
|
|
111
|
+
database_url (str): Database connection string
|
|
67
112
|
telemetry (TelemetryConfig): Configuration for tracing / logging
|
|
68
113
|
env (Dict[str,str]): Environment varialbes
|
|
69
114
|
application (Dict[str, Any]): Application-specific configuration section
|
|
@@ -71,12 +116,71 @@ class ConfigFile(TypedDict, total=False):
|
|
|
71
116
|
"""
|
|
72
117
|
|
|
73
118
|
name: str
|
|
74
|
-
language: str
|
|
75
119
|
runtimeConfig: RuntimeConfig
|
|
76
120
|
database: DatabaseConfig
|
|
121
|
+
database_url: Optional[str]
|
|
77
122
|
telemetry: Optional[TelemetryConfig]
|
|
78
123
|
env: Dict[str, str]
|
|
79
|
-
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def is_dbos_configfile(data: Union[ConfigFile, DBOSConfig]) -> TypeGuard[DBOSConfig]:
|
|
127
|
+
"""
|
|
128
|
+
Type guard to check if the provided data is a DBOSConfig.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
data: The configuration object to check
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
True if the data is a DBOSConfig, False otherwise
|
|
135
|
+
"""
|
|
136
|
+
return (
|
|
137
|
+
isinstance(data, dict)
|
|
138
|
+
and "name" in data
|
|
139
|
+
and (
|
|
140
|
+
"runtimeConfig" in data
|
|
141
|
+
or "database" in data
|
|
142
|
+
or "env" in data
|
|
143
|
+
or "telemetry" in data
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def translate_dbos_config_to_config_file(config: DBOSConfig) -> ConfigFile:
|
|
149
|
+
if "name" not in config:
|
|
150
|
+
raise DBOSInitializationError(f"Configuration must specify an application name")
|
|
151
|
+
|
|
152
|
+
translated_config: ConfigFile = {
|
|
153
|
+
"name": config["name"],
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
# Database config
|
|
157
|
+
db_config: DatabaseConfig = {}
|
|
158
|
+
database_url = config.get("database_url")
|
|
159
|
+
if database_url:
|
|
160
|
+
db_config = parse_database_url_to_dbconfig(database_url)
|
|
161
|
+
if "sys_db_name" in config:
|
|
162
|
+
db_config["sys_db_name"] = config.get("sys_db_name")
|
|
163
|
+
if db_config:
|
|
164
|
+
translated_config["database"] = db_config
|
|
165
|
+
|
|
166
|
+
# Admin port
|
|
167
|
+
if "admin_port" in config:
|
|
168
|
+
translated_config["runtimeConfig"] = {"admin_port": config["admin_port"]}
|
|
169
|
+
|
|
170
|
+
# Telemetry config
|
|
171
|
+
telemetry = {}
|
|
172
|
+
# Add OTLPExporter if traces endpoints exist
|
|
173
|
+
otlp_trace_endpoints = config.get("otlp_traces_endpoints")
|
|
174
|
+
if isinstance(otlp_trace_endpoints, list) and len(otlp_trace_endpoints) > 0:
|
|
175
|
+
telemetry["OTLPExporter"] = {"tracesEndpoint": otlp_trace_endpoints[0]}
|
|
176
|
+
# Default to INFO -- the logging seems to default to WARN otherwise.
|
|
177
|
+
log_level = config.get("log_level", "INFO")
|
|
178
|
+
if log_level:
|
|
179
|
+
telemetry["logs"] = {"logLevel": log_level}
|
|
180
|
+
if telemetry:
|
|
181
|
+
translated_config["telemetry"] = cast(TelemetryConfig, telemetry)
|
|
182
|
+
|
|
183
|
+
return translated_config
|
|
80
184
|
|
|
81
185
|
|
|
82
186
|
def _substitute_env_vars(content: str) -> str:
|
|
@@ -110,7 +214,7 @@ def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
|
|
|
110
214
|
str: Database URL for the application database
|
|
111
215
|
|
|
112
216
|
"""
|
|
113
|
-
dbos_config = load_config(config_file_path)
|
|
217
|
+
dbos_config = process_config(data=load_config(config_file_path))
|
|
114
218
|
db_url = URL.create(
|
|
115
219
|
"postgresql+psycopg",
|
|
116
220
|
username=dbos_config["database"]["username"],
|
|
@@ -125,6 +229,7 @@ def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
|
|
|
125
229
|
def load_config(
|
|
126
230
|
config_file_path: str = DBOS_CONFIG_PATH,
|
|
127
231
|
*,
|
|
232
|
+
run_process_config: bool = True,
|
|
128
233
|
use_db_wizard: bool = True,
|
|
129
234
|
silent: bool = False,
|
|
130
235
|
) -> ConfigFile:
|
|
@@ -141,13 +246,17 @@ def load_config(
|
|
|
141
246
|
|
|
142
247
|
"""
|
|
143
248
|
|
|
144
|
-
init_logger()
|
|
145
|
-
|
|
146
249
|
with open(config_file_path, "r") as file:
|
|
147
250
|
content = file.read()
|
|
148
251
|
substituted_content = _substitute_env_vars(content)
|
|
149
252
|
data = yaml.safe_load(substituted_content)
|
|
150
253
|
|
|
254
|
+
if not isinstance(data, dict):
|
|
255
|
+
raise DBOSInitializationError(
|
|
256
|
+
f"dbos-config.yaml must contain a dictionary, not {type(data)}"
|
|
257
|
+
)
|
|
258
|
+
data = cast(Dict[str, Any], data)
|
|
259
|
+
|
|
151
260
|
# Load the JSON schema relative to the package root
|
|
152
261
|
schema_file = resources.files("dbos").joinpath("dbos-config.schema.json")
|
|
153
262
|
with schema_file.open("r") as f:
|
|
@@ -159,37 +268,48 @@ def load_config(
|
|
|
159
268
|
except ValidationError as e:
|
|
160
269
|
raise DBOSInitializationError(f"Validation error: {e}")
|
|
161
270
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
raise DBOSInitializationError(
|
|
167
|
-
f"dbos-config.yaml must specify an application name"
|
|
168
|
-
)
|
|
271
|
+
data = cast(ConfigFile, data)
|
|
272
|
+
if run_process_config:
|
|
273
|
+
data = process_config(data=data, use_db_wizard=use_db_wizard, silent=silent)
|
|
274
|
+
return data # type: ignore
|
|
169
275
|
|
|
170
|
-
if "language" not in data:
|
|
171
|
-
raise DBOSInitializationError(
|
|
172
|
-
f"dbos-config.yaml must specify the application language is Python"
|
|
173
|
-
)
|
|
174
276
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
277
|
+
def process_config(
|
|
278
|
+
*,
|
|
279
|
+
use_db_wizard: bool = True,
|
|
280
|
+
data: ConfigFile,
|
|
281
|
+
silent: bool = False,
|
|
282
|
+
) -> ConfigFile:
|
|
179
283
|
|
|
180
|
-
if "
|
|
181
|
-
raise DBOSInitializationError(f"
|
|
284
|
+
if "name" not in data:
|
|
285
|
+
raise DBOSInitializationError(f"Configuration must specify an application name")
|
|
182
286
|
|
|
183
287
|
if not _is_valid_app_name(data["name"]):
|
|
184
288
|
raise DBOSInitializationError(
|
|
185
289
|
f'Invalid app name {data["name"]}. App names must be between 3 and 30 characters long and contain only lowercase letters, numbers, dashes, and underscores.'
|
|
186
290
|
)
|
|
187
291
|
|
|
188
|
-
if "
|
|
292
|
+
if "database" not in data:
|
|
293
|
+
data["database"] = {}
|
|
294
|
+
|
|
295
|
+
# database_url takes precedence over database config, but we need to preserve rollback and migrate if they exist
|
|
296
|
+
migrate = data["database"].get("migrate", False)
|
|
297
|
+
rollback = data["database"].get("rollback", False)
|
|
298
|
+
local_suffix = data["database"].get("local_suffix", False)
|
|
299
|
+
if data.get("database_url"):
|
|
300
|
+
dbconfig = parse_database_url_to_dbconfig(cast(str, data["database_url"]))
|
|
301
|
+
if migrate:
|
|
302
|
+
dbconfig["migrate"] = cast(List[str], migrate)
|
|
303
|
+
if rollback:
|
|
304
|
+
dbconfig["rollback"] = cast(List[str], rollback)
|
|
305
|
+
if local_suffix:
|
|
306
|
+
dbconfig["local_suffix"] = cast(bool, local_suffix)
|
|
307
|
+
data["database"] = dbconfig
|
|
308
|
+
|
|
309
|
+
if "app_db_name" not in data["database"] or not (data["database"]["app_db_name"]):
|
|
189
310
|
data["database"]["app_db_name"] = _app_name_to_db_name(data["name"])
|
|
190
311
|
|
|
191
312
|
# Load the DB connection file. Use its values for missing fields from dbos-config.yaml. Use defaults otherwise.
|
|
192
|
-
data = cast(ConfigFile, data)
|
|
193
313
|
db_connection = load_db_connection()
|
|
194
314
|
if not silent:
|
|
195
315
|
if os.getenv("DBOS_DBHOST"):
|
|
@@ -252,26 +372,24 @@ def load_config(
|
|
|
252
372
|
dbcon_local_suffix = db_connection.get("local_suffix")
|
|
253
373
|
if dbcon_local_suffix is not None:
|
|
254
374
|
local_suffix = dbcon_local_suffix
|
|
255
|
-
|
|
256
|
-
|
|
375
|
+
db_local_suffix = data["database"].get("local_suffix")
|
|
376
|
+
if db_local_suffix is not None:
|
|
377
|
+
local_suffix = db_local_suffix
|
|
257
378
|
if dbos_dblocalsuffix is not None:
|
|
258
379
|
local_suffix = dbos_dblocalsuffix
|
|
259
380
|
data["database"]["local_suffix"] = local_suffix
|
|
260
381
|
|
|
261
|
-
# Configure the DBOS logger
|
|
262
|
-
config_logger(data)
|
|
263
|
-
|
|
264
382
|
# Check the connectivity to the database and make sure it's properly configured
|
|
265
383
|
# Note, never use db wizard if the DBOS is running in debug mode (i.e. DBOS_DEBUG_WORKFLOW_ID env var is set)
|
|
266
384
|
debugWorkflowId = os.getenv("DBOS_DEBUG_WORKFLOW_ID")
|
|
267
385
|
if use_db_wizard and debugWorkflowId is None:
|
|
268
|
-
data = db_wizard(data
|
|
386
|
+
data = db_wizard(data)
|
|
269
387
|
|
|
270
388
|
if "local_suffix" in data["database"] and data["database"]["local_suffix"]:
|
|
271
389
|
data["database"]["app_db_name"] = f"{data['database']['app_db_name']}_local"
|
|
272
390
|
|
|
273
391
|
# Return data as ConfigFile type
|
|
274
|
-
return data
|
|
392
|
+
return data
|
|
275
393
|
|
|
276
394
|
|
|
277
395
|
def _is_valid_app_name(name: str) -> bool:
|
|
@@ -291,3 +409,105 @@ def set_env_vars(config: ConfigFile) -> None:
|
|
|
291
409
|
for env, value in config.get("env", {}).items():
|
|
292
410
|
if value is not None:
|
|
293
411
|
os.environ[env] = str(value)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def overwrite_config(provided_config: ConfigFile) -> ConfigFile:
|
|
415
|
+
# Load the DBOS configuration file and force the use of:
|
|
416
|
+
# 1. The database connection parameters (sub the file data to the provided config)
|
|
417
|
+
# 2. OTLP traces endpoints (add the config data to the provided config)
|
|
418
|
+
# 3. Use the application name from the file. This is a defensive measure to ensure the application name is whatever it was registered with in the cloud
|
|
419
|
+
# 4. Remove admin_port is provided in code
|
|
420
|
+
# 5. Remove env vars if provided in code
|
|
421
|
+
# Optimistically assume that expected fields in config_from_file are present
|
|
422
|
+
|
|
423
|
+
config_from_file = load_config(run_process_config=False)
|
|
424
|
+
# Be defensive
|
|
425
|
+
if config_from_file is None:
|
|
426
|
+
return provided_config
|
|
427
|
+
|
|
428
|
+
# Name
|
|
429
|
+
provided_config["name"] = config_from_file["name"]
|
|
430
|
+
|
|
431
|
+
# Database config. Note we disregard a potential database_url in config_from_file because it is not expected from DBOS Cloud
|
|
432
|
+
if "database" not in provided_config:
|
|
433
|
+
provided_config["database"] = {}
|
|
434
|
+
provided_config["database"]["hostname"] = config_from_file["database"]["hostname"]
|
|
435
|
+
provided_config["database"]["port"] = config_from_file["database"]["port"]
|
|
436
|
+
provided_config["database"]["username"] = config_from_file["database"]["username"]
|
|
437
|
+
provided_config["database"]["password"] = config_from_file["database"]["password"]
|
|
438
|
+
provided_config["database"]["app_db_name"] = config_from_file["database"][
|
|
439
|
+
"app_db_name"
|
|
440
|
+
]
|
|
441
|
+
provided_config["database"]["sys_db_name"] = config_from_file["database"][
|
|
442
|
+
"sys_db_name"
|
|
443
|
+
]
|
|
444
|
+
provided_config["database"]["ssl"] = config_from_file["database"]["ssl"]
|
|
445
|
+
provided_config["database"]["ssl_ca"] = config_from_file["database"]["ssl_ca"]
|
|
446
|
+
|
|
447
|
+
# Telemetry config
|
|
448
|
+
if "telemetry" not in provided_config or provided_config["telemetry"] is None:
|
|
449
|
+
provided_config["telemetry"] = {
|
|
450
|
+
"OTLPExporter": {},
|
|
451
|
+
}
|
|
452
|
+
elif "OTLPExporter" not in provided_config["telemetry"]:
|
|
453
|
+
provided_config["telemetry"]["OTLPExporter"] = {}
|
|
454
|
+
|
|
455
|
+
# This is a super messy from a typing perspective.
|
|
456
|
+
# Some of ConfigFile keys are optional -- but in practice they'll always be present in hosted environments
|
|
457
|
+
# So, for Mypy, we have to (1) check the keys are present in config_from_file and (2) cast telemetry/otlp_exporters to Dict[str, Any]
|
|
458
|
+
# (2) is required because, even tho we resolved these keys earlier, mypy doesn't remember that
|
|
459
|
+
if (
|
|
460
|
+
config_from_file.get("telemetry")
|
|
461
|
+
and config_from_file["telemetry"]
|
|
462
|
+
and config_from_file["telemetry"].get("OTLPExporter")
|
|
463
|
+
):
|
|
464
|
+
|
|
465
|
+
telemetry = cast(Dict[str, Any], provided_config["telemetry"])
|
|
466
|
+
otlp_exporter = cast(Dict[str, Any], telemetry["OTLPExporter"])
|
|
467
|
+
|
|
468
|
+
source_otlp = config_from_file["telemetry"]["OTLPExporter"]
|
|
469
|
+
if source_otlp:
|
|
470
|
+
tracesEndpoint = source_otlp.get("tracesEndpoint")
|
|
471
|
+
if tracesEndpoint:
|
|
472
|
+
otlp_exporter["tracesEndpoint"] = tracesEndpoint
|
|
473
|
+
logsEndpoint = source_otlp.get("logsEndpoint")
|
|
474
|
+
if logsEndpoint:
|
|
475
|
+
otlp_exporter["logsEndpoint"] = logsEndpoint
|
|
476
|
+
|
|
477
|
+
# Runtime config
|
|
478
|
+
if (
|
|
479
|
+
"runtimeConfig" in provided_config
|
|
480
|
+
and "admin_port" in provided_config["runtimeConfig"]
|
|
481
|
+
):
|
|
482
|
+
del provided_config["runtimeConfig"][
|
|
483
|
+
"admin_port"
|
|
484
|
+
] # Admin port is expected to be 3001 (the default in dbos/_admin_server.py::__init__ ) by DBOS Cloud
|
|
485
|
+
|
|
486
|
+
# Env should be set from the hosting provider (e.g., DBOS Cloud)
|
|
487
|
+
if "env" in provided_config:
|
|
488
|
+
del provided_config["env"]
|
|
489
|
+
|
|
490
|
+
return provided_config
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def check_config_consistency(
|
|
494
|
+
*,
|
|
495
|
+
name: str,
|
|
496
|
+
config_file_path: str = DBOS_CONFIG_PATH,
|
|
497
|
+
) -> None:
|
|
498
|
+
# First load the config file and check whether it is present
|
|
499
|
+
try:
|
|
500
|
+
config = load_config(config_file_path)
|
|
501
|
+
except FileNotFoundError:
|
|
502
|
+
dbos_logger.debug(
|
|
503
|
+
f"No configuration file {config_file_path} found. Skipping consistency check with provided config."
|
|
504
|
+
)
|
|
505
|
+
return
|
|
506
|
+
except Exception as e:
|
|
507
|
+
raise e
|
|
508
|
+
|
|
509
|
+
# Check the name
|
|
510
|
+
if name != config["name"]:
|
|
511
|
+
raise DBOSInitializationError(
|
|
512
|
+
f"Provided app name '{name}' does not match the app name '{config['name']}' in {config_file_path}."
|
|
513
|
+
)
|
dbos/_error.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Errors thrown by DBOS."""
|
|
2
2
|
|
|
3
3
|
from enum import Enum
|
|
4
|
-
from typing import Optional
|
|
4
|
+
from typing import Any, Optional
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class DBOSException(Exception):
|
|
@@ -124,12 +124,18 @@ class DBOSNotAuthorizedError(DBOSException):
|
|
|
124
124
|
class DBOSMaxStepRetriesExceeded(DBOSException):
|
|
125
125
|
"""Exception raised when a step was retried the maximimum number of times without success."""
|
|
126
126
|
|
|
127
|
-
def __init__(self) -> None:
|
|
127
|
+
def __init__(self, step_name: str, max_retries: int) -> None:
|
|
128
|
+
self.step_name = step_name
|
|
129
|
+
self.max_retries = max_retries
|
|
128
130
|
super().__init__(
|
|
129
|
-
"Step
|
|
131
|
+
f"Step {step_name} has exceeded its maximum of {max_retries} retries",
|
|
130
132
|
dbos_error_code=DBOSErrorCode.MaxStepRetriesExceeded.value,
|
|
131
133
|
)
|
|
132
134
|
|
|
135
|
+
def __reduce__(self) -> Any:
|
|
136
|
+
# Tell jsonpickle how to reconstruct this object
|
|
137
|
+
return (self.__class__, (self.step_name, self.max_retries))
|
|
138
|
+
|
|
133
139
|
|
|
134
140
|
class DBOSWorkflowCancelledError(DBOSException):
|
|
135
141
|
"""Exception raised when the workflow has already been cancelled."""
|
dbos/dbos-config.schema.json
CHANGED
|
@@ -12,10 +12,13 @@
|
|
|
12
12
|
"type": "string",
|
|
13
13
|
"description": "The language used in your application",
|
|
14
14
|
"enum": [
|
|
15
|
-
"typescript",
|
|
16
15
|
"python"
|
|
17
16
|
]
|
|
18
17
|
},
|
|
18
|
+
"database_url": {
|
|
19
|
+
"type": ["string", "null"],
|
|
20
|
+
"description": "The URL of the application database"
|
|
21
|
+
},
|
|
19
22
|
"database": {
|
|
20
23
|
"type": "object",
|
|
21
24
|
"additionalProperties": false,
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
dbos-0.24.
|
|
2
|
-
dbos-0.24.
|
|
3
|
-
dbos-0.24.
|
|
4
|
-
dbos-0.24.
|
|
5
|
-
dbos/__init__.py,sha256=
|
|
1
|
+
dbos-0.24.0a6.dist-info/METADATA,sha256=Aby88_NlOl0iBzLWuRrzQwOGOxAXypxoDLMyD4cCLnI,5555
|
|
2
|
+
dbos-0.24.0a6.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
|
|
3
|
+
dbos-0.24.0a6.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
|
|
4
|
+
dbos-0.24.0a6.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
|
|
5
|
+
dbos/__init__.py,sha256=uq9LP5uY96kIS9N0yKqlvDwADmtg_Hl30uSUhyuUr-4,754
|
|
6
6
|
dbos/__main__.py,sha256=P7jAr-7L9XE5mrsQ7i4b-bLr2ap1tCQfhMByLCRWDj0,568
|
|
7
7
|
dbos/_admin_server.py,sha256=YiVn5lywz2Vg8_juyNHOYl0HVEy48--7b4phwK7r92o,5732
|
|
8
8
|
dbos/_app_db.py,sha256=QFL1ceCugJFj_LBvK_G_0tt5jjyTM-4KnqmhbuC1ggg,5826
|
|
@@ -13,13 +13,13 @@ dbos/_cloudutils/databases.py,sha256=_shqaqSvhY4n2ScgQ8IP5PDZvzvcx3YBKV8fj-cxhSY
|
|
|
13
13
|
dbos/_conductor/conductor.py,sha256=tvB1k8V3ZsI3MBEcG9sIkfbsHHx_Cvh0FZAi9vSPrXU,12698
|
|
14
14
|
dbos/_conductor/protocol.py,sha256=Lo-DjvKevTS2uKyITLjjVp56rBRzaQfKkKpieZz3VAs,5088
|
|
15
15
|
dbos/_context.py,sha256=Ue5qu3rzLfRmPkz-UUZi9ZS8iXpapRN0NTM4mbA2QmQ,17738
|
|
16
|
-
dbos/_core.py,sha256=
|
|
16
|
+
dbos/_core.py,sha256=8TTIj5shcm5hpKjouMA4VzMWl7R7MJlU-mAHB6xQBxE,37585
|
|
17
17
|
dbos/_croniter.py,sha256=XHAyUyibs_59sJQfSNWkP7rqQY6_XrlfuuCxk4jYqek,47559
|
|
18
|
-
dbos/_db_wizard.py,sha256=
|
|
19
|
-
dbos/_dbos.py,sha256=
|
|
20
|
-
dbos/_dbos_config.py,sha256=
|
|
18
|
+
dbos/_db_wizard.py,sha256=YEW2qoy6hfHQv2fZ_4nHiPUeHMFofPpNTolJ1Kvw7AQ,8394
|
|
19
|
+
dbos/_dbos.py,sha256=2A1_5Fjv54G_bNKpSsYOFwEXDWJcaM-x1i2sY_h581w,42523
|
|
20
|
+
dbos/_dbos_config.py,sha256=MJOJu0dCzdcxJi2gUMkTcJaxm_XVr2a2Dfq7vL_nEL0,17844
|
|
21
21
|
dbos/_debug.py,sha256=mmgvLkqlrljMBBow9wk01PPur9kUf2rI_11dTJXY4gw,1822
|
|
22
|
-
dbos/_error.py,sha256=
|
|
22
|
+
dbos/_error.py,sha256=B6Y9XLS1f6yrawxB2uAEYFMxFwk9BHhdxPNddKco-Fw,5399
|
|
23
23
|
dbos/_fastapi.py,sha256=ke03vqsSYDnO6XeOtOVFXj0-f-v1MGsOxa9McaROvNc,3616
|
|
24
24
|
dbos/_flask.py,sha256=DZKUZR5-xOzPI7tYZ53r2PvvHVoAb8SYwLzMVFsVfjI,2608
|
|
25
25
|
dbos/_kafka.py,sha256=o6DbwnsYRDtvVTZVsN7BAK8cdP79AfoWX3Q7CGY2Yuo,4199
|
|
@@ -51,7 +51,7 @@ dbos/_templates/dbos-db-starter/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-
|
|
|
51
51
|
dbos/_templates/dbos-db-starter/__package/main.py,sha256=eI0SS9Nwj-fldtiuSzIlIG6dC91GXXwdRsoHxv6S_WI,2719
|
|
52
52
|
dbos/_templates/dbos-db-starter/__package/schema.py,sha256=7Z27JGC8yy7Z44cbVXIREYxtUhU4JVkLCp5Q7UahVQ0,260
|
|
53
53
|
dbos/_templates/dbos-db-starter/alembic.ini,sha256=VKBn4Gy8mMuCdY7Hip1jmo3wEUJ1VG1aW7EqY0_n-as,3695
|
|
54
|
-
dbos/_templates/dbos-db-starter/dbos-config.yaml.dbos,sha256=
|
|
54
|
+
dbos/_templates/dbos-db-starter/dbos-config.yaml.dbos,sha256=Z-JC7wp-E9l7NiacjT7E66M812fYFVU3FSS7mNjb6XE,492
|
|
55
55
|
dbos/_templates/dbos-db-starter/migrations/env.py.dbos,sha256=GUV6sjkDzf9Vl6wkGEd0RSkK-ftRfV6EUwSQdd0qFXg,2392
|
|
56
56
|
dbos/_templates/dbos-db-starter/migrations/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvDgl93dMj8,635
|
|
57
57
|
dbos/_templates/dbos-db-starter/migrations/versions/2024_07_31_180642_init.py,sha256=MpS7LGaJS0CpvsjhfDkp9EJqvMvVCjRPfUp4c0aE2ys,941
|
|
@@ -62,7 +62,7 @@ dbos/_workflow_commands.py,sha256=CEzR5XghoZscbc2RHb9G-7Eoo4MMuzfeTo-QBZu4VPY,46
|
|
|
62
62
|
dbos/cli/_github_init.py,sha256=Y_bDF9gfO2jB1id4FV5h1oIxEJRWyqVjhb7bNEa5nQ0,3224
|
|
63
63
|
dbos/cli/_template_init.py,sha256=AfuMaO8bmr9WsPNHr6j2cp7kjVVZDUpH7KpbTg0hhFs,2722
|
|
64
64
|
dbos/cli/cli.py,sha256=ThomRytw7EP5iOcrjEgwnpaWgXNTLfnFEBBvCGHxtJs,15590
|
|
65
|
-
dbos/dbos-config.schema.json,sha256=
|
|
65
|
+
dbos/dbos-config.schema.json,sha256=HtF_njVTGHLdzBGZ4OrGQz3qbPPT0Go-iwd1PgFVTNg,5847
|
|
66
66
|
dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
|
|
67
67
|
version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
|
|
68
|
-
dbos-0.24.
|
|
68
|
+
dbos-0.24.0a6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|