dbos 0.23.0a14__py3-none-any.whl → 0.24.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbos might be problematic. Click here for more details.
- dbos/__init__.py +2 -1
- dbos/_app_db.py +16 -2
- dbos/_conductor/conductor.py +122 -56
- dbos/_conductor/protocol.py +29 -0
- dbos/_core.py +159 -39
- dbos/_db_wizard.py +18 -14
- dbos/_dbos.py +96 -31
- dbos/_dbos_config.py +352 -58
- dbos/_debug.py +7 -1
- dbos/_error.py +9 -3
- dbos/_logger.py +8 -7
- dbos/_queue.py +14 -3
- dbos/_scheduler.py +5 -2
- dbos/_sys_db.py +10 -4
- dbos/_templates/dbos-db-starter/dbos-config.yaml.dbos +2 -4
- dbos/_tracer.py +5 -6
- dbos/cli/cli.py +1 -1
- dbos/dbos-config.schema.json +4 -1
- {dbos-0.23.0a14.dist-info → dbos-0.24.0.dist-info}/METADATA +1 -2
- {dbos-0.23.0a14.dist-info → dbos-0.24.0.dist-info}/RECORD +23 -23
- {dbos-0.23.0a14.dist-info → dbos-0.24.0.dist-info}/WHEEL +0 -0
- {dbos-0.23.0a14.dist-info → dbos-0.24.0.dist-info}/entry_points.txt +0 -0
- {dbos-0.23.0a14.dist-info → dbos-0.24.0.dist-info}/licenses/LICENSE +0 -0
dbos/_dbos_config.py
CHANGED
|
@@ -1,25 +1,61 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
+
import sys
|
|
4
5
|
from importlib import resources
|
|
5
|
-
from typing import Any, Dict, List, Optional, TypedDict, cast
|
|
6
|
+
from typing import Any, Dict, List, Optional, TypedDict, Union, cast
|
|
7
|
+
|
|
8
|
+
if sys.version_info < (3, 10):
|
|
9
|
+
from typing_extensions import TypeGuard
|
|
10
|
+
else:
|
|
11
|
+
from typing import TypeGuard
|
|
6
12
|
|
|
7
13
|
import yaml
|
|
8
14
|
from jsonschema import ValidationError, validate
|
|
9
15
|
from rich import print
|
|
10
|
-
from sqlalchemy import URL
|
|
16
|
+
from sqlalchemy import URL, make_url
|
|
11
17
|
|
|
12
18
|
from ._db_wizard import db_wizard, load_db_connection
|
|
13
19
|
from ._error import DBOSInitializationError
|
|
14
|
-
from ._logger import
|
|
20
|
+
from ._logger import dbos_logger
|
|
15
21
|
|
|
16
22
|
DBOS_CONFIG_PATH = "dbos-config.yaml"
|
|
17
23
|
|
|
18
24
|
|
|
25
|
+
class DBOSConfig(TypedDict, total=False):
|
|
26
|
+
"""
|
|
27
|
+
Data structure containing the DBOS library configuration.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
name (str): Application name
|
|
31
|
+
database_url (str): Database connection string
|
|
32
|
+
app_db_pool_size (int): Application database pool size
|
|
33
|
+
sys_db_name (str): System database name
|
|
34
|
+
sys_db_pool_size (int): System database pool size
|
|
35
|
+
log_level (str): Log level
|
|
36
|
+
otlp_traces_endpoints: List[str]: OTLP traces endpoints
|
|
37
|
+
otlp_logs_endpoints: List[str]: OTLP logs endpoints
|
|
38
|
+
admin_port (int): Admin port
|
|
39
|
+
run_admin_server (bool): Whether to run the DBOS admin server
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
name: str
|
|
43
|
+
database_url: Optional[str]
|
|
44
|
+
app_db_pool_size: Optional[int]
|
|
45
|
+
sys_db_name: Optional[str]
|
|
46
|
+
sys_db_pool_size: Optional[int]
|
|
47
|
+
log_level: Optional[str]
|
|
48
|
+
otlp_traces_endpoints: Optional[List[str]]
|
|
49
|
+
otlp_logs_endpoints: Optional[List[str]]
|
|
50
|
+
admin_port: Optional[int]
|
|
51
|
+
run_admin_server: Optional[bool]
|
|
52
|
+
|
|
53
|
+
|
|
19
54
|
class RuntimeConfig(TypedDict, total=False):
|
|
20
55
|
start: List[str]
|
|
21
56
|
setup: Optional[List[str]]
|
|
22
57
|
admin_port: Optional[int]
|
|
58
|
+
run_admin_server: Optional[bool]
|
|
23
59
|
|
|
24
60
|
|
|
25
61
|
class DatabaseConfig(TypedDict, total=False):
|
|
@@ -29,18 +65,39 @@ class DatabaseConfig(TypedDict, total=False):
|
|
|
29
65
|
password: str
|
|
30
66
|
connectionTimeoutMillis: Optional[int]
|
|
31
67
|
app_db_name: str
|
|
68
|
+
app_db_pool_size: Optional[int]
|
|
32
69
|
sys_db_name: Optional[str]
|
|
70
|
+
sys_db_pool_size: Optional[int]
|
|
33
71
|
ssl: Optional[bool]
|
|
34
72
|
ssl_ca: Optional[str]
|
|
35
73
|
local_suffix: Optional[bool]
|
|
36
|
-
app_db_client: Optional[str]
|
|
37
74
|
migrate: Optional[List[str]]
|
|
38
75
|
rollback: Optional[List[str]]
|
|
39
76
|
|
|
40
77
|
|
|
78
|
+
def parse_database_url_to_dbconfig(database_url: str) -> DatabaseConfig:
|
|
79
|
+
db_url = make_url(database_url)
|
|
80
|
+
db_config = {
|
|
81
|
+
"hostname": db_url.host,
|
|
82
|
+
"port": db_url.port or 5432,
|
|
83
|
+
"username": db_url.username,
|
|
84
|
+
"password": db_url.password,
|
|
85
|
+
"app_db_name": db_url.database,
|
|
86
|
+
}
|
|
87
|
+
for key, value in db_url.query.items():
|
|
88
|
+
str_value = value[0] if isinstance(value, tuple) else value
|
|
89
|
+
if key == "connect_timeout":
|
|
90
|
+
db_config["connectionTimeoutMillis"] = int(str_value) * 1000
|
|
91
|
+
elif key == "sslmode":
|
|
92
|
+
db_config["ssl"] = str_value == "require"
|
|
93
|
+
elif key == "sslrootcert":
|
|
94
|
+
db_config["ssl_ca"] = str_value
|
|
95
|
+
return cast(DatabaseConfig, db_config)
|
|
96
|
+
|
|
97
|
+
|
|
41
98
|
class OTLPExporterConfig(TypedDict, total=False):
|
|
42
|
-
logsEndpoint: Optional[str]
|
|
43
|
-
tracesEndpoint: Optional[str]
|
|
99
|
+
logsEndpoint: Optional[List[str]]
|
|
100
|
+
tracesEndpoint: Optional[List[str]]
|
|
44
101
|
|
|
45
102
|
|
|
46
103
|
class LoggerConfig(TypedDict, total=False):
|
|
@@ -61,9 +118,9 @@ class ConfigFile(TypedDict, total=False):
|
|
|
61
118
|
|
|
62
119
|
Attributes:
|
|
63
120
|
name (str): Application name
|
|
64
|
-
language (str): The app language (probably `python`)
|
|
65
121
|
runtimeConfig (RuntimeConfig): Configuration for request serving
|
|
66
122
|
database (DatabaseConfig): Configuration for the application and system databases
|
|
123
|
+
database_url (str): Database connection string
|
|
67
124
|
telemetry (TelemetryConfig): Configuration for tracing / logging
|
|
68
125
|
env (Dict[str,str]): Environment varialbes
|
|
69
126
|
application (Dict[str, Any]): Application-specific configuration section
|
|
@@ -71,15 +128,93 @@ class ConfigFile(TypedDict, total=False):
|
|
|
71
128
|
"""
|
|
72
129
|
|
|
73
130
|
name: str
|
|
74
|
-
language: str
|
|
75
131
|
runtimeConfig: RuntimeConfig
|
|
76
132
|
database: DatabaseConfig
|
|
133
|
+
database_url: Optional[str]
|
|
77
134
|
telemetry: Optional[TelemetryConfig]
|
|
78
135
|
env: Dict[str, str]
|
|
79
|
-
application: Dict[str, Any]
|
|
80
136
|
|
|
81
137
|
|
|
82
|
-
def
|
|
138
|
+
def is_dbos_configfile(data: Union[ConfigFile, DBOSConfig]) -> TypeGuard[DBOSConfig]:
|
|
139
|
+
"""
|
|
140
|
+
Type guard to check if the provided data is a DBOSConfig.
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
data: The configuration object to check
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
True if the data is a DBOSConfig, False otherwise
|
|
147
|
+
"""
|
|
148
|
+
return (
|
|
149
|
+
isinstance(data, dict)
|
|
150
|
+
and "name" in data
|
|
151
|
+
and (
|
|
152
|
+
"runtimeConfig" in data
|
|
153
|
+
or "database" in data
|
|
154
|
+
or "env" in data
|
|
155
|
+
or "telemetry" in data
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def translate_dbos_config_to_config_file(config: DBOSConfig) -> ConfigFile:
|
|
161
|
+
if "name" not in config:
|
|
162
|
+
raise DBOSInitializationError(f"Configuration must specify an application name")
|
|
163
|
+
|
|
164
|
+
translated_config: ConfigFile = {
|
|
165
|
+
"name": config["name"],
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
# Database config
|
|
169
|
+
db_config: DatabaseConfig = {}
|
|
170
|
+
database_url = config.get("database_url")
|
|
171
|
+
if database_url:
|
|
172
|
+
db_config = parse_database_url_to_dbconfig(database_url)
|
|
173
|
+
if "sys_db_name" in config:
|
|
174
|
+
db_config["sys_db_name"] = config.get("sys_db_name")
|
|
175
|
+
if "app_db_pool_size" in config:
|
|
176
|
+
db_config["app_db_pool_size"] = config.get("app_db_pool_size")
|
|
177
|
+
if "sys_db_pool_size" in config:
|
|
178
|
+
db_config["sys_db_pool_size"] = config.get("sys_db_pool_size")
|
|
179
|
+
if db_config:
|
|
180
|
+
translated_config["database"] = db_config
|
|
181
|
+
|
|
182
|
+
# Runtime config
|
|
183
|
+
translated_config["runtimeConfig"] = {"run_admin_server": True}
|
|
184
|
+
if "admin_port" in config:
|
|
185
|
+
translated_config["runtimeConfig"]["admin_port"] = config["admin_port"]
|
|
186
|
+
if "run_admin_server" in config:
|
|
187
|
+
translated_config["runtimeConfig"]["run_admin_server"] = config[
|
|
188
|
+
"run_admin_server"
|
|
189
|
+
]
|
|
190
|
+
|
|
191
|
+
# Telemetry config
|
|
192
|
+
telemetry: TelemetryConfig = {
|
|
193
|
+
"OTLPExporter": {"tracesEndpoint": [], "logsEndpoint": []}
|
|
194
|
+
}
|
|
195
|
+
# For mypy
|
|
196
|
+
assert telemetry["OTLPExporter"] is not None
|
|
197
|
+
|
|
198
|
+
# Add OTLPExporter if traces endpoints exist
|
|
199
|
+
otlp_trace_endpoints = config.get("otlp_traces_endpoints")
|
|
200
|
+
if isinstance(otlp_trace_endpoints, list) and len(otlp_trace_endpoints) > 0:
|
|
201
|
+
telemetry["OTLPExporter"]["tracesEndpoint"] = otlp_trace_endpoints
|
|
202
|
+
# Same for the logs
|
|
203
|
+
otlp_logs_endpoints = config.get("otlp_logs_endpoints")
|
|
204
|
+
if isinstance(otlp_logs_endpoints, list) and len(otlp_logs_endpoints) > 0:
|
|
205
|
+
telemetry["OTLPExporter"]["logsEndpoint"] = otlp_logs_endpoints
|
|
206
|
+
|
|
207
|
+
# Default to INFO -- the logging seems to default to WARN otherwise.
|
|
208
|
+
log_level = config.get("log_level", "INFO")
|
|
209
|
+
if log_level:
|
|
210
|
+
telemetry["logs"] = {"logLevel": log_level}
|
|
211
|
+
if telemetry:
|
|
212
|
+
translated_config["telemetry"] = telemetry
|
|
213
|
+
|
|
214
|
+
return translated_config
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _substitute_env_vars(content: str, silent: bool = False) -> str:
|
|
83
218
|
regex = r"\$\{([^}]+)\}" # Regex to match ${VAR_NAME} style placeholders
|
|
84
219
|
|
|
85
220
|
def replace_func(match: re.Match[str]) -> str:
|
|
@@ -87,7 +222,7 @@ def _substitute_env_vars(content: str) -> str:
|
|
|
87
222
|
value = os.environ.get(
|
|
88
223
|
var_name, ""
|
|
89
224
|
) # If the env variable is not set, return an empty string
|
|
90
|
-
if value == "":
|
|
225
|
+
if value == "" and not silent:
|
|
91
226
|
dbos_logger.warning(
|
|
92
227
|
f"Variable {var_name} would be substituted from the process environment into dbos-config.yaml, but is not defined"
|
|
93
228
|
)
|
|
@@ -110,7 +245,7 @@ def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
|
|
|
110
245
|
str: Database URL for the application database
|
|
111
246
|
|
|
112
247
|
"""
|
|
113
|
-
dbos_config = load_config(config_file_path)
|
|
248
|
+
dbos_config = load_config(config_file_path, run_process_config=True)
|
|
114
249
|
db_url = URL.create(
|
|
115
250
|
"postgresql+psycopg",
|
|
116
251
|
username=dbos_config["database"]["username"],
|
|
@@ -125,6 +260,7 @@ def get_dbos_database_url(config_file_path: str = DBOS_CONFIG_PATH) -> str:
|
|
|
125
260
|
def load_config(
|
|
126
261
|
config_file_path: str = DBOS_CONFIG_PATH,
|
|
127
262
|
*,
|
|
263
|
+
run_process_config: bool = True,
|
|
128
264
|
use_db_wizard: bool = True,
|
|
129
265
|
silent: bool = False,
|
|
130
266
|
) -> ConfigFile:
|
|
@@ -141,13 +277,17 @@ def load_config(
|
|
|
141
277
|
|
|
142
278
|
"""
|
|
143
279
|
|
|
144
|
-
init_logger()
|
|
145
|
-
|
|
146
280
|
with open(config_file_path, "r") as file:
|
|
147
281
|
content = file.read()
|
|
148
|
-
substituted_content = _substitute_env_vars(content)
|
|
282
|
+
substituted_content = _substitute_env_vars(content, silent=silent)
|
|
149
283
|
data = yaml.safe_load(substituted_content)
|
|
150
284
|
|
|
285
|
+
if not isinstance(data, dict):
|
|
286
|
+
raise DBOSInitializationError(
|
|
287
|
+
f"dbos-config.yaml must contain a dictionary, not {type(data)}"
|
|
288
|
+
)
|
|
289
|
+
data = cast(Dict[str, Any], data)
|
|
290
|
+
|
|
151
291
|
# Load the JSON schema relative to the package root
|
|
152
292
|
schema_file = resources.files("dbos").joinpath("dbos-config.schema.json")
|
|
153
293
|
with schema_file.open("r") as f:
|
|
@@ -159,55 +299,69 @@ def load_config(
|
|
|
159
299
|
except ValidationError as e:
|
|
160
300
|
raise DBOSInitializationError(f"Validation error: {e}")
|
|
161
301
|
|
|
162
|
-
|
|
163
|
-
|
|
302
|
+
# Special case: convert logsEndpoint and tracesEndpoint from strings to lists of strings, if present
|
|
303
|
+
if "telemetry" in data and "OTLPExporter" in data["telemetry"]:
|
|
304
|
+
if "logsEndpoint" in data["telemetry"]["OTLPExporter"]:
|
|
305
|
+
data["telemetry"]["OTLPExporter"]["logsEndpoint"] = [
|
|
306
|
+
data["telemetry"]["OTLPExporter"]["logsEndpoint"]
|
|
307
|
+
]
|
|
308
|
+
if "tracesEndpoint" in data["telemetry"]["OTLPExporter"]:
|
|
309
|
+
data["telemetry"]["OTLPExporter"]["tracesEndpoint"] = [
|
|
310
|
+
data["telemetry"]["OTLPExporter"]["tracesEndpoint"]
|
|
311
|
+
]
|
|
164
312
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
if "language" not in data:
|
|
171
|
-
raise DBOSInitializationError(
|
|
172
|
-
f"dbos-config.yaml must specify the application language is Python"
|
|
173
|
-
)
|
|
313
|
+
data = cast(ConfigFile, data)
|
|
314
|
+
if run_process_config:
|
|
315
|
+
data = process_config(data=data, use_db_wizard=use_db_wizard, silent=silent)
|
|
316
|
+
return data # type: ignore
|
|
174
317
|
|
|
175
|
-
if data["language"] != "python":
|
|
176
|
-
raise DBOSInitializationError(
|
|
177
|
-
f'dbos-config.yaml specifies invalid language { data["language"] }'
|
|
178
|
-
)
|
|
179
318
|
|
|
180
|
-
|
|
181
|
-
|
|
319
|
+
def process_config(
|
|
320
|
+
*,
|
|
321
|
+
use_db_wizard: bool = True,
|
|
322
|
+
data: ConfigFile,
|
|
323
|
+
silent: bool = False,
|
|
324
|
+
) -> ConfigFile:
|
|
325
|
+
if "name" not in data:
|
|
326
|
+
raise DBOSInitializationError(f"Configuration must specify an application name")
|
|
182
327
|
|
|
183
328
|
if not _is_valid_app_name(data["name"]):
|
|
184
329
|
raise DBOSInitializationError(
|
|
185
330
|
f'Invalid app name {data["name"]}. App names must be between 3 and 30 characters long and contain only lowercase letters, numbers, dashes, and underscores.'
|
|
186
331
|
)
|
|
187
332
|
|
|
188
|
-
if "
|
|
333
|
+
if data.get("telemetry") is None:
|
|
334
|
+
data["telemetry"] = {}
|
|
335
|
+
telemetry = cast(TelemetryConfig, data["telemetry"])
|
|
336
|
+
if telemetry.get("logs") is None:
|
|
337
|
+
telemetry["logs"] = {}
|
|
338
|
+
logs = cast(LoggerConfig, telemetry["logs"])
|
|
339
|
+
if logs.get("logLevel") is None:
|
|
340
|
+
logs["logLevel"] = "INFO"
|
|
341
|
+
|
|
342
|
+
if "database" not in data:
|
|
343
|
+
data["database"] = {}
|
|
344
|
+
|
|
345
|
+
# database_url takes precedence over database config, but we need to preserve rollback and migrate if they exist
|
|
346
|
+
migrate = data["database"].get("migrate", False)
|
|
347
|
+
rollback = data["database"].get("rollback", False)
|
|
348
|
+
local_suffix = data["database"].get("local_suffix", False)
|
|
349
|
+
if data.get("database_url"):
|
|
350
|
+
dbconfig = parse_database_url_to_dbconfig(cast(str, data["database_url"]))
|
|
351
|
+
if migrate:
|
|
352
|
+
dbconfig["migrate"] = cast(List[str], migrate)
|
|
353
|
+
if rollback:
|
|
354
|
+
dbconfig["rollback"] = cast(List[str], rollback)
|
|
355
|
+
if local_suffix:
|
|
356
|
+
dbconfig["local_suffix"] = cast(bool, local_suffix)
|
|
357
|
+
data["database"] = dbconfig
|
|
358
|
+
|
|
359
|
+
if "app_db_name" not in data["database"] or not (data["database"]["app_db_name"]):
|
|
189
360
|
data["database"]["app_db_name"] = _app_name_to_db_name(data["name"])
|
|
190
361
|
|
|
191
|
-
# Load the DB connection file. Use its values for missing
|
|
192
|
-
data = cast(ConfigFile, data)
|
|
362
|
+
# Load the DB connection file. Use its values for missing connection parameters. Use defaults otherwise.
|
|
193
363
|
db_connection = load_db_connection()
|
|
194
|
-
|
|
195
|
-
if os.getenv("DBOS_DBHOST"):
|
|
196
|
-
print(
|
|
197
|
-
"[bold blue]Loading database connection parameters from debug environment variables[/bold blue]"
|
|
198
|
-
)
|
|
199
|
-
elif data["database"].get("hostname"):
|
|
200
|
-
print(
|
|
201
|
-
"[bold blue]Loading database connection parameters from dbos-config.yaml[/bold blue]"
|
|
202
|
-
)
|
|
203
|
-
elif db_connection.get("hostname"):
|
|
204
|
-
print(
|
|
205
|
-
"[bold blue]Loading database connection parameters from .dbos/db_connection[/bold blue]"
|
|
206
|
-
)
|
|
207
|
-
else:
|
|
208
|
-
print(
|
|
209
|
-
"[bold blue]Using default database connection parameters (localhost)[/bold blue]"
|
|
210
|
-
)
|
|
364
|
+
connection_passed_in = data["database"].get("hostname", None) is not None
|
|
211
365
|
|
|
212
366
|
dbos_dbport: Optional[int] = None
|
|
213
367
|
dbport_env = os.getenv("DBOS_DBPORT")
|
|
@@ -252,26 +406,60 @@ def load_config(
|
|
|
252
406
|
dbcon_local_suffix = db_connection.get("local_suffix")
|
|
253
407
|
if dbcon_local_suffix is not None:
|
|
254
408
|
local_suffix = dbcon_local_suffix
|
|
255
|
-
|
|
256
|
-
|
|
409
|
+
db_local_suffix = data["database"].get("local_suffix")
|
|
410
|
+
if db_local_suffix is not None:
|
|
411
|
+
local_suffix = db_local_suffix
|
|
257
412
|
if dbos_dblocalsuffix is not None:
|
|
258
413
|
local_suffix = dbos_dblocalsuffix
|
|
259
414
|
data["database"]["local_suffix"] = local_suffix
|
|
260
415
|
|
|
261
|
-
|
|
262
|
-
|
|
416
|
+
if not data["database"].get("app_db_pool_size"):
|
|
417
|
+
data["database"]["app_db_pool_size"] = 20
|
|
418
|
+
if not data["database"].get("sys_db_pool_size"):
|
|
419
|
+
data["database"]["sys_db_pool_size"] = 20
|
|
420
|
+
if not data["database"].get("connectionTimeoutMillis"):
|
|
421
|
+
data["database"]["connectionTimeoutMillis"] = 10000
|
|
422
|
+
|
|
423
|
+
if not data.get("runtimeConfig"):
|
|
424
|
+
data["runtimeConfig"] = {
|
|
425
|
+
"run_admin_server": True,
|
|
426
|
+
}
|
|
427
|
+
elif "run_admin_server" not in data["runtimeConfig"]:
|
|
428
|
+
data["runtimeConfig"]["run_admin_server"] = True
|
|
263
429
|
|
|
264
430
|
# Check the connectivity to the database and make sure it's properly configured
|
|
265
431
|
# Note, never use db wizard if the DBOS is running in debug mode (i.e. DBOS_DEBUG_WORKFLOW_ID env var is set)
|
|
266
432
|
debugWorkflowId = os.getenv("DBOS_DEBUG_WORKFLOW_ID")
|
|
433
|
+
|
|
434
|
+
# Pretty-print where we've loaded database connection information from, respecting the log level
|
|
435
|
+
if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
|
|
436
|
+
d = data["database"]
|
|
437
|
+
conn_string = f"postgresql://{d['username']}:*****@{d['hostname']}:{d['port']}/{d['app_db_name']}"
|
|
438
|
+
if os.getenv("DBOS_DBHOST"):
|
|
439
|
+
print(
|
|
440
|
+
f"[bold blue]Loading database connection string from debug environment variables: {conn_string}[/bold blue]"
|
|
441
|
+
)
|
|
442
|
+
elif connection_passed_in:
|
|
443
|
+
print(
|
|
444
|
+
f"[bold blue]Using database connection string: {conn_string}[/bold blue]"
|
|
445
|
+
)
|
|
446
|
+
elif db_connection.get("hostname"):
|
|
447
|
+
print(
|
|
448
|
+
f"[bold blue]Loading database connection string from .dbos/db_connection: {conn_string}[/bold blue]"
|
|
449
|
+
)
|
|
450
|
+
else:
|
|
451
|
+
print(
|
|
452
|
+
f"[bold blue]Using default database connection string: {conn_string}[/bold blue]"
|
|
453
|
+
)
|
|
454
|
+
|
|
267
455
|
if use_db_wizard and debugWorkflowId is None:
|
|
268
|
-
data = db_wizard(data
|
|
456
|
+
data = db_wizard(data)
|
|
269
457
|
|
|
270
458
|
if "local_suffix" in data["database"] and data["database"]["local_suffix"]:
|
|
271
459
|
data["database"]["app_db_name"] = f"{data['database']['app_db_name']}_local"
|
|
272
460
|
|
|
273
461
|
# Return data as ConfigFile type
|
|
274
|
-
return data
|
|
462
|
+
return data
|
|
275
463
|
|
|
276
464
|
|
|
277
465
|
def _is_valid_app_name(name: str) -> bool:
|
|
@@ -291,3 +479,109 @@ def set_env_vars(config: ConfigFile) -> None:
|
|
|
291
479
|
for env, value in config.get("env", {}).items():
|
|
292
480
|
if value is not None:
|
|
293
481
|
os.environ[env] = str(value)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def overwrite_config(provided_config: ConfigFile) -> ConfigFile:
|
|
485
|
+
# Load the DBOS configuration file and force the use of:
|
|
486
|
+
# 1. The database connection parameters (sub the file data to the provided config)
|
|
487
|
+
# 2. OTLP traces endpoints (add the config data to the provided config)
|
|
488
|
+
# 3. Use the application name from the file. This is a defensive measure to ensure the application name is whatever it was registered with in the cloud
|
|
489
|
+
# 4. Remove admin_port is provided in code
|
|
490
|
+
# 5. Remove env vars if provided in code
|
|
491
|
+
# Optimistically assume that expected fields in config_from_file are present
|
|
492
|
+
|
|
493
|
+
config_from_file = load_config(run_process_config=False)
|
|
494
|
+
# Be defensive
|
|
495
|
+
if config_from_file is None:
|
|
496
|
+
return provided_config
|
|
497
|
+
|
|
498
|
+
# Name
|
|
499
|
+
provided_config["name"] = config_from_file["name"]
|
|
500
|
+
|
|
501
|
+
# Database config. Note we disregard a potential database_url in config_from_file because it is not expected from DBOS Cloud
|
|
502
|
+
if "database" not in provided_config:
|
|
503
|
+
provided_config["database"] = {}
|
|
504
|
+
provided_config["database"]["hostname"] = config_from_file["database"]["hostname"]
|
|
505
|
+
provided_config["database"]["port"] = config_from_file["database"]["port"]
|
|
506
|
+
provided_config["database"]["username"] = config_from_file["database"]["username"]
|
|
507
|
+
provided_config["database"]["password"] = config_from_file["database"]["password"]
|
|
508
|
+
provided_config["database"]["app_db_name"] = config_from_file["database"][
|
|
509
|
+
"app_db_name"
|
|
510
|
+
]
|
|
511
|
+
provided_config["database"]["sys_db_name"] = config_from_file["database"][
|
|
512
|
+
"sys_db_name"
|
|
513
|
+
]
|
|
514
|
+
provided_config["database"]["ssl"] = config_from_file["database"]["ssl"]
|
|
515
|
+
provided_config["database"]["ssl_ca"] = config_from_file["database"]["ssl_ca"]
|
|
516
|
+
|
|
517
|
+
# Telemetry config
|
|
518
|
+
if "telemetry" not in provided_config or provided_config["telemetry"] is None:
|
|
519
|
+
provided_config["telemetry"] = {
|
|
520
|
+
"OTLPExporter": {"tracesEndpoint": [], "logsEndpoint": []},
|
|
521
|
+
}
|
|
522
|
+
elif "OTLPExporter" not in provided_config["telemetry"]:
|
|
523
|
+
provided_config["telemetry"]["OTLPExporter"] = {
|
|
524
|
+
"tracesEndpoint": [],
|
|
525
|
+
"logsEndpoint": [],
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
# This is a super messy from a typing perspective.
|
|
529
|
+
# Some of ConfigFile keys are optional -- but in practice they'll always be present in hosted environments
|
|
530
|
+
# So, for Mypy, we have to (1) check the keys are present in config_from_file and (2) cast telemetry/otlp_exporters to Dict[str, Any]
|
|
531
|
+
# (2) is required because, even tho we resolved these keys earlier, mypy doesn't remember that
|
|
532
|
+
if (
|
|
533
|
+
config_from_file.get("telemetry")
|
|
534
|
+
and config_from_file["telemetry"]
|
|
535
|
+
and config_from_file["telemetry"].get("OTLPExporter")
|
|
536
|
+
):
|
|
537
|
+
|
|
538
|
+
telemetry = cast(Dict[str, Any], provided_config["telemetry"])
|
|
539
|
+
otlp_exporter = cast(Dict[str, Any], telemetry["OTLPExporter"])
|
|
540
|
+
|
|
541
|
+
# Merge the logsEndpoint and tracesEndpoint lists from the file with what we have
|
|
542
|
+
source_otlp = config_from_file["telemetry"]["OTLPExporter"]
|
|
543
|
+
if source_otlp:
|
|
544
|
+
tracesEndpoint = source_otlp.get("tracesEndpoint")
|
|
545
|
+
if tracesEndpoint:
|
|
546
|
+
otlp_exporter["tracesEndpoint"].extend(tracesEndpoint)
|
|
547
|
+
logsEndpoint = source_otlp.get("logsEndpoint")
|
|
548
|
+
if logsEndpoint:
|
|
549
|
+
otlp_exporter["logsEndpoint"].extend(logsEndpoint)
|
|
550
|
+
|
|
551
|
+
# Runtime config
|
|
552
|
+
if "runtimeConfig" in provided_config:
|
|
553
|
+
if "admin_port" in provided_config["runtimeConfig"]:
|
|
554
|
+
del provided_config["runtimeConfig"][
|
|
555
|
+
"admin_port"
|
|
556
|
+
] # Admin port is expected to be 3001 (the default in dbos/_admin_server.py::__init__ ) by DBOS Cloud
|
|
557
|
+
if "run_admin_server" in provided_config["runtimeConfig"]:
|
|
558
|
+
del provided_config["runtimeConfig"]["run_admin_server"]
|
|
559
|
+
|
|
560
|
+
# Env should be set from the hosting provider (e.g., DBOS Cloud)
|
|
561
|
+
if "env" in provided_config:
|
|
562
|
+
del provided_config["env"]
|
|
563
|
+
|
|
564
|
+
return provided_config
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def check_config_consistency(
|
|
568
|
+
*,
|
|
569
|
+
name: str,
|
|
570
|
+
config_file_path: str = DBOS_CONFIG_PATH,
|
|
571
|
+
) -> None:
|
|
572
|
+
# First load the config file and check whether it is present
|
|
573
|
+
try:
|
|
574
|
+
config = load_config(config_file_path, silent=True, run_process_config=False)
|
|
575
|
+
except FileNotFoundError:
|
|
576
|
+
dbos_logger.debug(
|
|
577
|
+
f"No configuration file {config_file_path} found. Skipping consistency check with provided config."
|
|
578
|
+
)
|
|
579
|
+
return
|
|
580
|
+
except Exception as e:
|
|
581
|
+
raise e
|
|
582
|
+
|
|
583
|
+
# Check the name
|
|
584
|
+
if name != config["name"]:
|
|
585
|
+
raise DBOSInitializationError(
|
|
586
|
+
f"Provided app name '{name}' does not match the app name '{config['name']}' in {config_file_path}."
|
|
587
|
+
)
|
dbos/_debug.py
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import re
|
|
2
2
|
import runpy
|
|
3
3
|
import sys
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
from typing import Union
|
|
5
6
|
|
|
7
|
+
from fastapi_cli.discover import get_module_data_from_path
|
|
8
|
+
|
|
6
9
|
from dbos import DBOS
|
|
7
10
|
|
|
8
11
|
|
|
@@ -33,7 +36,10 @@ def debug_workflow(workflow_id: str, entrypoint: Union[str, PythonModule]) -> No
|
|
|
33
36
|
def parse_start_command(command: str) -> Union[str, PythonModule]:
|
|
34
37
|
match = re.match(r"fastapi\s+run\s+(\.?[\w/]+\.py)", command)
|
|
35
38
|
if match:
|
|
36
|
-
|
|
39
|
+
# Mirror the logic in fastapi's run command by converting the path argument to a module
|
|
40
|
+
mod_data = get_module_data_from_path(Path(match.group(1)))
|
|
41
|
+
sys.path.insert(0, str(mod_data.extra_sys_path))
|
|
42
|
+
return PythonModule(mod_data.module_import_str)
|
|
37
43
|
match = re.match(r"python3?\s+(\.?[\w/]+\.py)", command)
|
|
38
44
|
if match:
|
|
39
45
|
return match.group(1)
|
dbos/_error.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Errors thrown by DBOS."""
|
|
2
2
|
|
|
3
3
|
from enum import Enum
|
|
4
|
-
from typing import Optional
|
|
4
|
+
from typing import Any, Optional
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class DBOSException(Exception):
|
|
@@ -124,12 +124,18 @@ class DBOSNotAuthorizedError(DBOSException):
|
|
|
124
124
|
class DBOSMaxStepRetriesExceeded(DBOSException):
|
|
125
125
|
"""Exception raised when a step was retried the maximimum number of times without success."""
|
|
126
126
|
|
|
127
|
-
def __init__(self) -> None:
|
|
127
|
+
def __init__(self, step_name: str, max_retries: int) -> None:
|
|
128
|
+
self.step_name = step_name
|
|
129
|
+
self.max_retries = max_retries
|
|
128
130
|
super().__init__(
|
|
129
|
-
"Step
|
|
131
|
+
f"Step {step_name} has exceeded its maximum of {max_retries} retries",
|
|
130
132
|
dbos_error_code=DBOSErrorCode.MaxStepRetriesExceeded.value,
|
|
131
133
|
)
|
|
132
134
|
|
|
135
|
+
def __reduce__(self) -> Any:
|
|
136
|
+
# Tell jsonpickle how to reconstruct this object
|
|
137
|
+
return (self.__class__, (self.step_name, self.max_retries))
|
|
138
|
+
|
|
133
139
|
|
|
134
140
|
class DBOSWorkflowCancelledError(DBOSException):
|
|
135
141
|
"""Exception raised when the workflow has already been cancelled."""
|
dbos/_logger.py
CHANGED
|
@@ -56,10 +56,10 @@ def config_logger(config: "ConfigFile") -> None:
|
|
|
56
56
|
dbos_logger.setLevel(log_level)
|
|
57
57
|
|
|
58
58
|
# Log to the OTLP endpoint if provided
|
|
59
|
-
|
|
59
|
+
otlp_logs_endpoints = (
|
|
60
60
|
config.get("telemetry", {}).get("OTLPExporter", {}).get("logsEndpoint") # type: ignore
|
|
61
61
|
)
|
|
62
|
-
if
|
|
62
|
+
if otlp_logs_endpoints:
|
|
63
63
|
log_provider = PatchedOTLPLoggerProvider(
|
|
64
64
|
Resource.create(
|
|
65
65
|
attributes={
|
|
@@ -68,12 +68,13 @@ def config_logger(config: "ConfigFile") -> None:
|
|
|
68
68
|
)
|
|
69
69
|
)
|
|
70
70
|
set_logger_provider(log_provider)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
71
|
+
for e in otlp_logs_endpoints:
|
|
72
|
+
log_provider.add_log_record_processor(
|
|
73
|
+
BatchLogRecordProcessor(
|
|
74
|
+
OTLPLogExporter(endpoint=e),
|
|
75
|
+
export_timeout_millis=5000,
|
|
76
|
+
)
|
|
75
77
|
)
|
|
76
|
-
)
|
|
77
78
|
global _otlp_handler
|
|
78
79
|
_otlp_handler = LoggingHandler(logger_provider=log_provider)
|
|
79
80
|
|
dbos/_queue.py
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import threading
|
|
2
2
|
import traceback
|
|
3
|
-
from typing import TYPE_CHECKING, Optional, TypedDict
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Coroutine, Optional, TypedDict
|
|
4
4
|
|
|
5
5
|
from psycopg import errors
|
|
6
6
|
from sqlalchemy.exc import OperationalError
|
|
7
7
|
|
|
8
8
|
from dbos._utils import GlobalParams
|
|
9
9
|
|
|
10
|
-
from ._core import P, R, execute_workflow_by_id, start_workflow
|
|
10
|
+
from ._core import P, R, execute_workflow_by_id, start_workflow, start_workflow_async
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
|
-
from ._dbos import DBOS, Workflow, WorkflowHandle
|
|
13
|
+
from ._dbos import DBOS, Workflow, WorkflowHandle, WorkflowHandleAsync
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class QueueRateLimit(TypedDict):
|
|
@@ -66,6 +66,17 @@ class Queue:
|
|
|
66
66
|
dbos = _get_dbos_instance()
|
|
67
67
|
return start_workflow(dbos, func, self.name, False, *args, **kwargs)
|
|
68
68
|
|
|
69
|
+
async def enqueue_async(
|
|
70
|
+
self,
|
|
71
|
+
func: "Workflow[P, Coroutine[Any, Any, R]]",
|
|
72
|
+
*args: P.args,
|
|
73
|
+
**kwargs: P.kwargs,
|
|
74
|
+
) -> "WorkflowHandleAsync[R]":
|
|
75
|
+
from ._dbos import _get_dbos_instance
|
|
76
|
+
|
|
77
|
+
dbos = _get_dbos_instance()
|
|
78
|
+
return await start_workflow_async(dbos, func, self.name, False, *args, **kwargs)
|
|
79
|
+
|
|
69
80
|
|
|
70
81
|
def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
|
|
71
82
|
while not stop_event.is_set():
|