dbos 1.15.0a1__tar.gz → 1.15.0a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbos might be problematic. Click here for more details.
- {dbos-1.15.0a1 → dbos-1.15.0a2}/PKG-INFO +1 -1
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_client.py +17 -21
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_core.py +3 -1
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_dbos.py +10 -14
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_dbos_config.py +15 -13
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_workflow_commands.py +9 -5
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/cli/cli.py +7 -5
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/cli/migration.py +15 -10
- {dbos-1.15.0a1 → dbos-1.15.0a2}/pyproject.toml +1 -1
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/script_without_fastapi.py +6 -1
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_config.py +9 -9
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_dbos.py +45 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_package.py +4 -4
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_scheduler.py +1 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_schema_migration.py +87 -59
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_workflow_management.py +1 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/LICENSE +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/README.md +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/__init__.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/__main__.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_admin_server.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_app_db.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_classproperty.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_conductor/conductor.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_conductor/protocol.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_context.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_croniter.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_debouncer.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_debug.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_docker_pg_helper.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_error.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_event_loop.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_fastapi.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_flask.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_kafka.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_kafka_message.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_logger.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_migration.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_outcome.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_queue.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_recovery.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_registrations.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_roles.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_scheduler.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_schemas/__init__.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_schemas/application_database.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_schemas/system_database.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_serialization.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_sys_db.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_sys_db_postgres.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_sys_db_sqlite.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/README.md +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/__package/__init__.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/__package/main.py.dbos +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/__package/schema.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/dbos-config.yaml.dbos +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/migrations/create_table.py.dbos +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/start_postgres_docker.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_tracer.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_utils.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/cli/_github_init.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/cli/_template_init.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/dbos-config.schema.json +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/py.typed +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/__init__.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/atexit_no_ctor.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/atexit_no_launch.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/classdefs.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/client_collateral.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/client_worker.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/conftest.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/dupname_classdefs1.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/dupname_classdefsa.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/more_classdefs.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/queuedworkflow.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_admin_server.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_async.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_async_workflow_management.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_classdecorators.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_cli.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_client.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_concurrency.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_croniter.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_debouncer.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_debug.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_docker_secrets.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_failures.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_fastapi.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_fastapi_roles.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_flask.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_kafka.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_outcome.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_queue.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_singleton.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_spans.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_sqlalchemy.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_streaming.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/tests/test_workflow_introspection.py +0 -0
- {dbos-1.15.0a1 → dbos-1.15.0a2}/version/__init__.py +0 -0
|
@@ -22,11 +22,7 @@ from dbos._sys_db import SystemDatabase
|
|
|
22
22
|
if TYPE_CHECKING:
|
|
23
23
|
from dbos._dbos import WorkflowHandle, WorkflowHandleAsync
|
|
24
24
|
|
|
25
|
-
from dbos._dbos_config import
|
|
26
|
-
get_application_database_url,
|
|
27
|
-
get_system_database_url,
|
|
28
|
-
is_valid_database_url,
|
|
29
|
-
)
|
|
25
|
+
from dbos._dbos_config import get_system_database_url, is_valid_database_url
|
|
30
26
|
from dbos._error import DBOSException, DBOSNonExistentWorkflowError
|
|
31
27
|
from dbos._registrations import DEFAULT_MAX_RECOVERY_ATTEMPTS
|
|
32
28
|
from dbos._serialization import WorkflowInputs
|
|
@@ -118,6 +114,9 @@ class WorkflowHandleClientAsyncPolling(Generic[R]):
|
|
|
118
114
|
|
|
119
115
|
|
|
120
116
|
class DBOSClient:
|
|
117
|
+
|
|
118
|
+
_app_db: ApplicationDatabase | None = None
|
|
119
|
+
|
|
121
120
|
def __init__(
|
|
122
121
|
self,
|
|
123
122
|
database_url: Optional[str] = None, # DEPRECATED
|
|
@@ -126,13 +125,8 @@ class DBOSClient:
|
|
|
126
125
|
application_database_url: Optional[str] = None,
|
|
127
126
|
system_database: Optional[str] = None, # DEPRECATED
|
|
128
127
|
):
|
|
129
|
-
application_database_url =
|
|
130
|
-
|
|
131
|
-
"system_database_url": system_database_url,
|
|
132
|
-
"database_url": (
|
|
133
|
-
database_url if database_url else application_database_url
|
|
134
|
-
),
|
|
135
|
-
}
|
|
128
|
+
application_database_url = (
|
|
129
|
+
database_url if database_url else application_database_url
|
|
136
130
|
)
|
|
137
131
|
system_database_url = get_system_database_url(
|
|
138
132
|
{
|
|
@@ -142,7 +136,8 @@ class DBOSClient:
|
|
|
142
136
|
}
|
|
143
137
|
)
|
|
144
138
|
assert is_valid_database_url(system_database_url)
|
|
145
|
-
|
|
139
|
+
if application_database_url:
|
|
140
|
+
assert is_valid_database_url(application_database_url)
|
|
146
141
|
# We only create database connections but do not run migrations
|
|
147
142
|
self._sys_db = SystemDatabase.create(
|
|
148
143
|
system_database_url=system_database_url,
|
|
@@ -153,14 +148,15 @@ class DBOSClient:
|
|
|
153
148
|
},
|
|
154
149
|
)
|
|
155
150
|
self._sys_db.check_connection()
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
151
|
+
if application_database_url:
|
|
152
|
+
self._app_db = ApplicationDatabase.create(
|
|
153
|
+
database_url=application_database_url,
|
|
154
|
+
engine_kwargs={
|
|
155
|
+
"pool_timeout": 30,
|
|
156
|
+
"max_overflow": 0,
|
|
157
|
+
"pool_size": 2,
|
|
158
|
+
},
|
|
159
|
+
)
|
|
164
160
|
|
|
165
161
|
def destroy(self) -> None:
|
|
166
162
|
self._sys_db.destroy()
|
|
@@ -896,7 +896,9 @@ def decorate_transaction(
|
|
|
896
896
|
raise DBOSWorkflowCancelledError(
|
|
897
897
|
f"Workflow {ctx.workflow_id} is cancelled. Aborting transaction {transaction_name}."
|
|
898
898
|
)
|
|
899
|
-
|
|
899
|
+
assert (
|
|
900
|
+
dbos._app_db
|
|
901
|
+
), "Transactions can only be used if DBOS is configured with an application_database_url"
|
|
900
902
|
with dbos._app_db.sessionmaker() as session:
|
|
901
903
|
attributes: TracedAttributes = {
|
|
902
904
|
"name": transaction_name,
|
|
@@ -409,13 +409,8 @@ class DBOS:
|
|
|
409
409
|
return rv
|
|
410
410
|
|
|
411
411
|
@property
|
|
412
|
-
def _app_db(self) -> ApplicationDatabase:
|
|
413
|
-
|
|
414
|
-
raise DBOSException(
|
|
415
|
-
"Application database accessed before DBOS was launched"
|
|
416
|
-
)
|
|
417
|
-
rv: ApplicationDatabase = self._app_db_field
|
|
418
|
-
return rv
|
|
412
|
+
def _app_db(self) -> ApplicationDatabase | None:
|
|
413
|
+
return self._app_db_field
|
|
419
414
|
|
|
420
415
|
@property
|
|
421
416
|
def _admin_server(self) -> AdminServer:
|
|
@@ -448,7 +443,6 @@ class DBOS:
|
|
|
448
443
|
dbos_logger.info(f"Application version: {GlobalParams.app_version}")
|
|
449
444
|
self._executor_field = ThreadPoolExecutor(max_workers=sys.maxsize)
|
|
450
445
|
self._background_event_loop.start()
|
|
451
|
-
assert self._config["database_url"] is not None
|
|
452
446
|
assert self._config["database"]["sys_db_engine_kwargs"] is not None
|
|
453
447
|
self._sys_db_field = SystemDatabase.create(
|
|
454
448
|
system_database_url=get_system_database_url(self._config),
|
|
@@ -456,18 +450,20 @@ class DBOS:
|
|
|
456
450
|
debug_mode=debug_mode,
|
|
457
451
|
)
|
|
458
452
|
assert self._config["database"]["db_engine_kwargs"] is not None
|
|
459
|
-
self.
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
453
|
+
if self._config["database_url"]:
|
|
454
|
+
self._app_db_field = ApplicationDatabase.create(
|
|
455
|
+
database_url=self._config["database_url"],
|
|
456
|
+
engine_kwargs=self._config["database"]["db_engine_kwargs"],
|
|
457
|
+
debug_mode=debug_mode,
|
|
458
|
+
)
|
|
464
459
|
|
|
465
460
|
if debug_mode:
|
|
466
461
|
return
|
|
467
462
|
|
|
468
463
|
# Run migrations for the system and application databases
|
|
469
464
|
self._sys_db.run_migrations()
|
|
470
|
-
self._app_db
|
|
465
|
+
if self._app_db:
|
|
466
|
+
self._app_db.run_migrations()
|
|
471
467
|
|
|
472
468
|
admin_port = self._config.get("runtimeConfig", {}).get("admin_port")
|
|
473
469
|
if admin_port is None:
|
|
@@ -408,22 +408,20 @@ def process_config(
|
|
|
408
408
|
url = url.set(database=f"{url.database}{SystemSchema.sysdb_suffix}")
|
|
409
409
|
data["system_database_url"] = url.render_as_string(hide_password=False)
|
|
410
410
|
|
|
411
|
-
# If a system database URL is provided but not an application database URL,
|
|
412
|
-
#
|
|
411
|
+
# If a system database URL is provided but not an application database URL,
|
|
412
|
+
# do not create an application database.
|
|
413
413
|
if data.get("system_database_url") and not data.get("database_url"):
|
|
414
414
|
assert data["system_database_url"]
|
|
415
|
-
data["database_url"] =
|
|
415
|
+
data["database_url"] = None
|
|
416
416
|
|
|
417
|
-
# If neither URL is provided, use a default SQLite database URL.
|
|
417
|
+
# If neither URL is provided, use a default SQLite system database URL.
|
|
418
418
|
if not data.get("database_url") and not data.get("system_database_url"):
|
|
419
419
|
_app_db_name = _app_name_to_db_name(data["name"])
|
|
420
|
-
data["system_database_url"] =
|
|
421
|
-
|
|
422
|
-
)
|
|
420
|
+
data["system_database_url"] = f"sqlite:///{_app_db_name}.sqlite"
|
|
421
|
+
data["database_url"] = None
|
|
423
422
|
|
|
424
423
|
configure_db_engine_parameters(data["database"], connect_timeout=connect_timeout)
|
|
425
424
|
|
|
426
|
-
assert data["database_url"] is not None
|
|
427
425
|
assert data["system_database_url"] is not None
|
|
428
426
|
# Pretty-print connection information, respecting log level
|
|
429
427
|
if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
|
|
@@ -431,7 +429,12 @@ def process_config(
|
|
|
431
429
|
hide_password=True
|
|
432
430
|
)
|
|
433
431
|
print(f"DBOS system database URL: {printable_sys_db_url}")
|
|
434
|
-
if data["database_url"]
|
|
432
|
+
if data["database_url"]:
|
|
433
|
+
printable_app_db_url = make_url(data["database_url"]).render_as_string(
|
|
434
|
+
hide_password=True
|
|
435
|
+
)
|
|
436
|
+
print(f"DBOS application database URL: {printable_app_db_url}")
|
|
437
|
+
if data["system_database_url"].startswith("sqlite"):
|
|
435
438
|
print(
|
|
436
439
|
f"Using SQLite as a system database. The SQLite system database is for development and testing. PostgreSQL is recommended for production use."
|
|
437
440
|
)
|
|
@@ -615,12 +618,11 @@ def get_system_database_url(config: ConfigFile) -> str:
|
|
|
615
618
|
)
|
|
616
619
|
|
|
617
620
|
|
|
618
|
-
def get_application_database_url(config: ConfigFile) -> str:
|
|
621
|
+
def get_application_database_url(config: ConfigFile) -> str | None:
|
|
619
622
|
# For backwards compatibility, the application database URL is "database_url"
|
|
620
623
|
if config.get("database_url"):
|
|
621
624
|
assert config["database_url"]
|
|
622
625
|
return config["database_url"]
|
|
623
626
|
else:
|
|
624
|
-
# If the application database URL is not specified,
|
|
625
|
-
|
|
626
|
-
return config["system_database_url"]
|
|
627
|
+
# If the application database URL is not specified, return None
|
|
628
|
+
return None
|
|
@@ -98,10 +98,10 @@ def get_workflow(sys_db: SystemDatabase, workflow_id: str) -> Optional[WorkflowS
|
|
|
98
98
|
|
|
99
99
|
|
|
100
100
|
def list_workflow_steps(
|
|
101
|
-
sys_db: SystemDatabase, app_db: ApplicationDatabase, workflow_id: str
|
|
101
|
+
sys_db: SystemDatabase, app_db: Optional[ApplicationDatabase], workflow_id: str
|
|
102
102
|
) -> List[StepInfo]:
|
|
103
103
|
steps = sys_db.get_workflow_steps(workflow_id)
|
|
104
|
-
transactions = app_db.get_transactions(workflow_id)
|
|
104
|
+
transactions = app_db.get_transactions(workflow_id) if app_db else []
|
|
105
105
|
merged_steps = steps + transactions
|
|
106
106
|
merged_steps.sort(key=lambda step: step["function_id"])
|
|
107
107
|
return merged_steps
|
|
@@ -109,7 +109,7 @@ def list_workflow_steps(
|
|
|
109
109
|
|
|
110
110
|
def fork_workflow(
|
|
111
111
|
sys_db: SystemDatabase,
|
|
112
|
-
app_db: ApplicationDatabase,
|
|
112
|
+
app_db: Optional[ApplicationDatabase],
|
|
113
113
|
workflow_id: str,
|
|
114
114
|
start_step: int,
|
|
115
115
|
*,
|
|
@@ -122,7 +122,8 @@ def fork_workflow(
|
|
|
122
122
|
ctx.id_assigned_for_next_workflow = ""
|
|
123
123
|
else:
|
|
124
124
|
forked_workflow_id = str(uuid.uuid4())
|
|
125
|
-
app_db
|
|
125
|
+
if app_db:
|
|
126
|
+
app_db.clone_workflow_transactions(workflow_id, forked_workflow_id, start_step)
|
|
126
127
|
sys_db.fork_workflow(
|
|
127
128
|
workflow_id,
|
|
128
129
|
forked_workflow_id,
|
|
@@ -145,7 +146,10 @@ def garbage_collect(
|
|
|
145
146
|
)
|
|
146
147
|
if result is not None:
|
|
147
148
|
cutoff_epoch_timestamp_ms, pending_workflow_ids = result
|
|
148
|
-
dbos._app_db
|
|
149
|
+
if dbos._app_db:
|
|
150
|
+
dbos._app_db.garbage_collect(
|
|
151
|
+
cutoff_epoch_timestamp_ms, pending_workflow_ids
|
|
152
|
+
)
|
|
149
153
|
|
|
150
154
|
|
|
151
155
|
def global_timeout(dbos: "DBOS", cutoff_epoch_timestamp_ms: int) -> None:
|
|
@@ -38,7 +38,7 @@ class DefaultEncoder(json.JSONEncoder):
|
|
|
38
38
|
|
|
39
39
|
def _get_db_url(
|
|
40
40
|
*, system_database_url: Optional[str], application_database_url: Optional[str]
|
|
41
|
-
) -> Tuple[str, str]:
|
|
41
|
+
) -> Tuple[str, str | None]:
|
|
42
42
|
"""
|
|
43
43
|
Get the database URL to use for the DBOS application.
|
|
44
44
|
Order of precedence:
|
|
@@ -294,7 +294,8 @@ def migrate(
|
|
|
294
294
|
)
|
|
295
295
|
|
|
296
296
|
typer.echo(f"Starting DBOS migrations")
|
|
297
|
-
|
|
297
|
+
if application_database_url:
|
|
298
|
+
typer.echo(f"Application database: {sa.make_url(application_database_url)}")
|
|
298
299
|
typer.echo(f"System database: {sa.make_url(system_database_url)}")
|
|
299
300
|
|
|
300
301
|
# First, run DBOS migrations on the system database and the application database
|
|
@@ -305,9 +306,10 @@ def migrate(
|
|
|
305
306
|
|
|
306
307
|
# Next, assign permissions on the DBOS schema to the application role, if any
|
|
307
308
|
if application_role:
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
309
|
+
if application_database_url:
|
|
310
|
+
grant_dbos_schema_permissions(
|
|
311
|
+
database_url=application_database_url, role_name=application_role
|
|
312
|
+
)
|
|
311
313
|
grant_dbos_schema_permissions(
|
|
312
314
|
database_url=system_database_url, role_name=application_role
|
|
313
315
|
)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
1
3
|
import sqlalchemy as sa
|
|
2
4
|
import typer
|
|
3
5
|
|
|
@@ -5,7 +7,9 @@ from dbos._app_db import ApplicationDatabase
|
|
|
5
7
|
from dbos._sys_db import SystemDatabase
|
|
6
8
|
|
|
7
9
|
|
|
8
|
-
def migrate_dbos_databases(
|
|
10
|
+
def migrate_dbos_databases(
|
|
11
|
+
app_database_url: Optional[str], system_database_url: str
|
|
12
|
+
) -> None:
|
|
9
13
|
app_db = None
|
|
10
14
|
sys_db = None
|
|
11
15
|
try:
|
|
@@ -17,16 +21,17 @@ def migrate_dbos_databases(app_database_url: str, system_database_url: str) -> N
|
|
|
17
21
|
"pool_size": 2,
|
|
18
22
|
},
|
|
19
23
|
)
|
|
20
|
-
app_db = ApplicationDatabase.create(
|
|
21
|
-
database_url=app_database_url,
|
|
22
|
-
engine_kwargs={
|
|
23
|
-
"pool_timeout": 30,
|
|
24
|
-
"max_overflow": 0,
|
|
25
|
-
"pool_size": 2,
|
|
26
|
-
},
|
|
27
|
-
)
|
|
28
24
|
sys_db.run_migrations()
|
|
29
|
-
|
|
25
|
+
if app_database_url:
|
|
26
|
+
app_db = ApplicationDatabase.create(
|
|
27
|
+
database_url=app_database_url,
|
|
28
|
+
engine_kwargs={
|
|
29
|
+
"pool_timeout": 30,
|
|
30
|
+
"max_overflow": 0,
|
|
31
|
+
"pool_size": 2,
|
|
32
|
+
},
|
|
33
|
+
)
|
|
34
|
+
app_db.run_migrations()
|
|
30
35
|
except Exception as e:
|
|
31
36
|
typer.echo(f"DBOS migrations failed: {e}")
|
|
32
37
|
raise typer.Exit(code=1)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
|
+
|
|
2
3
|
from dbos import DBOS, DBOSConfig
|
|
3
4
|
|
|
4
5
|
config: DBOSConfig = {
|
|
@@ -7,19 +8,23 @@ config: DBOSConfig = {
|
|
|
7
8
|
}
|
|
8
9
|
DBOS(config=config)
|
|
9
10
|
|
|
11
|
+
|
|
10
12
|
@DBOS.step()
|
|
11
13
|
def step_one() -> None:
|
|
12
14
|
print("Step one completed!")
|
|
13
15
|
|
|
16
|
+
|
|
14
17
|
@DBOS.step()
|
|
15
18
|
def step_two() -> None:
|
|
16
19
|
print("Step two completed!")
|
|
17
20
|
|
|
21
|
+
|
|
18
22
|
@DBOS.workflow()
|
|
19
23
|
def dbos_workflow() -> None:
|
|
20
24
|
step_one()
|
|
21
25
|
step_two()
|
|
22
26
|
|
|
27
|
+
|
|
23
28
|
if __name__ == "__main__":
|
|
24
29
|
DBOS.launch()
|
|
25
|
-
dbos_workflow()
|
|
30
|
+
dbos_workflow()
|
|
@@ -279,7 +279,7 @@ def test_process_config_only_system_database():
|
|
|
279
279
|
configFile = process_config(data=config)
|
|
280
280
|
assert configFile["name"] == "some-app"
|
|
281
281
|
assert configFile["system_database_url"] == config["system_database_url"]
|
|
282
|
-
assert configFile["database_url"]
|
|
282
|
+
assert configFile["database_url"] is None
|
|
283
283
|
|
|
284
284
|
|
|
285
285
|
def test_process_config_sqlite():
|
|
@@ -331,8 +331,8 @@ def test_process_config_load_defaults():
|
|
|
331
331
|
}
|
|
332
332
|
processed_config = process_config(data=config)
|
|
333
333
|
assert processed_config["name"] == "some-app"
|
|
334
|
-
assert processed_config["database_url"]
|
|
335
|
-
assert processed_config["system_database_url"] ==
|
|
334
|
+
assert processed_config["database_url"] is None
|
|
335
|
+
assert processed_config["system_database_url"] == f"sqlite:///some_app.sqlite"
|
|
336
336
|
assert processed_config["database"]["db_engine_kwargs"] is not None
|
|
337
337
|
assert processed_config["database"]["sys_db_engine_kwargs"] is not None
|
|
338
338
|
assert processed_config["telemetry"]["logs"]["logLevel"] == "INFO"
|
|
@@ -346,8 +346,8 @@ def test_process_config_load_default_with_None_database_url():
|
|
|
346
346
|
}
|
|
347
347
|
processed_config = process_config(data=config)
|
|
348
348
|
assert processed_config["name"] == "some-app"
|
|
349
|
-
assert processed_config["database_url"]
|
|
350
|
-
assert processed_config["system_database_url"] ==
|
|
349
|
+
assert processed_config["database_url"] is None
|
|
350
|
+
assert processed_config["system_database_url"] == f"sqlite:///some_app.sqlite"
|
|
351
351
|
assert processed_config["database"]["db_engine_kwargs"] is not None
|
|
352
352
|
assert processed_config["database"]["sys_db_engine_kwargs"] is not None
|
|
353
353
|
assert processed_config["telemetry"]["logs"]["logLevel"] == "INFO"
|
|
@@ -361,8 +361,8 @@ def test_process_config_load_default_with_empty_database_url():
|
|
|
361
361
|
}
|
|
362
362
|
processed_config = process_config(data=config)
|
|
363
363
|
assert processed_config["name"] == "some-app"
|
|
364
|
-
assert processed_config["database_url"]
|
|
365
|
-
assert processed_config["system_database_url"] ==
|
|
364
|
+
assert processed_config["database_url"] is None
|
|
365
|
+
assert processed_config["system_database_url"] == f"sqlite:///some_app.sqlite"
|
|
366
366
|
assert processed_config["database"]["db_engine_kwargs"] is not None
|
|
367
367
|
assert processed_config["database"]["sys_db_engine_kwargs"] is not None
|
|
368
368
|
assert processed_config["telemetry"]["logs"]["logLevel"] == "INFO"
|
|
@@ -396,8 +396,8 @@ def test_config_mixed_params():
|
|
|
396
396
|
|
|
397
397
|
configFile = process_config(data=config)
|
|
398
398
|
assert configFile["name"] == "some-app"
|
|
399
|
-
assert configFile["database_url"]
|
|
400
|
-
assert configFile["system_database_url"] ==
|
|
399
|
+
assert configFile["database_url"] is None
|
|
400
|
+
assert configFile["system_database_url"] == f"sqlite:///some_app.sqlite"
|
|
401
401
|
assert configFile["database"]["db_engine_kwargs"] is not None
|
|
402
402
|
assert configFile["database"]["sys_db_engine_kwargs"] is not None
|
|
403
403
|
assert configFile["telemetry"]["logs"]["logLevel"] == "INFO"
|
|
@@ -24,6 +24,7 @@ from dbos import (
|
|
|
24
24
|
)
|
|
25
25
|
|
|
26
26
|
# Private API because this is a test
|
|
27
|
+
from dbos._client import DBOSClient
|
|
27
28
|
from dbos._context import assert_current_dbos_context, get_local_dbos_context
|
|
28
29
|
from dbos._error import (
|
|
29
30
|
DBOSAwaitedWorkflowCancelledError,
|
|
@@ -1730,3 +1731,47 @@ def test_destroy(dbos: DBOS, config: DBOSConfig) -> None:
|
|
|
1730
1731
|
blocking_event.set()
|
|
1731
1732
|
with pytest.raises(DBOSException):
|
|
1732
1733
|
handle.get_result()
|
|
1734
|
+
|
|
1735
|
+
|
|
1736
|
+
def test_without_appdb(config: DBOSConfig, cleanup_test_databases: None) -> None:
|
|
1737
|
+
DBOS.destroy(destroy_registry=True)
|
|
1738
|
+
config["application_database_url"] = None
|
|
1739
|
+
dbos = DBOS(config=config)
|
|
1740
|
+
DBOS.launch()
|
|
1741
|
+
assert dbos._app_db is None
|
|
1742
|
+
|
|
1743
|
+
@DBOS.step()
|
|
1744
|
+
def step() -> None:
|
|
1745
|
+
return
|
|
1746
|
+
|
|
1747
|
+
@DBOS.workflow()
|
|
1748
|
+
def workflow() -> str:
|
|
1749
|
+
step()
|
|
1750
|
+
step()
|
|
1751
|
+
step()
|
|
1752
|
+
assert DBOS.workflow_id
|
|
1753
|
+
return DBOS.workflow_id
|
|
1754
|
+
|
|
1755
|
+
wfid = workflow()
|
|
1756
|
+
assert wfid
|
|
1757
|
+
steps = DBOS.list_workflow_steps(wfid)
|
|
1758
|
+
assert len(steps) == 3
|
|
1759
|
+
for s in steps:
|
|
1760
|
+
assert s["function_name"] == step.__qualname__
|
|
1761
|
+
forked_handle = DBOS.fork_workflow(wfid, start_step=1)
|
|
1762
|
+
assert forked_handle.get_result() == forked_handle.workflow_id
|
|
1763
|
+
|
|
1764
|
+
@DBOS.transaction()
|
|
1765
|
+
def transaction() -> None:
|
|
1766
|
+
return
|
|
1767
|
+
|
|
1768
|
+
with pytest.raises(AssertionError):
|
|
1769
|
+
transaction()
|
|
1770
|
+
|
|
1771
|
+
DBOS.destroy(destroy_registry=True)
|
|
1772
|
+
|
|
1773
|
+
client = DBOSClient(system_database_url=config["system_database_url"])
|
|
1774
|
+
steps = client.list_workflow_steps(wfid)
|
|
1775
|
+
assert len(steps) == 3
|
|
1776
|
+
for s in steps:
|
|
1777
|
+
assert s["function_name"] == step.__qualname__
|
|
@@ -28,9 +28,7 @@ def test_package(
|
|
|
28
28
|
with db_engine.connect() as connection:
|
|
29
29
|
connection.execution_options(isolation_level="AUTOCOMMIT")
|
|
30
30
|
connection.execute(sa.text(f"DROP DATABASE IF EXISTS {app_db_name}"))
|
|
31
|
-
connection.execute(
|
|
32
|
-
sa.text(f"DROP DATABASE IF EXISTS {sys_db_name}")
|
|
33
|
-
)
|
|
31
|
+
connection.execute(sa.text(f"DROP DATABASE IF EXISTS {sys_db_name}"))
|
|
34
32
|
|
|
35
33
|
with tempfile.TemporaryDirectory() as temp_path:
|
|
36
34
|
temp_path = tempfile.mkdtemp(prefix="dbos-")
|
|
@@ -62,7 +60,9 @@ def test_package(
|
|
|
62
60
|
)
|
|
63
61
|
|
|
64
62
|
# Next, verify a simple DBOS-only script runs
|
|
65
|
-
subprocess.check_call(
|
|
63
|
+
subprocess.check_call(
|
|
64
|
+
["python3", "tests/script_without_fastapi.py"], env=venv
|
|
65
|
+
)
|
|
66
66
|
|
|
67
67
|
# Install FastAPI into the virtual environment
|
|
68
68
|
subprocess.check_call(
|
|
@@ -186,65 +186,93 @@ def test_migrate(db_engine: sa.Engine, skip_with_sqlite: None) -> None:
|
|
|
186
186
|
db_url = db_engine.url.set(database=database_name).set(drivername="postgresql")
|
|
187
187
|
db_url_string = db_url.render_as_string(hide_password=False)
|
|
188
188
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
connection.execute(sa.text(f'DROP ROLE IF EXISTS "{role_name}"'))
|
|
196
|
-
connection.execute(
|
|
197
|
-
sa.text(
|
|
198
|
-
f"CREATE ROLE \"{role_name}\" WITH LOGIN PASSWORD '{role_password}'"
|
|
189
|
+
for use_app_db in [True, False]:
|
|
190
|
+
# Drop the DBOS database if it exists. Create a test role with no permissions.
|
|
191
|
+
with db_engine.connect() as connection:
|
|
192
|
+
connection.execution_options(isolation_level="AUTOCOMMIT")
|
|
193
|
+
connection.execute(
|
|
194
|
+
sa.text(f"DROP DATABASE IF EXISTS {database_name} WITH (FORCE)")
|
|
199
195
|
)
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
["dbos", "migrate", "-D", db_url_string, "-s", db_url_string, "-r", role_name]
|
|
206
|
-
)
|
|
207
|
-
with db_engine.connect() as c:
|
|
208
|
-
c.execution_options(isolation_level="AUTOCOMMIT")
|
|
209
|
-
result = c.execute(
|
|
210
|
-
sa.text(
|
|
211
|
-
f"SELECT COUNT(*) FROM pg_database WHERE datname = '{database_name}'"
|
|
196
|
+
connection.execute(sa.text(f'DROP ROLE IF EXISTS "{role_name}"'))
|
|
197
|
+
connection.execute(
|
|
198
|
+
sa.text(
|
|
199
|
+
f"CREATE ROLE \"{role_name}\" WITH LOGIN PASSWORD '{role_password}'"
|
|
200
|
+
)
|
|
212
201
|
)
|
|
213
|
-
).scalar()
|
|
214
|
-
assert result == 1
|
|
215
|
-
|
|
216
|
-
# Initialize DBOS with the test role. Verify various operations work.
|
|
217
|
-
test_db_url = (
|
|
218
|
-
db_url.set(username=role_name).set(password=role_password)
|
|
219
|
-
).render_as_string(hide_password=False)
|
|
220
|
-
DBOS.destroy(destroy_registry=True)
|
|
221
|
-
config: DBOSConfig = {
|
|
222
|
-
"name": "test_migrate",
|
|
223
|
-
"database_url": test_db_url,
|
|
224
|
-
"system_database_url": test_db_url,
|
|
225
|
-
}
|
|
226
|
-
DBOS(config=config)
|
|
227
|
-
|
|
228
|
-
@DBOS.transaction()
|
|
229
|
-
def test_transaction() -> str:
|
|
230
|
-
rows = DBOS.sql_session.execute(sa.text("SELECT 1")).fetchall()
|
|
231
|
-
return str(rows[0][0])
|
|
232
|
-
|
|
233
|
-
@DBOS.workflow()
|
|
234
|
-
def test_workflow() -> str:
|
|
235
|
-
assert test_transaction() == "1"
|
|
236
|
-
id = DBOS.workflow_id
|
|
237
|
-
assert id
|
|
238
|
-
DBOS.set_event(id, id)
|
|
239
|
-
return id
|
|
240
202
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
203
|
+
# Using the admin role, create the DBOS database and verify it exists.
|
|
204
|
+
# Set permissions for the test role.
|
|
205
|
+
if use_app_db:
|
|
206
|
+
subprocess.check_call(
|
|
207
|
+
[
|
|
208
|
+
"dbos",
|
|
209
|
+
"migrate",
|
|
210
|
+
"-D",
|
|
211
|
+
db_url_string,
|
|
212
|
+
"-s",
|
|
213
|
+
db_url_string,
|
|
214
|
+
"-r",
|
|
215
|
+
role_name,
|
|
216
|
+
]
|
|
217
|
+
)
|
|
218
|
+
else:
|
|
219
|
+
subprocess.check_call(
|
|
220
|
+
["dbos", "migrate", "-s", db_url_string, "-r", role_name]
|
|
221
|
+
)
|
|
222
|
+
with db_engine.connect() as c:
|
|
223
|
+
c.execution_options(isolation_level="AUTOCOMMIT")
|
|
224
|
+
result = c.execute(
|
|
225
|
+
sa.text(
|
|
226
|
+
f"SELECT COUNT(*) FROM pg_database WHERE datname = '{database_name}'"
|
|
227
|
+
)
|
|
228
|
+
).scalar()
|
|
229
|
+
assert result == 1
|
|
230
|
+
|
|
231
|
+
# Initialize DBOS with the test role. Verify various operations work.
|
|
232
|
+
test_db_url = (
|
|
233
|
+
db_url.set(username=role_name).set(password=role_password)
|
|
234
|
+
).render_as_string(hide_password=False)
|
|
235
|
+
DBOS.destroy(destroy_registry=True)
|
|
236
|
+
config: DBOSConfig = {
|
|
237
|
+
"name": "test_migrate",
|
|
238
|
+
"database_url": test_db_url if use_app_db else None,
|
|
239
|
+
"system_database_url": test_db_url,
|
|
240
|
+
}
|
|
241
|
+
dbos = DBOS(config=config)
|
|
242
|
+
if not use_app_db:
|
|
243
|
+
assert dbos._app_db is None
|
|
244
|
+
|
|
245
|
+
@DBOS.transaction()
|
|
246
|
+
def test_transaction() -> str:
|
|
247
|
+
rows = DBOS.sql_session.execute(sa.text("SELECT 1")).fetchall()
|
|
248
|
+
return str(rows[0][0])
|
|
249
|
+
|
|
250
|
+
@DBOS.step()
|
|
251
|
+
def test_step() -> str:
|
|
252
|
+
return "1"
|
|
253
|
+
|
|
254
|
+
@DBOS.workflow()
|
|
255
|
+
def test_workflow() -> str:
|
|
256
|
+
if use_app_db:
|
|
257
|
+
assert test_transaction() == "1"
|
|
258
|
+
else:
|
|
259
|
+
assert test_step() == "1"
|
|
260
|
+
id = DBOS.workflow_id
|
|
261
|
+
assert id
|
|
262
|
+
DBOS.set_event(id, id)
|
|
263
|
+
return id
|
|
264
|
+
|
|
265
|
+
DBOS.launch()
|
|
266
|
+
|
|
267
|
+
workflow_id = test_workflow()
|
|
268
|
+
assert workflow_id
|
|
269
|
+
assert DBOS.get_event(workflow_id, workflow_id) == workflow_id
|
|
270
|
+
|
|
271
|
+
steps = DBOS.list_workflow_steps(workflow_id)
|
|
272
|
+
assert len(steps) == 2
|
|
273
|
+
assert (
|
|
274
|
+
steps[0]["function_name"] == test_transaction.__qualname__
|
|
275
|
+
if use_app_db
|
|
276
|
+
else test_step.__qualname__
|
|
277
|
+
)
|
|
278
|
+
assert steps[1]["function_name"] == "DBOS.setEvent"
|
|
@@ -669,6 +669,7 @@ def test_garbage_collection(dbos: DBOS, skip_with_sqlite_imprecise_time: None) -
|
|
|
669
669
|
assert len(workflows) == 2
|
|
670
670
|
assert workflows[0].workflow_id == handle.workflow_id
|
|
671
671
|
# Verify txn outputs are preserved only for the remaining workflows
|
|
672
|
+
assert dbos._app_db
|
|
672
673
|
with dbos._app_db.engine.begin() as c:
|
|
673
674
|
rows = c.execute(
|
|
674
675
|
sa.select(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dbos-1.15.0a1 → dbos-1.15.0a2}/dbos/_templates/dbos-db-starter/migrations/create_table.py.dbos
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|