dbos 1.12.0a3__py3-none-any.whl → 1.13.0a5__py3-none-any.whl

dbos/_dbos_config.py

@@ -33,6 +33,9 @@ class DBOSConfig(TypedDict, total=False):
         admin_port (int): Admin port
         run_admin_server (bool): Whether to run the DBOS admin server
         otlp_attributes (dict[str, str]): A set of custom attributes to apply OTLP-exported logs and traces
+        application_version (str): Application version
+        executor_id (str): Executor ID, used to identify the application instance in distributed environments
+        disable_otlp (bool): If True, disables OTLP tracing and logging. Defaults to False.
     """
 
     name: str
@@ -49,6 +52,7 @@ class DBOSConfig(TypedDict, total=False):
     otlp_attributes: Optional[dict[str, str]]
     application_version: Optional[str]
     executor_id: Optional[str]
+    disable_otlp: Optional[bool]
 
 
 class RuntimeConfig(TypedDict, total=False):
@@ -91,6 +95,7 @@ class TelemetryConfig(TypedDict, total=False):
     logs: Optional[LoggerConfig]
     OTLPExporter: Optional[OTLPExporterConfig]
     otlp_attributes: Optional[dict[str, str]]
+    disable_otlp: Optional[bool]
 
 
 class ConfigFile(TypedDict, total=False):
@@ -157,6 +162,7 @@ def translate_dbos_config_to_config_file(config: DBOSConfig) -> ConfigFile:
     telemetry: TelemetryConfig = {
         "OTLPExporter": {"tracesEndpoint": [], "logsEndpoint": []},
         "otlp_attributes": config.get("otlp_attributes", {}),
+        "disable_otlp": config.get("disable_otlp", False),
     }
     # For mypy
     assert telemetry["OTLPExporter"] is not None
@@ -290,19 +296,12 @@ def process_config(
     """
     If a database_url is provided, pass it as is in the config.
 
-    Else, build a database_url from defaults.
+    Else, default to SQLite.
 
     Also build SQL Alchemy "kwargs" base on user input + defaults.
     Specifically, db_engine_kwargs takes precedence over app_db_pool_size
 
     In debug mode, apply overrides from DBOS_DBHOST, DBOS_DBPORT, DBOS_DBUSER, and DBOS_DBPASSWORD.
-
-    Default configuration:
-        - Hostname: localhost
-        - Port: 5432
-        - Username: postgres
-        - Password: $PGPASSWORD
-        - Database name: transformed application name.
     """
 
     if "name" not in data:
@@ -344,9 +343,14 @@ def process_config(
 
         url = make_url(data["database_url"])
 
-        if not data["database"].get("sys_db_name"):
+        if data["database_url"].startswith("sqlite"):
+            data["system_database_url"] = data["database_url"]
+        else:
             assert url.database is not None
-            data["database"]["sys_db_name"] = url.database + SystemSchema.sysdb_suffix
+            if not data["database"].get("sys_db_name"):
+                data["database"]["sys_db_name"] = (
+                    url.database + SystemSchema.sysdb_suffix
+                )
 
         # Gather connect_timeout from the URL if provided. It should be used in engine kwargs if not provided there (instead of our default)
         connect_timeout_str = url.query.get("connect_timeout")
@@ -374,23 +378,24 @@ def process_config(
             ).render_as_string(hide_password=False)
     else:
         _app_db_name = _app_name_to_db_name(data["name"])
-        _password = os.environ.get("PGPASSWORD", "dbos")
-        data["database_url"] = (
-            f"postgres://postgres:{_password}@localhost:5432/{_app_db_name}?connect_timeout=10&sslmode=prefer"
-        )
-        if not data["database"].get("sys_db_name"):
-            data["database"]["sys_db_name"] = _app_db_name + SystemSchema.sysdb_suffix
-        assert data["database_url"] is not None
+        data["database_url"] = f"sqlite:///{_app_db_name}.sqlite"
+        data["system_database_url"] = data["database_url"]
 
     configure_db_engine_parameters(data["database"], connect_timeout=connect_timeout)
 
     # Pretty-print where we've loaded database connection information from, respecting the log level
+    assert data["database_url"] is not None
     if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
         log_url = make_url(data["database_url"]).render_as_string(hide_password=True)
         print(f"[bold blue]Using database connection string: {log_url}[/bold blue]")
-        print(
-            f"[bold blue]Database engine parameters: {data['database']['db_engine_kwargs']}[/bold blue]"
-        )
+        if data["database_url"].startswith("sqlite"):
+            print(
+                f"[bold blue]Using SQLite as a system database. The SQLite system database is for development and testing. PostgreSQL is recommended for production use.[/bold blue]"
+            )
+        else:
+            print(
+                f"[bold blue]Database engine parameters: {data['database']['db_engine_kwargs']}[/bold blue]"
+            )
 
     # Return data as ConfigFile type
     return data
@@ -439,6 +444,8 @@ def configure_db_engine_parameters(
 
 
 def is_valid_database_url(database_url: str) -> bool:
+    if database_url.startswith("sqlite"):
+        return True
     url = make_url(database_url)
     required_fields = [
         ("username", "Username must be specified in the connection URL"),
@@ -553,6 +560,8 @@ def get_system_database_url(config: ConfigFile) -> str:
         return config["system_database_url"]
     else:
         assert config["database_url"] is not None
+        if config["database_url"].startswith("sqlite"):
+            return config["database_url"]
         app_db_url = make_url(config["database_url"])
         if config["database"].get("sys_db_name") is not None:
             sys_db_name = config["database"]["sys_db_name"]

dbos/_fastapi.py

@@ -49,7 +49,7 @@ class LifespanMiddleware:
                     if not self.dbos._launched:
                         self.dbos._launch()
                 elif message["type"] == "lifespan.shutdown.complete":
-                    self.dbos._destroy()
+                    self.dbos.destroy()
                 await send(message)
 
             # Call the original app with our wrapped functions

dbos/_logger.py

@@ -77,7 +77,9 @@ def config_logger(config: "ConfigFile") -> None:
     otlp_logs_endpoints = (
         config.get("telemetry", {}).get("OTLPExporter", {}).get("logsEndpoint")  # type: ignore
     )
-    if otlp_logs_endpoints:
+    disable_otlp = config.get("telemetry", {}).get("disable_otlp", False)  # type: ignore
+
+    if not disable_otlp and otlp_logs_endpoints:
         log_provider = PatchedOTLPLoggerProvider(
             Resource.create(
                 attributes={

dbos/_migration.py

@@ -0,0 +1,322 @@
+import logging
+import os
+import re
+import sys
+
+import sqlalchemy as sa
+from alembic import command
+from alembic.config import Config
+
+from ._logger import dbos_logger
+
+
+def ensure_dbos_schema(engine: sa.Engine) -> bool:
+    """
+    True if using DBOS migrations (DBOS schema and migrations table already exist or were created)
+    False if using Alembic migrations (DBOS schema exists, but dbos_migrations table doesn't)
+    """
+    with engine.begin() as conn:
+        # Check if dbos schema exists
+        schema_result = conn.execute(
+            sa.text(
+                "SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'dbos'"
+            )
+        )
+        schema_existed = schema_result.fetchone() is not None
+
+        # Create schema if it doesn't exist
+        if not schema_existed:
+            conn.execute(sa.text("CREATE SCHEMA dbos"))
+
+        # Check if dbos_migrations table exists
+        table_result = conn.execute(
+            sa.text(
+                "SELECT table_name FROM information_schema.tables WHERE table_schema = 'dbos' AND table_name = 'dbos_migrations'"
+            )
+        )
+        table_exists = table_result.fetchone() is not None
+
+        if table_exists:
+            return True
+        elif schema_existed:
+            return False
+        else:
+            conn.execute(
+                sa.text(
+                    "CREATE TABLE dbos.dbos_migrations (version BIGINT NOT NULL PRIMARY KEY)"
+                )
+            )
+            return True
+
+
+def run_alembic_migrations(engine: sa.Engine) -> None:
+    """Run system database schema migrations with Alembic.
+    This is DEPRECATED in favor of DBOS-managed migrations.
+    It is retained only for backwards compatibility and
+    will be removed in the next major version."""
+    # Run a schema migration for the system database
+    migration_dir = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "_alembic_migrations"
+    )
+    alembic_cfg = Config()
+    alembic_cfg.set_main_option("script_location", migration_dir)
+    logging.getLogger("alembic").setLevel(logging.WARNING)
+    # Alembic requires the % in URL-escaped parameters to itself be escaped to %%.
+    escaped_conn_string = re.sub(
+        r"%(?=[0-9A-Fa-f]{2})",
+        "%%",
+        engine.url.render_as_string(hide_password=False),
+    )
+    alembic_cfg.set_main_option("sqlalchemy.url", escaped_conn_string)
+    try:
+        command.upgrade(alembic_cfg, "head")
+    except Exception as e:
+        dbos_logger.warning(
+            f"Exception during system database construction. This is most likely because the system database was configured using a later version of DBOS: {e}"
+        )
+
+
+def run_dbos_migrations(engine: sa.Engine) -> None:
+    """Run DBOS-managed migrations by executing each SQL command in dbos_migrations."""
+    with engine.begin() as conn:
+        # Get current migration version
+        result = conn.execute(sa.text("SELECT version FROM dbos.dbos_migrations"))
+        current_version = result.fetchone()
+        last_applied = current_version[0] if current_version else 0
+
+        # Apply migrations starting from the next version
+        for i, migration_sql in enumerate(dbos_migrations, 1):
+            if i <= last_applied:
+                continue
+
+            # Execute the migration
+            dbos_logger.info(f"Applying DBOS system database schema migration {i}")
+            conn.execute(sa.text(migration_sql))
+
+            # Update the single row with the new version
+            if last_applied == 0:
+                conn.execute(
+                    sa.text(
+                        "INSERT INTO dbos.dbos_migrations (version) VALUES (:version)"
+                    ),
+                    {"version": i},
+                )
+            else:
+                conn.execute(
+                    sa.text("UPDATE dbos.dbos_migrations SET version = :version"),
+                    {"version": i},
+                )
+            last_applied = i
+
+
+dbos_migration_one = """
+-- Enable uuid extension for generating UUIDs
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+CREATE TABLE dbos.workflow_status (
+    workflow_uuid TEXT PRIMARY KEY,
+    status TEXT,
+    name TEXT,
+    authenticated_user TEXT,
+    assumed_role TEXT,
+    authenticated_roles TEXT,
+    request TEXT,
+    output TEXT,
+    error TEXT,
+    executor_id TEXT,
+    created_at BIGINT NOT NULL DEFAULT (EXTRACT(epoch FROM now()) * 1000::numeric)::bigint,
+    updated_at BIGINT NOT NULL DEFAULT (EXTRACT(epoch FROM now()) * 1000::numeric)::bigint,
+    application_version TEXT,
+    application_id TEXT,
+    class_name VARCHAR(255) DEFAULT NULL,
+    config_name VARCHAR(255) DEFAULT NULL,
+    recovery_attempts BIGINT DEFAULT 0,
+    queue_name TEXT,
+    workflow_timeout_ms BIGINT,
+    workflow_deadline_epoch_ms BIGINT,
+    inputs TEXT,
+    started_at_epoch_ms BIGINT,
+    deduplication_id TEXT,
+    priority INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE INDEX workflow_status_created_at_index ON dbos.workflow_status (created_at);
+CREATE INDEX workflow_status_executor_id_index ON dbos.workflow_status (executor_id);
+CREATE INDEX workflow_status_status_index ON dbos.workflow_status (status);
+
+ALTER TABLE dbos.workflow_status 
+ADD CONSTRAINT uq_workflow_status_queue_name_dedup_id 
+UNIQUE (queue_name, deduplication_id);
+
+CREATE TABLE dbos.operation_outputs (
+    workflow_uuid TEXT NOT NULL,
+    function_id INTEGER NOT NULL,
+    function_name TEXT NOT NULL DEFAULT '',
+    output TEXT,
+    error TEXT,
+    child_workflow_id TEXT,
+    PRIMARY KEY (workflow_uuid, function_id),
+    FOREIGN KEY (workflow_uuid) REFERENCES dbos.workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+
+CREATE TABLE dbos.notifications (
+    destination_uuid TEXT NOT NULL,
+    topic TEXT,
+    message TEXT NOT NULL,
+    created_at_epoch_ms BIGINT NOT NULL DEFAULT (EXTRACT(epoch FROM now()) * 1000::numeric)::bigint,
+    message_uuid TEXT NOT NULL DEFAULT gen_random_uuid(), -- Built-in function
+    FOREIGN KEY (destination_uuid) REFERENCES dbos.workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+CREATE INDEX idx_workflow_topic ON dbos.notifications (destination_uuid, topic);
+
+-- Create notification function
+CREATE OR REPLACE FUNCTION dbos.notifications_function() RETURNS TRIGGER AS $$
+DECLARE
+    payload text := NEW.destination_uuid || '::' || NEW.topic;
+BEGIN
+    PERFORM pg_notify('dbos_notifications_channel', payload);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Create notification trigger
+CREATE TRIGGER dbos_notifications_trigger
+AFTER INSERT ON dbos.notifications
+FOR EACH ROW EXECUTE FUNCTION dbos.notifications_function();
+
+CREATE TABLE dbos.workflow_events (
+    workflow_uuid TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    PRIMARY KEY (workflow_uuid, key),
+    FOREIGN KEY (workflow_uuid) REFERENCES dbos.workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+
+-- Create events function
+CREATE OR REPLACE FUNCTION dbos.workflow_events_function() RETURNS TRIGGER AS $$
+DECLARE
+    payload text := NEW.workflow_uuid || '::' || NEW.key;
+BEGIN
+    PERFORM pg_notify('dbos_workflow_events_channel', payload);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Create events trigger
+CREATE TRIGGER dbos_workflow_events_trigger
+AFTER INSERT ON dbos.workflow_events
+FOR EACH ROW EXECUTE FUNCTION dbos.workflow_events_function();
+
+CREATE TABLE dbos.streams (
+    workflow_uuid TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    "offset" INTEGER NOT NULL,
+    PRIMARY KEY (workflow_uuid, key, "offset"),
+    FOREIGN KEY (workflow_uuid) REFERENCES dbos.workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+
+CREATE TABLE dbos.event_dispatch_kv (
+    service_name TEXT NOT NULL,
+    workflow_fn_name TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT,
+    update_seq NUMERIC(38,0),
+    update_time NUMERIC(38,15),
+    PRIMARY KEY (service_name, workflow_fn_name, key)
+);
+"""
+
+
+def get_sqlite_timestamp_expr() -> str:
+    """Get SQLite timestamp expression with millisecond precision for Python >= 3.12."""
+    if sys.version_info >= (3, 12):
+        return "(unixepoch('subsec') * 1000)"
+    else:
+        return "(strftime('%s','now') * 1000)"
+
+
+sqlite_migration_one = f"""
+CREATE TABLE workflow_status (
+    workflow_uuid TEXT PRIMARY KEY,
+    status TEXT,
+    name TEXT,
+    authenticated_user TEXT,
+    assumed_role TEXT,
+    authenticated_roles TEXT,
+    request TEXT,
+    output TEXT,
+    error TEXT,
+    executor_id TEXT,
+    created_at INTEGER NOT NULL DEFAULT {get_sqlite_timestamp_expr()},
+    updated_at INTEGER NOT NULL DEFAULT {get_sqlite_timestamp_expr()},
+    application_version TEXT,
+    application_id TEXT,
+    class_name TEXT DEFAULT NULL,
+    config_name TEXT DEFAULT NULL,
+    recovery_attempts INTEGER DEFAULT 0,
+    queue_name TEXT,
+    workflow_timeout_ms INTEGER,
+    workflow_deadline_epoch_ms INTEGER,
+    inputs TEXT,
+    started_at_epoch_ms INTEGER,
+    deduplication_id TEXT,
+    priority INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE INDEX workflow_status_created_at_index ON workflow_status (created_at);
+CREATE INDEX workflow_status_executor_id_index ON workflow_status (executor_id);
+CREATE INDEX workflow_status_status_index ON workflow_status (status);
+
+CREATE UNIQUE INDEX uq_workflow_status_queue_name_dedup_id 
+ON workflow_status (queue_name, deduplication_id);
+
+CREATE TABLE operation_outputs (
+    workflow_uuid TEXT NOT NULL,
+    function_id INTEGER NOT NULL,
+    function_name TEXT NOT NULL DEFAULT '',
+    output TEXT,
+    error TEXT,
+    child_workflow_id TEXT,
+    PRIMARY KEY (workflow_uuid, function_id),
+    FOREIGN KEY (workflow_uuid) REFERENCES workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+
+CREATE TABLE notifications (
+    destination_uuid TEXT NOT NULL,
+    topic TEXT,
+    message TEXT NOT NULL,
+    created_at_epoch_ms INTEGER NOT NULL DEFAULT {get_sqlite_timestamp_expr()},
+    message_uuid TEXT NOT NULL DEFAULT (hex(randomblob(16))),
+    FOREIGN KEY (destination_uuid) REFERENCES workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+CREATE INDEX idx_workflow_topic ON notifications (destination_uuid, topic);
+
+CREATE TABLE workflow_events (
+    workflow_uuid TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    PRIMARY KEY (workflow_uuid, key),
+    FOREIGN KEY (workflow_uuid) REFERENCES workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+
+CREATE TABLE streams (
+    workflow_uuid TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    "offset" INTEGER NOT NULL,
+    PRIMARY KEY (workflow_uuid, key, "offset"),
+    FOREIGN KEY (workflow_uuid) REFERENCES workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+"""
+
+dbos_migrations = [dbos_migration_one]
+sqlite_migrations = [sqlite_migration_one]