dbos 2.4.0a7__py3-none-any.whl → 2.6.0a8__py3-none-any.whl

dbos/_dbos.py

@@ -38,6 +38,7 @@ from dbos._workflow_commands import fork_workflow, list_queued_workflows, list_w
 from ._classproperty import classproperty
 from ._core import (
     DEBOUNCER_WORKFLOW_NAME,
+    DEFAULT_POLLING_INTERVAL,
     TEMP_SEND_WF_NAME,
     WorkflowHandleAsyncPolling,
     WorkflowHandlePolling,
@@ -111,7 +112,7 @@ from ._logger import (
     dbos_logger,
     init_logger,
 )
-from ._workflow_commands import get_workflow, list_workflow_steps
+from ._workflow_commands import get_workflow
 
 # Most DBOS functions are just any callable F, so decorators / wrappers work on F
 # There are cases where the parameters P and return value R should be separate
@@ -335,6 +336,8 @@ class DBOS:
         self._executor_field: Optional[ThreadPoolExecutor] = None
         self._background_threads: List[threading.Thread] = []
         self.conductor_url: Optional[str] = conductor_url
+        if config.get("conductor_url"):
+            self.conductor_url = config.get("conductor_url")
         self.conductor_key: Optional[str] = conductor_key
         if config.get("conductor_key"):
             self.conductor_key = config.get("conductor_key")
@@ -569,8 +572,8 @@ class DBOS:
                 conductor_registration_url = (
                     f"https://console.dbos.dev/self-host?appname={app_name}"
                 )
-                print(
-                    f"To view and manage workflows, connect to DBOS Conductor at:{conductor_registration_url}"
+                dbos_logger.info(
+                    f"To view and manage workflows, connect to DBOS Conductor at: {conductor_registration_url}"
                 )
 
             # Flush handlers and add OTLP to all loggers if enabled
@@ -1090,7 +1093,6 @@ class DBOS:
             dbos_logger.info(f"Forking workflow: {workflow_id} from step {start_step}")
             return fork_workflow(
                 _get_dbos_instance()._sys_db,
-                _get_dbos_instance()._app_db,
                 workflow_id,
                 start_step,
                 application_version=application_version,
@@ -1267,9 +1269,7 @@ class DBOS:
     @classmethod
     def list_workflow_steps(cls, workflow_id: str) -> List[StepInfo]:
         def fn() -> List[StepInfo]:
-            return list_workflow_steps(
-                _get_dbos_instance()._sys_db, _get_dbos_instance()._app_db, workflow_id
-            )
+            return _get_dbos_instance()._sys_db.list_workflow_steps(workflow_id)
 
         return _get_dbos_instance()._sys_db.call_function_as_step(
             fn, "DBOS.listWorkflowSteps"
@@ -1326,11 +1326,10 @@ class DBOS:
             return None
 
     @classproperty
-    def span(cls) -> "Span":
+    def span(cls) -> Optional["Span"]:
         """Return the tracing `Span` associated with the current context."""
         ctx = assert_current_dbos_context()
         span = ctx.get_current_active_span()
-        assert span
         return span
 
     @classproperty
@@ -1384,7 +1383,7 @@ class DBOS:
                     )
             elif ctx.is_step():
                 _get_dbos_instance()._sys_db.write_stream_from_step(
-                    ctx.workflow_id, key, value
+                    ctx.workflow_id, ctx.function_id, key, value
                 )
             else:
                 raise DBOSException(
@@ -1551,7 +1550,9 @@ class WorkflowHandle(Generic[R], Protocol):
         """Return the applicable workflow ID."""
         ...
 
-    def get_result(self) -> R:
+    def get_result(
+        self, *, polling_interval_sec: float = DEFAULT_POLLING_INTERVAL
+    ) -> R:
         """Return the result of the workflow function invocation, waiting if necessary."""
         ...
 
@@ -1580,7 +1581,9 @@ class WorkflowHandleAsync(Generic[R], Protocol):
         """Return the applicable workflow ID."""
         ...
 
-    async def get_result(self) -> R:
+    async def get_result(
+        self, *, polling_interval_sec: float = DEFAULT_POLLING_INTERVAL
+    ) -> R:
         """Return the result of the workflow function invocation, waiting if necessary."""
         ...
 

dbos/_dbos_config.py

@@ -39,6 +39,7 @@ class DBOSConfig(TypedDict, total=False):
         enable_otlp (bool): If True, enable built-in DBOS OTLP tracing and logging.
         system_database_engine (sa.Engine): A custom system database engine. If provided, DBOS will not create an engine but use this instead.
         conductor_key (str): An API key for DBOS Conductor. Pass this in to connect your process to Conductor.
+        conductor_url (str): The websockets URL for your DBOS Conductor service. Only set if you're self-hosting Conductor.
         serializer (Serializer): A custom serializer and deserializer DBOS uses when storing program data in the system database
     """
 
@@ -60,6 +61,7 @@ class DBOSConfig(TypedDict, total=False):
     enable_otlp: Optional[bool]
     system_database_engine: Optional[sa.Engine]
     conductor_key: Optional[str]
+    conductor_url: Optional[str]
     serializer: Optional[Serializer]
 
 
@@ -406,25 +408,6 @@ def process_config(
 
     configure_db_engine_parameters(data["database"], connect_timeout=connect_timeout)
 
-    assert data["system_database_url"] is not None
-    # Pretty-print connection information, respecting log level
-    if not silent and logs["logLevel"] == "INFO" or logs["logLevel"] == "DEBUG":
-        printable_sys_db_url = make_url(data["system_database_url"]).render_as_string(
-            hide_password=True
-        )
-        print(f"DBOS system database URL: {printable_sys_db_url}")
-        if data["database_url"]:
-            printable_app_db_url = make_url(data["database_url"]).render_as_string(
-                hide_password=True
-            )
-            print(f"DBOS application database URL: {printable_app_db_url}")
-        if data["system_database_url"].startswith("sqlite"):
-            print(
-                f"Using SQLite as a system database. The SQLite system database is for development and testing. PostgreSQL is recommended for production use."
-            )
-        else:
-            print(f"Database engine parameters: {data['database']['db_engine_kwargs']}")
-
     # Return data as ConfigFile type
     return data
 

dbos/_fastapi.py

@@ -87,5 +87,6 @@ def setup_fastapi_middleware(app: FastAPI, dbos: DBOS) -> None:
                 and not dbos._config["telemetry"]["disable_otlp"]
                 and hasattr(response, "status_code")
             ):
-                DBOS.span.set_attribute("responseCode", response.status_code)
+                if DBOS.span is not None:
+                    DBOS.span.set_attribute("responseCode", response.status_code)
         return response

dbos/_logger.py

@@ -71,6 +71,7 @@ def config_logger(config: "ConfigFile") -> None:
     if not disable_otlp:
 
         from opentelemetry._logs import get_logger_provider, set_logger_provider
+        from opentelemetry._logs._internal import ProxyLoggerProvider
         from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
         from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
         from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
@@ -80,7 +81,8 @@ def config_logger(config: "ConfigFile") -> None:
         # Only set up OTLP provider and exporter if endpoints are provided
         log_provider = get_logger_provider()
         if otlp_logs_endpoints is not None and len(otlp_logs_endpoints) > 0:
-            if not isinstance(log_provider, LoggerProvider):
+            if isinstance(log_provider, ProxyLoggerProvider):
+                # Set a real LoggerProvider if it was previously a ProxyLoggerProvider
                 log_provider = LoggerProvider(
                     Resource.create(
                         attributes={
@@ -91,7 +93,7 @@ def config_logger(config: "ConfigFile") -> None:
                 set_logger_provider(log_provider)
 
             for e in otlp_logs_endpoints:
-                log_provider.add_log_record_processor(
+                log_provider.add_log_record_processor(  # type: ignore
                     BatchLogRecordProcessor(
                         OTLPLogExporter(endpoint=e),
                         export_timeout_millis=5000,
@@ -101,10 +103,14 @@ def config_logger(config: "ConfigFile") -> None:
         # Even if no endpoints are provided, we still need a LoggerProvider to create the LoggingHandler
         global _otlp_handler
         if _otlp_handler is None:
-            _otlp_handler = LoggingHandler(logger_provider=log_provider)
-
-            # Direct DBOS logs to OTLP
-            dbos_logger.addHandler(_otlp_handler)
+            if isinstance(log_provider, ProxyLoggerProvider):
+                dbos_logger.warning(
+                    "OTLP is enabled but logger provider not set, skipping log exporter setup."
+                )
+            else:
+                _otlp_handler = LoggingHandler(logger_provider=log_provider)
+                # Direct DBOS logs to OTLP
+                dbos_logger.addHandler(_otlp_handler)
 
     # Attach DBOS-specific attributes to all log entries.
     global _dbos_log_transformer

dbos/_migration.py

@@ -228,6 +228,21 @@ ALTER TABLE \"{schema}\".operation_outputs ADD COLUMN started_at_epoch_ms BIGINT
 """
 
 
+def get_dbos_migration_six(schema: str) -> str:
+    return f"""
+CREATE TABLE \"{schema}\".workflow_events_history (
+    workflow_uuid TEXT NOT NULL,
+    function_id INTEGER NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    PRIMARY KEY (workflow_uuid, function_id, key),
+    FOREIGN KEY (workflow_uuid) REFERENCES \"{schema}\".workflow_status(workflow_uuid) 
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+ALTER TABLE \"{schema}\".streams ADD COLUMN function_id INTEGER NOT NULL DEFAULT 0;
+"""
+
+
 def get_dbos_migrations(schema: str) -> list[str]:
     return [
         get_dbos_migration_one(schema),
@@ -235,6 +250,7 @@ def get_dbos_migrations(schema: str) -> list[str]:
         get_dbos_migration_three(schema),
         get_dbos_migration_four(schema),
         get_dbos_migration_five(schema),
+        get_dbos_migration_six(schema),
     ]
 
 
@@ -343,6 +359,19 @@ ALTER TABLE operation_outputs ADD COLUMN started_at_epoch_ms BIGINT;
 ALTER TABLE operation_outputs ADD COLUMN completed_at_epoch_ms BIGINT;
 """
 
+sqlite_migration_six = """
+CREATE TABLE workflow_events_history (
+    workflow_uuid TEXT NOT NULL,
+    function_id INTEGER NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    PRIMARY KEY (workflow_uuid, function_id, key),
+    FOREIGN KEY (workflow_uuid) REFERENCES workflow_status(workflow_uuid)
+        ON UPDATE CASCADE ON DELETE CASCADE
+);
+ALTER TABLE streams ADD COLUMN function_id INTEGER NOT NULL DEFAULT 0;
+"""
+
 
 sqlite_migrations = [
     sqlite_migration_one,
@@ -350,4 +379,5 @@ sqlite_migrations = [
     sqlite_migration_three,
     sqlite_migration_four,
     sqlite_migration_five,
+    sqlite_migration_six,
 ]

dbos/_queue.py

@@ -44,6 +44,7 @@ class Queue:
         worker_concurrency: Optional[int] = None,
         priority_enabled: bool = False,
         partition_queue: bool = False,
+        polling_interval_sec: float = 1.0,
     ) -> None:
         if (
             worker_concurrency is not None
@@ -53,12 +54,15 @@ class Queue:
             raise ValueError(
                 "worker_concurrency must be less than or equal to concurrency"
             )
+        if polling_interval_sec <= 0.0:
+            raise ValueError("polling_interval_sec must be positive")
         self.name = name
         self.concurrency = concurrency
         self.worker_concurrency = worker_concurrency
         self.limiter = limiter
         self.priority_enabled = priority_enabled
         self.partition_queue = partition_queue
+        self.polling_interval_sec = polling_interval_sec
         from ._dbos import _get_or_create_dbos_registry
 
         registry = _get_or_create_dbos_registry()
@@ -108,50 +112,103 @@ class Queue:
         return await start_workflow_async(dbos, func, self.name, False, *args, **kwargs)
 
 
-def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
-    polling_interval = 1.0
-    min_polling_interval = 1.0
-    max_polling_interval = 120.0
+def queue_worker_thread(
+    stop_event: threading.Event, dbos: "DBOS", queue: Queue
+) -> None:
+    """Worker thread for processing a single queue."""
+    polling_interval = queue.polling_interval_sec
+    min_polling_interval = queue.polling_interval_sec
+    max_polling_interval = max(queue.polling_interval_sec, 120.0)
+
     while not stop_event.is_set():
         # Wait for the polling interval with jitter
         if stop_event.wait(timeout=polling_interval * random.uniform(0.95, 1.05)):
             return
-        queues = dict(dbos._registry.queue_info_map)
-        for _, queue in queues.items():
-            try:
-                if queue.partition_queue:
-                    dequeued_workflows = []
-                    queue_partition_keys = dbos._sys_db.get_queue_partitions(queue.name)
-                    for key in queue_partition_keys:
-                        dequeued_workflows += dbos._sys_db.start_queued_workflows(
-                            queue,
-                            GlobalParams.executor_id,
-                            GlobalParams.app_version,
-                            key,
-                        )
-                else:
+
+        try:
+            if queue.partition_queue:
+                queue_partition_keys = dbos._sys_db.get_queue_partitions(queue.name)
+                for key in queue_partition_keys:
                     dequeued_workflows = dbos._sys_db.start_queued_workflows(
-                        queue, GlobalParams.executor_id, GlobalParams.app_version, None
+                        queue,
+                        GlobalParams.executor_id,
+                        GlobalParams.app_version,
+                        key,
                     )
+                    for id in dequeued_workflows:
+                        execute_workflow_by_id(dbos, id)
+            else:
+                dequeued_workflows = dbos._sys_db.start_queued_workflows(
+                    queue, GlobalParams.executor_id, GlobalParams.app_version, None
+                )
                 for id in dequeued_workflows:
                     execute_workflow_by_id(dbos, id)
-            except OperationalError as e:
-                if isinstance(
-                    e.orig, (errors.SerializationFailure, errors.LockNotAvailable)
-                ):
-                    # If a serialization error is encountered, increase the polling interval
-                    polling_interval = min(
-                        max_polling_interval,
-                        polling_interval * 2.0,
-                    )
-                    dbos.logger.warning(
-                        f"Contention detected in queue thread for {queue.name}. Increasing polling interval to {polling_interval:.2f}."
-                    )
-                else:
-                    dbos.logger.warning(f"Exception encountered in queue thread: {e}")
-            except Exception as e:
-                if not stop_event.is_set():
-                    # Only print the error if the thread is not stopping
-                    dbos.logger.warning(f"Exception encountered in queue thread: {e}")
+        except OperationalError as e:
+            if isinstance(
+                e.orig, (errors.SerializationFailure, errors.LockNotAvailable)
+            ):
+                # If a serialization error is encountered, increase the polling interval
+                polling_interval = min(
+                    max_polling_interval,
+                    polling_interval * 2.0,
+                )
+                dbos.logger.warning(
+                    f"Contention detected in queue thread for {queue.name}. Increasing polling interval to {polling_interval:.2f}."
+                )
+            else:
+                dbos.logger.warning(
+                    f"Exception encountered in queue thread for {queue.name}: {e}"
+                )
+        except Exception as e:
+            if not stop_event.is_set():
+                # Only print the error if the thread is not stopping
+                dbos.logger.warning(
+                    f"Exception encountered in queue thread for {queue.name}: {e}"
+                )
+
         # Attempt to scale back the polling interval on each iteration
         polling_interval = max(min_polling_interval, polling_interval * 0.9)
+
+
+def queue_thread(stop_event: threading.Event, dbos: "DBOS") -> None:
+    """Main queue manager thread that spawns and monitors worker threads for each queue."""
+    queue_threads: dict[str, threading.Thread] = {}
+    check_interval = 1.0  # Check for new queues every second
+
+    while not stop_event.is_set():
+        # Check for new queues
+        current_queues = dict(dbos._registry.queue_info_map)
+
+        # Start threads for new queues
+        for queue_name, queue in current_queues.items():
+            if (
+                queue_name not in queue_threads
+                or not queue_threads[queue_name].is_alive()
+            ):
+                thread = threading.Thread(
+                    target=queue_worker_thread,
+                    args=(stop_event, dbos, queue),
+                    name=f"queue-worker-{queue_name}",
+                    daemon=True,
+                )
+                thread.start()
+                queue_threads[queue_name] = thread
+                dbos.logger.debug(f"Started worker thread for queue: {queue_name}")
+
+        # Wait for the check interval or stop event
+        if stop_event.wait(timeout=check_interval):
+            break
+
+    # Join all queue worker threads
+    dbos.logger.info("Stopping queue manager, joining all worker threads...")
+    for queue_name, thread in queue_threads.items():
+        if thread.is_alive():
+            thread.join(timeout=10.0)  # Give each thread 10 seconds to finish
+            if thread.is_alive():
+                dbos.logger.debug(
+                    f"Queue worker thread for {queue_name} did not stop in time"
+                )
+            else:
+                dbos.logger.debug(
+                    f"Queue worker thread for {queue_name} stopped successfully"
+                )

dbos/_schemas/system_database.py

@@ -35,6 +35,7 @@ class SystemSchema:
         cls.notifications.schema = schema_name
         cls.workflow_events.schema = schema_name
         cls.streams.schema = schema_name
+        cls.workflow_events_history.schema = schema_name
 
     workflow_status = Table(
         "workflow_status",
@@ -154,6 +155,24 @@ class SystemSchema:
         PrimaryKeyConstraint("workflow_uuid", "key"),
     )
 
+    # This is an immutable version of workflow_events. Two tables are needed for backwards compatibility.
+    workflow_events_history = Table(
+        "workflow_events_history",
+        metadata_obj,
+        Column(
+            "workflow_uuid",
+            Text,
+            ForeignKey(
+                "workflow_status.workflow_uuid", onupdate="CASCADE", ondelete="CASCADE"
+            ),
+            nullable=False,
+        ),
+        Column("key", Text, nullable=False),
+        Column("value", Text, nullable=False),
+        Column("function_id", Integer, nullable=False, server_default=text("'0'::int")),
+        PrimaryKeyConstraint("workflow_uuid", "key", "function_id"),
+    )
+
     streams = Table(
         "streams",
         metadata_obj,
@@ -168,5 +187,6 @@ class SystemSchema:
         Column("key", Text, nullable=False),
         Column("value", Text, nullable=False),
         Column("offset", Integer, nullable=False),
+        Column("function_id", Integer, nullable=False, server_default=text("'0'::int")),
         PrimaryKeyConstraint("workflow_uuid", "key", "offset"),
     )