dbos 1.13.0a3__py3-none-any.whl → 1.13.0a5__py3-none-any.whl

dbos/_sys_db_sqlite.py

@@ -0,0 +1,185 @@
+import os
+import time
+from typing import Any, Dict, Optional, Tuple
+
+import sqlalchemy as sa
+from sqlalchemy.exc import DBAPIError
+
+from dbos._migration import sqlite_migrations
+from dbos._schemas.system_database import SystemSchema
+
+from ._logger import dbos_logger
+from ._sys_db import SystemDatabase
+
+
+class SQLiteSystemDatabase(SystemDatabase):
+    """SQLite-specific implementation of SystemDatabase."""
+
+    def _create_engine(
+        self, system_database_url: str, engine_kwargs: Dict[str, Any]
+    ) -> sa.Engine:
+        """Create a SQLite engine."""
+        # TODO: Make the schema dynamic so this isn't needed
+        SystemSchema.workflow_status.schema = None
+        SystemSchema.operation_outputs.schema = None
+        SystemSchema.notifications.schema = None
+        SystemSchema.workflow_events.schema = None
+        SystemSchema.streams.schema = None
+        return sa.create_engine(system_database_url)
+
+    def run_migrations(self) -> None:
+        """Run SQLite-specific migrations."""
+        if self._debug_mode:
+            dbos_logger.warning("System database migrations are skipped in debug mode.")
+            return
+
+        with self.engine.begin() as conn:
+            # Enable foreign keys for SQLite
+            conn.execute(sa.text("PRAGMA foreign_keys = ON"))
+
+            # Check if migrations table exists
+            result = conn.execute(
+                sa.text(
+                    "SELECT name FROM sqlite_master WHERE type='table' AND name='dbos_migrations'"
+                )
+            ).fetchone()
+
+            if result is None:
+                # Create migrations table
+                conn.execute(
+                    sa.text(
+                        "CREATE TABLE dbos_migrations (version INTEGER NOT NULL PRIMARY KEY)"
+                    )
+                )
+                last_applied = 0
+            else:
+                # Get current migration version
+                version_result = conn.execute(
+                    sa.text("SELECT version FROM dbos_migrations")
+                ).fetchone()
+                last_applied = version_result[0] if version_result else 0
+
+            # Apply migrations starting from the next version
+            for i, migration_sql in enumerate(sqlite_migrations, 1):
+                if i <= last_applied:
+                    continue
+
+                # Execute the migration
+                dbos_logger.info(
+                    f"Applying DBOS SQLite system database schema migration {i}"
+                )
+
+                # SQLite only allows one statement at a time, so split by semicolon
+                statements = [
+                    stmt.strip() for stmt in migration_sql.split(";") if stmt.strip()
+                ]
+                for statement in statements:
+                    conn.execute(sa.text(statement))
+
+                # Update the single row with the new version
+                if last_applied == 0:
+                    conn.execute(
+                        sa.text(
+                            "INSERT INTO dbos_migrations (version) VALUES (:version)"
+                        ),
+                        {"version": i},
+                    )
+                else:
+                    conn.execute(
+                        sa.text("UPDATE dbos_migrations SET version = :version"),
+                        {"version": i},
+                    )
+                last_applied = i
+
+    def _cleanup_connections(self) -> None:
+        """Clean up SQLite-specific connections."""
+        # SQLite doesn't require special connection cleanup
+        pass
+
+    def _is_unique_constraint_violation(self, dbapi_error: DBAPIError) -> bool:
+        """Check if the error is a unique constraint violation in SQLite."""
+        # SQLite UNIQUE constraint error
+        return "UNIQUE constraint failed" in str(dbapi_error.orig)
+
+    def _is_foreign_key_violation(self, dbapi_error: DBAPIError) -> bool:
+        """Check if the error is a foreign key violation in SQLite."""
+        # SQLite FOREIGN KEY constraint error
+        return "FOREIGN KEY constraint failed" in str(dbapi_error.orig)
+
+    @staticmethod
+    def _reset_system_database(database_url: str) -> None:
+        """Reset the SQLite system database by deleting the database file."""
+
+        # Parse the SQLite database URL to get the file path
+        url = sa.make_url(database_url)
+        db_path = url.database
+
+        if db_path is None:
+            raise ValueError(f"System database path not found in URL {url}")
+
+        try:
+            if os.path.exists(db_path):
+                os.remove(db_path)
+                dbos_logger.info(f"Deleted SQLite database file: {db_path}")
+            else:
+                dbos_logger.info(f"SQLite database file does not exist: {db_path}")
+        except OSError as e:
+            dbos_logger.error(
+                f"Error deleting SQLite database file {db_path}: {str(e)}"
+            )
+            raise e
+
+    def _notification_listener(self) -> None:
+        """Poll for notifications and workflow events in SQLite."""
+
+        def split_payload(payload: str) -> Tuple[str, Optional[str]]:
+            """Split payload into components (first::second format)."""
+            if "::" in payload:
+                parts = payload.split("::", 1)
+                return parts[0], parts[1]
+            return payload, None
+
+        def signal_condition(condition_map: Any, payload: str) -> None:
+            """Signal a condition variable if it exists."""
+            condition = condition_map.get(payload)
+            if condition:
+                condition.acquire()
+                condition.notify_all()
+                condition.release()
+                dbos_logger.debug(f"Signaled condition for {payload}")
+
+        while self._run_background_processes:
+            try:
+                # Poll every second
+                time.sleep(1)
+
+                # Check all payloads in the notifications_map
+                for payload in list(self.notifications_map._dict.keys()):
+                    dest_uuid, topic = split_payload(payload)
+                    with self.engine.begin() as conn:
+                        result = conn.execute(
+                            sa.text(
+                                "SELECT 1 FROM notifications WHERE destination_uuid = :dest_uuid AND topic = :topic LIMIT 1"
+                            ),
+                            {"dest_uuid": dest_uuid, "topic": topic},
+                        )
+                        if result.fetchone():
+                            signal_condition(self.notifications_map, payload)
+
+                # Check all payloads in the workflow_events_map
+                for payload in list(self.workflow_events_map._dict.keys()):
+                    workflow_uuid, key = split_payload(payload)
+                    with self.engine.begin() as conn:
+                        result = conn.execute(
+                            sa.text(
+                                "SELECT 1 FROM workflow_events WHERE workflow_uuid = :workflow_uuid AND key = :key LIMIT 1"
+                            ),
+                            {"workflow_uuid": workflow_uuid, "key": key},
+                        )
+                        if result.fetchone():
+                            signal_condition(self.workflow_events_map, payload)
+
+            except Exception as e:
+                if self._run_background_processes:
+                    dbos_logger.warning(f"SQLite notification poller error: {e}")
+                    time.sleep(1)

dbos/_utils.py

@@ -20,7 +20,9 @@ class GlobalParams:
         dbos_version = "unknown"
 
 
-def retriable_postgres_exception(e: DBAPIError) -> bool:
+def retriable_postgres_exception(e: Exception) -> bool:
+    if not isinstance(e, DBAPIError):
+        return False
     if e.connection_invalidated:
         return True
     if isinstance(e.orig, psycopg.OperationalError):
@@ -48,3 +50,10 @@ def retriable_postgres_exception(e: DBAPIError) -> bool:
             return False
     else:
         return False
+
+
+def retriable_sqlite_exception(e: Exception) -> bool:
+    if "database is locked" in str(e):
+        return True
+    else:
+        return False

dbos/cli/cli.py

@@ -29,7 +29,7 @@ from .._dbos_config import (
 )
 from .._docker_pg_helper import start_docker_pg, stop_docker_pg
 from .._schemas.system_database import SystemSchema
-from .._sys_db import SystemDatabase, reset_system_database
+from .._sys_db import SystemDatabase
 from .._utils import GlobalParams
 from ..cli._github_init import create_template_from_github
 from ._template_init import copy_template, get_project_name, get_templates_directory
@@ -371,22 +371,12 @@ def reset(
             typer.echo("Operation cancelled.")
             raise typer.Exit()
     try:
-        # Make a SA url out of the user-provided URL and verify a database name is present
         database_url = _get_db_url(db_url)
-        pg_db_url = sa.make_url(database_url)
-        assert (
-            pg_db_url.database is not None
-        ), f"Database name is required in URL: {pg_db_url.render_as_string(hide_password=True)}"
-        # Resolve system database name
-        sysdb_name = (
-            sys_db_name
-            if sys_db_name
-            else (pg_db_url.database + SystemSchema.sysdb_suffix)
+        system_database_url = get_system_database_url(
+            {"database_url": database_url, "database": {"sys_db_name": sys_db_name}}
         )
-        reset_system_database(
-            postgres_db_url=pg_db_url.set(database="postgres"), sysdb_name=sysdb_name
-        )
-    except sa.exc.SQLAlchemyError as e:
+        SystemDatabase.reset_system_database(system_database_url)
+    except Exception as e:
         typer.echo(f"Error resetting system database: {str(e)}")
         return
 

dbos/cli/migration.py

@@ -9,7 +9,7 @@ def migrate_dbos_databases(app_database_url: str, system_database_url: str) -> N
     app_db = None
     sys_db = None
     try:
-        sys_db = SystemDatabase(
+        sys_db = SystemDatabase.create(
             system_database_url=system_database_url,
             engine_kwargs={
                 "pool_timeout": 30,
@@ -17,7 +17,7 @@ def migrate_dbos_databases(app_database_url: str, system_database_url: str) -> N
                 "pool_size": 2,
             },
         )
-        app_db = ApplicationDatabase(
+        app_db = ApplicationDatabase.create(
             database_url=app_database_url,
             engine_kwargs={
                 "pool_timeout": 30,

{dbos-1.13.0a3.dist-info → dbos-1.13.0a5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 1.13.0a3
+Version: 1.13.0a5
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-1.13.0a3.dist-info → dbos-1.13.0a5.dist-info}/RECORD

@@ -1,7 +1,7 @@
-dbos-1.13.0a3.dist-info/METADATA,sha256=EkcdC671U6Va-Oi2RP7Zs1v7Ag6ZaN99Gy4lil5JVss,13268
-dbos-1.13.0a3.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
-dbos-1.13.0a3.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
-dbos-1.13.0a3.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
+dbos-1.13.0a5.dist-info/METADATA,sha256=obXcr7i8Wu_nO0bUihx40pr-fopo32nU3rqFBBe3D0o,13268
+dbos-1.13.0a5.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
+dbos-1.13.0a5.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
+dbos-1.13.0a5.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
 dbos/__init__.py,sha256=NssPCubaBxdiKarOWa-wViz1hdJSkmBGcpLX_gQ4NeA,891
 dbos/__main__.py,sha256=G7Exn-MhGrVJVDbgNlpzhfh8WMX_72t3_oJaFT9Lmt8,653
 dbos/_admin_server.py,sha256=e8ELhcDWqR3_PNobnNgUvLGh5lzZq0yFSF6dvtzoQRI,16267
@@ -22,16 +22,16 @@ dbos/_alembic_migrations/versions/d76646551a6c_workflow_queue.py,sha256=G942noph
 dbos/_alembic_migrations/versions/d994145b47b6_consolidate_inputs.py,sha256=_J0jP247fuo66fzOmLlKFO9FJ_CRBXlqa2lnLrcXugQ,672
 dbos/_alembic_migrations/versions/eab0cc1d9a14_job_queue.py,sha256=uvhFOtqbBreCePhAxZfIT0qCAI7BiZTou9wt6QnbY7c,1412
 dbos/_alembic_migrations/versions/f4b9b32ba814_functionname_childid_op_outputs.py,sha256=m90Lc5YH0ZISSq1MyxND6oq3RZrZKrIqEsZtwJ1jWxA,1049
-dbos/_app_db.py,sha256=bUXQqzc0C9PHh4Zl2tHfBrQWNBURdI7F7XXjCpYirmw,10959
+dbos/_app_db.py,sha256=e8mASnDX_tyE9DWf_yAZgIi7ZWeyVWWTrhCZK_Jy50g,16190
 dbos/_classproperty.py,sha256=f0X-_BySzn3yFDRKB2JpCbLYQ9tLwt1XftfshvY7CBs,626
-dbos/_client.py,sha256=_wMe4qnRSwiRZo74xdqTBetbHlIVy3vQifdSd7os1ZY,18213
+dbos/_client.py,sha256=hSSIqWZEbGVAyRy80rS4Sywaf_biQOF17ThW3Eyaaso,18267
 dbos/_conductor/conductor.py,sha256=3E_hL3c9g9yWqKZkvI6KA0-ZzPMPRo06TOzT1esMiek,24114
 dbos/_conductor/protocol.py,sha256=q3rgLxINFtWFigdOONc-4gX4vn66UmMlJQD6Kj8LnL4,7420
 dbos/_context.py,sha256=IMboNgbCqTxfIORqeifE3da-Ce5siMz7MYMLPk5M-AQ,26851
-dbos/_core.py,sha256=bEtIXf0OEzEwiS4sYLJR3vT-bmt5Hg1SBHUcCkaHmMQ,49005
+dbos/_core.py,sha256=2IWEkewiYL3d3MrhMfqj0JDWNyNM54rpW7CRGjBTPfk,48788
 dbos/_croniter.py,sha256=XHAyUyibs_59sJQfSNWkP7rqQY6_XrlfuuCxk4jYqek,47559
-dbos/_dbos.py,sha256=kNTJLQn1uLnzbglyRZLSkJYOYhQseoVp84yaV1lAc00,58458
-dbos/_dbos_config.py,sha256=qduDBmrpISNRZcCO-NcARQMeO2AzExyC2h3HCd9lRqU,22128
+dbos/_dbos.py,sha256=wKLN7W1ej6cyEOHnCtt3-awnU6SR6MQAlOvTTt5_N-E,58219
+dbos/_dbos_config.py,sha256=jv-k2WyXgfoR3R0NaPxAFx5OzZYjr0rjH2npsbokGRM,22321
 dbos/_debug.py,sha256=99j2SChWmCPAlZoDmjsJGe77tpU2LEa8E2TtLAnnh7o,1831
 dbos/_docker_pg_helper.py,sha256=tLJXWqZ4S-ExcaPnxg_i6cVxL6ZxrYlZjaGsklY-s2I,6115
 dbos/_error.py,sha256=GwO0Ng4d4iB52brY09-Ss6Cz_V28Xc0D0cRCzZ6XmNM,8688
@@ -41,7 +41,7 @@ dbos/_flask.py,sha256=Npnakt-a3W5OykONFRkDRnumaDhTQmA0NPdUCGRYKXE,1652
 dbos/_kafka.py,sha256=Gm4fHWl7gYb-i5BMvwNwm5Km3z8zQpseqdMgqgFjlGI,4252
 dbos/_kafka_message.py,sha256=NYvOXNG3Qn7bghn1pv3fg4Pbs86ILZGcK4IB-MLUNu0,409
 dbos/_logger.py,sha256=vvBL4kzJWBPtfcHIuL0nJAq5cgpeo_D3IiIUDicWLOg,4732
-dbos/_migration.py,sha256=C2ODGlZKjnGKS3N8jlokzRDdQIHlQMPHnEtBsSXYX34,8016
+dbos/_migration.py,sha256=wJrSTYerlkYDFYmkTqo4a7n4WaKmqXh8BdkUEzgSEQQ,10898
 dbos/_outcome.py,sha256=Kz3aL7517q9UEFTx3Cq9zzztjWyWVOx_08fZyHo9dvg,7035
 dbos/_queue.py,sha256=0kJTPwXy3nZ4Epzt-lHky9M9S4L31645drPGFR8fIJY,4854
 dbos/_recovery.py,sha256=K-wlFhdf4yGRm6cUzyhcTjQUS0xp2T5rdNMLiiBErYg,2882
@@ -52,7 +52,9 @@ dbos/_schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_schemas/application_database.py,sha256=SypAS9l9EsaBHFn9FR8jmnqt01M74d9AF1AMa4m2hhI,1040
 dbos/_schemas/system_database.py,sha256=-dAKk-_Y3vzbpLT4ei-sIrBQgFyQiwPj1enZb1TYc8I,4943
 dbos/_serialization.py,sha256=bWuwhXSQcGmiazvhJHA5gwhrRWxtmFmcCFQSDJnqqkU,3666
-dbos/_sys_db.py,sha256=YLoiAUiY1-kaGOVnJxoY_dVI2QeR50m_7ijlIdZYWFk,84768
+dbos/_sys_db.py,sha256=zlAJ3YPLuzN8Ju32_lfhsq3-MxnH4V45Rf6X9Imxqf0,81838
+dbos/_sys_db_postgres.py,sha256=WcG-f1CUzUNBGEOjqKEp6DDraN63jTnJ6CAfieCcxOs,7555
+dbos/_sys_db_sqlite.py,sha256=58Fv7paGloCmpXdWZVCEpcZFvHWLuEkw4XBKCWprGsA,7500
 dbos/_templates/dbos-db-starter/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
 dbos/_templates/dbos-db-starter/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_templates/dbos-db-starter/__package/main.py.dbos,sha256=aQnBPSSQpkB8ERfhf7gB7P9tsU6OPKhZscfeh0yiaD8,2702
@@ -64,13 +66,13 @@ dbos/_templates/dbos-db-starter/migrations/script.py.mako,sha256=MEqL-2qATlST9TA
 dbos/_templates/dbos-db-starter/migrations/versions/2024_07_31_180642_init.py,sha256=MpS7LGaJS0CpvsjhfDkp9EJqvMvVCjRPfUp4c0aE2ys,941
 dbos/_templates/dbos-db-starter/start_postgres_docker.py,sha256=lQVLlYO5YkhGPEgPqwGc7Y8uDKse9HsWv5fynJEFJHM,1681
 dbos/_tracer.py,sha256=8aOAVTBnj2q9DcOb5KJCfo56CVZ1ZvsWBscaNlIX-7k,3150
-dbos/_utils.py,sha256=uywq1QrjMwy17btjxW4bES49povlQwYwYbvKwMT6C2U,1575
+dbos/_utils.py,sha256=ZdoM1MDbHnlJrh31zfhp3iX62bAxK1kyvMwXnltC_84,1779
 dbos/_workflow_commands.py,sha256=EmmAaQfRWeOZm_WPTznuU-O3he3jiSzzT9VpYrhxugE,4835
 dbos/cli/_github_init.py,sha256=Y_bDF9gfO2jB1id4FV5h1oIxEJRWyqVjhb7bNEa5nQ0,3224
 dbos/cli/_template_init.py,sha256=7JBcpMqP1r2mfCnvWatu33z8ctEGHJarlZYKgB83cXE,2972
-dbos/cli/cli.py,sha256=btUbl0L_1cf46W8z0Hi6nPBCLaSqY9I4c1eZCG7obow,23128
-dbos/cli/migration.py,sha256=eI0sc0vYq2iUP3cBHPfTa6WHCyDBr8ld9nRxEZZzFrU,3316
+dbos/cli/cli.py,sha256=basgruBuvOMIb5nrsa1cVKOOYvSx_Lqx3M51P0d6NEU,22698
+dbos/cli/migration.py,sha256=5GiyagLZkyVvDz3StYxtFdkFoKFCmh6eSXjzsIGhZ_A,3330
 dbos/dbos-config.schema.json,sha256=CjaspeYmOkx6Ip_pcxtmfXJTn_YGdSx_0pcPBF7KZmo,6060
 dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
 version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
-dbos-1.13.0a3.dist-info/RECORD,,
+dbos-1.13.0a5.dist-info/RECORD,,