dagster-postgres 0.26.1__py3-none-any.whl → 0.27.15__py3-none-any.whl

dagster_postgres/__init__.py

@@ -1,4 +1,4 @@
-from dagster._core.libraries import DagsterLibraryRegistry
+from dagster_shared.libraries import DagsterLibraryRegistry
 
 from dagster_postgres.event_log import PostgresEventLogStorage
 from dagster_postgres.run_storage import PostgresRunStorage

dagster_postgres/event_log/event_log.py

@@ -112,12 +112,15 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
                 SqlEventLogStorageMetadata.create_all(conn)
                 stamp_alembic_rev(pg_alembic_config(__file__), conn)
 
-    def optimize_for_webserver(self, statement_timeout: int, pool_recycle: int) -> None:
+    def optimize_for_webserver(
+        self, statement_timeout: int, pool_recycle: int, max_overflow: int
+    ) -> None:
         # When running in dagster-webserver, hold an open connection and set statement_timeout
         kwargs = {
             "isolation_level": "AUTOCOMMIT",
             "pool_size": 1,
             "pool_recycle": pool_recycle,
+            "max_overflow": max_overflow,
         }
         existing_options = self._engine.url.query.get("options")
         if existing_options:
@@ -173,7 +176,6 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
             event (EventLogEntry): The event to store.
         """
         check.inst_param(event, "event", EventLogEntry)
-
         insert_event_statement = self.prepare_insert_event(event)  # from SqlEventLogStorage.py
         with self._connect() as conn:
             result = conn.execute(
@@ -209,25 +211,45 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
             self.store_asset_check_event(event, event_id)
 
     def store_event_batch(self, events: Sequence[EventLogEntry]) -> None:
+        from dagster import DagsterEventType
+
         check.sequence_param(events, "event", of_type=EventLogEntry)
 
+        event_types = {event.get_dagster_event().event_type for event in events}
+
         check.invariant(
-            all(event.get_dagster_event().event_type in BATCH_WRITABLE_EVENTS for event in events),
+            all(event_type in BATCH_WRITABLE_EVENTS for event_type in event_types),
             f"{BATCH_WRITABLE_EVENTS} are the only currently supported events for batch writes.",
         )
+        events = [
+            event
+            for event in events
+            if not event.get_dagster_event().is_asset_failed_to_materialize
+        ]
+        if len(events) == 0:
+            return
 
-        insert_event_statement = self.prepare_insert_event_batch(events)
-        with self._connect() as conn:
-            result = conn.execute(insert_event_statement.returning(SqlEventLogStorageTable.c.id))
-            event_ids = [cast(int, row[0]) for row in result.fetchall()]
+        if event_types == {DagsterEventType.ASSET_MATERIALIZATION} or event_types == {
+            DagsterEventType.ASSET_OBSERVATION
+        }:
+            insert_event_statement = self.prepare_insert_event_batch(events)
+            with self._connect() as conn:
+                result = conn.execute(
+                    insert_event_statement.returning(SqlEventLogStorageTable.c.id)
+                )
+                event_ids = [cast("int", row[0]) for row in result.fetchall()]
 
-        # We only update the asset table with the last event
-        self.store_asset_event(events[-1], event_ids[-1])
+            # We only update the asset table with the last event
+            self.store_asset_event(events[-1], event_ids[-1])
 
-        if any(event_id is None for event_id in event_ids):
-            raise DagsterInvariantViolationError("Cannot store asset event tags for null event id.")
+            if any(event_id is None for event_id in event_ids):
+                raise DagsterInvariantViolationError(
+                    "Cannot store asset event tags for null event id."
+                )
 
-        self.store_asset_event_tags(events, event_ids)
+            self.store_asset_event_tags(events, event_ids)
+        else:
+            return super().store_event_batch(events)
 
     def store_asset_event(self, event: EventLogEntry, event_id: int) -> None:
         check.inst_param(event, "event", EventLogEntry)

dagster_postgres/run_storage/run_storage.py

@@ -108,12 +108,15 @@ class PostgresRunStorage(SqlRunStorage, ConfigurableClass):
                 # This revision may be shared by any other dagster storage classes using the same DB
                 stamp_alembic_rev(pg_alembic_config(__file__), conn)
 
-    def optimize_for_webserver(self, statement_timeout: int, pool_recycle: int) -> None:
+    def optimize_for_webserver(
+        self, statement_timeout: int, pool_recycle: int, max_overflow: int
+    ) -> None:
         # When running in dagster-webserver, hold an open connection and set statement_timeout
         kwargs = {
             "isolation_level": "AUTOCOMMIT",
             "pool_size": 1,
             "pool_recycle": pool_recycle,
+            "max_overflow": max_overflow,
         }
         existing_options = self._engine.url.query.get("options")
         if existing_options:
@@ -134,7 +137,7 @@ class PostgresRunStorage(SqlRunStorage, ConfigurableClass):
         return pg_config()
 
     @classmethod
-    def from_config_value(
+    def from_config_value(  # pyright: ignore[reportIncompatibleMethodOverride]
         cls, inst_data: Optional[ConfigurableClassData], config_value: PostgresStorageConfig
     ):
         return PostgresRunStorage(

dagster_postgres/schedule_storage/schedule_storage.py

@@ -104,12 +104,15 @@ class PostgresScheduleStorage(SqlScheduleStorage, ConfigurableClass):
         self.migrate()
         self.optimize()
 
-    def optimize_for_webserver(self, statement_timeout: int, pool_recycle: int) -> None:
+    def optimize_for_webserver(
+        self, statement_timeout: int, pool_recycle: int, max_overflow: int
+    ) -> None:
         # When running in dagster-webserver, hold an open connection and set statement_timeout
         kwargs = {
             "isolation_level": "AUTOCOMMIT",
             "pool_size": 1,
             "pool_recycle": pool_recycle,
+            "max_overflow": max_overflow,
         }
         existing_options = self._engine.url.query.get("options")
         if existing_options:
@@ -130,7 +133,7 @@ class PostgresScheduleStorage(SqlScheduleStorage, ConfigurableClass):
         return pg_config()
 
     @classmethod
-    def from_config_value(
+    def from_config_value(  # pyright: ignore[reportIncompatibleMethodOverride]
         cls, inst_data: Optional[ConfigurableClassData], config_value: PostgresStorageConfig
     ) -> "PostgresScheduleStorage":
         return PostgresScheduleStorage(

dagster_postgres/storage.py

@@ -59,7 +59,7 @@ class DagsterPostgresStorage(DagsterStorage, ConfigurableClass):
         return pg_config()
 
     @classmethod
-    def from_config_value(
+    def from_config_value(  # pyright: ignore[reportIncompatibleMethodOverride]
         cls, inst_data: Optional[ConfigurableClassData], config_value: PostgresStorageConfig
     ) -> "DagsterPostgresStorage":
         return DagsterPostgresStorage(

dagster_postgres/test_fixtures/__init__.py

@@ -0,0 +1,75 @@
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+from dagster._core.test_utils import instance_for_test
+from dagster._utils.merger import merge_dicts
+from dagster_test.fixtures import docker_compose_cm, network_name_from_yml
+
+from dagster_postgres.utils import get_conn_string, wait_for_connection
+
+compose_file = Path(__file__).parent / "docker-compose.yml"
+
+
+@pytest.fixture(scope="session")
+def postgres_network():
+    yield network_name_from_yml(compose_file)
+
+
+@pytest.fixture(scope="session")
+def postgres_hostname():
+    with docker_compose_cm(docker_compose_yml=compose_file) as hostnames:
+        yield hostnames["postgres"]
+
+
+@pytest.fixture(scope="session")
+def postgres_conn_str(postgres_hostname):
+    conn_str = get_conn_string(
+        username="test",
+        password="test",
+        hostname=postgres_hostname,
+        db_name="test",
+        params=dict(connect_timeout=5),
+    )
+    wait_for_connection(
+        conn_str,
+        retry_limit=10,
+        retry_wait=3,
+    )
+
+    yield conn_str
+
+
+@pytest.fixture
+def postgres_instance(postgres_conn_str):
+    @contextmanager
+    def _instance(overrides=None):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            with instance_for_test(
+                temp_dir=temp_dir,
+                overrides=merge_dicts(
+                    {
+                        "run_storage": {
+                            "module": "dagster_postgres.run_storage.run_storage",
+                            "class": "PostgresRunStorage",
+                            "config": {"postgres_url": postgres_conn_str},
+                        },
+                        "event_log_storage": {
+                            "module": "dagster_postgres.event_log.event_log",
+                            "class": "PostgresEventLogStorage",
+                            "config": {"postgres_url": postgres_conn_str},
+                        },
+                        "schedule_storage": {
+                            "module": "dagster_postgres.schedule_storage.schedule_storage",
+                            "class": "PostgresScheduleStorage",
+                            "config": {"postgres_url": postgres_conn_str},
+                        },
+                    },
+                    overrides if overrides else {},
+                ),
+            ) as instance:
+                instance.wipe()
+                yield instance
+
+    return _instance

dagster_postgres/test_fixtures/docker-compose.yml

@@ -0,0 +1,10 @@
+services:
+  postgres:
+    image: postgres:16
+    container_name: postgres
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: "test"
+      POSTGRES_USER: "test"
+      POSTGRES_DB: "test"

dagster_postgres/version.py

@@ -1 +1 @@
-__version__ = "0.26.1"
+__version__ = "0.27.15"

{dagster_postgres-0.26.1.dist-info → dagster_postgres-0.27.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-postgres
-Version: 0.26.1
+Version: 0.27.15
 Summary: A Dagster integration for postgres
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-postgres
 Author: Dagster Labs
@@ -10,10 +10,19 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.13
+Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster ==1.10.1
+Requires-Dist: dagster==1.11.15
 Requires-Dist: psycopg2-binary
-
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_postgres-0.27.15.dist-info/RECORD

@@ -0,0 +1,19 @@
+dagster_postgres/__init__.py,sha256=rZfjhBDGC_SIGnKuiSCzi7D0xD36mAnchDLRL-n1Tmk,561
+dagster_postgres/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagster_postgres/storage.py,sha256=CiPA773j0BQvPtCEz5vVqKjap9h0ODAxR_R_CvbI-bI,4310
+dagster_postgres/utils.py,sha256=AjuUYZ3y4Q3oaa-6I2zd6SNwl66FE8cJ_e2rIKrI9IQ,5930
+dagster_postgres/version.py,sha256=nh4rkWKwcS4_2NEvEKWozrztgCUEn_1ODIolwNpGd8g,24
+dagster_postgres/alembic/alembic.ini,sha256=GovyDEhu_6HvkWV6txqjdDBOe4BseSM0YDWGxXM5_cA,986
+dagster_postgres/event_log/__init__.py,sha256=wRcUR-StRNrPCqpEzi0MRY8b-r_TEWV17OsEynFqlLs,100
+dagster_postgres/event_log/event_log.py,sha256=w8_ZsvDb0vt7qDVSS912jmNRpLDI6F0KQrwribyAmwM,16145
+dagster_postgres/run_storage/__init__.py,sha256=oW_546mJ5K-e-RF0Ou7r-4fHWxFthHgPPhWxklsVK1g,94
+dagster_postgres/run_storage/run_storage.py,sha256=VMXT5sb5n04ViWOYj2eOoZqOk78hqL38J6CuikM79No,9757
+dagster_postgres/schedule_storage/__init__.py,sha256=-jW-1S4Xf5Ew-cz-DjKjU5sVs9EEly_2ELMLOXTewv0,123
+dagster_postgres/schedule_storage/schedule_storage.py,sha256=iPKcvnj1eEeG5LH73GoFvoS3hmE8MFddiFyf7QoKZ3g,8826
+dagster_postgres/test_fixtures/__init__.py,sha256=ZIcbGYa_CMmVF0YkuiX4uI3wRjBke_2VE8QXKcji0Y4,2447
+dagster_postgres/test_fixtures/docker-compose.yml,sha256=hp2VTnENYK6CL2Yae3IsktssahPFQiHyhpNcG36CivM,208
+dagster_postgres-0.27.15.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_postgres-0.27.15.dist-info/METADATA,sha256=_4kuw4rRwvg4462KMMaJ5KzTNVwInGJQvzfgiZJU7rs,945
+dagster_postgres-0.27.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_postgres-0.27.15.dist-info/top_level.txt,sha256=lScMtAEKDX1yIv2tGa1nzntBa0HEStfWPfCwD8FWlHk,17
+dagster_postgres-0.27.15.dist-info/RECORD,,

{dagster_postgres-0.26.1.dist-info → dagster_postgres-0.27.15.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{dagster_postgres-0.26.1.dist-info → dagster_postgres-0.27.15.dist-info/licenses}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Dagster Labs, Inc".
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

dagster_postgres-0.26.1.dist-info/RECORD

@@ -1,17 +0,0 @@
-dagster_postgres/__init__.py,sha256=h8rM1BA27G8jSAak_WnjfZ_dqyz79x9rfTeroUveu_w,560
-dagster_postgres/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
-dagster_postgres/storage.py,sha256=whjOJeT4-W41bZ8xDBg77NGSVJPQzc33IsxpmGNdpVM,4257
-dagster_postgres/utils.py,sha256=AjuUYZ3y4Q3oaa-6I2zd6SNwl66FE8cJ_e2rIKrI9IQ,5930
-dagster_postgres/version.py,sha256=u0eTmljUU0kO8AAW-e1ESQ49mK2SuhpCy7eCliBLlDU,23
-dagster_postgres/alembic/alembic.ini,sha256=GovyDEhu_6HvkWV6txqjdDBOe4BseSM0YDWGxXM5_cA,986
-dagster_postgres/event_log/__init__.py,sha256=wRcUR-StRNrPCqpEzi0MRY8b-r_TEWV17OsEynFqlLs,100
-dagster_postgres/event_log/event_log.py,sha256=Y73l4Vf_Tp3HXeEI7WdIpdjDmFTFbMT8HDL62NFg3qI,15430
-dagster_postgres/run_storage/__init__.py,sha256=oW_546mJ5K-e-RF0Ou7r-4fHWxFthHgPPhWxklsVK1g,94
-dagster_postgres/run_storage/run_storage.py,sha256=dX_x8xZGcz4byCC-Jette87rAGTQAuNpKgkXKIhxgmc,9629
-dagster_postgres/schedule_storage/__init__.py,sha256=-jW-1S4Xf5Ew-cz-DjKjU5sVs9EEly_2ELMLOXTewv0,123
-dagster_postgres/schedule_storage/schedule_storage.py,sha256=6DSsUFb4nnEvzw0AtY64oBJU5mHsNlB35skwi_gKIZE,8698
-dagster_postgres-0.26.1.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_postgres-0.26.1.dist-info/METADATA,sha256=Z4hkEG7zpC3gvq3pwrua5DJqgzePHT1tMwPrHbgdOio,713
-dagster_postgres-0.26.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_postgres-0.26.1.dist-info/top_level.txt,sha256=lScMtAEKDX1yIv2tGa1nzntBa0HEStfWPfCwD8FWlHk,17
-dagster_postgres-0.26.1.dist-info/RECORD,,