dagster-postgres 0.23.7__py3-none-any.whl → 0.28.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_postgres/__init__.py +6 -6
- dagster_postgres/event_log/__init__.py +1 -1
- dagster_postgres/event_log/event_log.py +52 -29
- dagster_postgres/run_storage/__init__.py +1 -1
- dagster_postgres/run_storage/run_storage.py +27 -23
- dagster_postgres/schedule_storage/__init__.py +3 -1
- dagster_postgres/schedule_storage/schedule_storage.py +28 -21
- dagster_postgres/storage.py +5 -5
- dagster_postgres/test_fixtures/__init__.py +75 -0
- dagster_postgres/test_fixtures/docker-compose.yml +10 -0
- dagster_postgres/utils.py +6 -3
- dagster_postgres/version.py +1 -1
- {dagster_postgres-0.23.7.dist-info → dagster_postgres-0.28.11.dist-info}/METADATA +15 -7
- dagster_postgres-0.28.11.dist-info/RECORD +19 -0
- {dagster_postgres-0.23.7.dist-info → dagster_postgres-0.28.11.dist-info}/WHEEL +1 -1
- {dagster_postgres-0.23.7.dist-info → dagster_postgres-0.28.11.dist-info/licenses}/LICENSE +1 -1
- dagster_postgres-0.23.7.dist-info/RECORD +0 -17
- {dagster_postgres-0.23.7.dist-info → dagster_postgres-0.28.11.dist-info}/top_level.txt +0 -0
dagster_postgres/__init__.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
from
|
|
1
|
+
from dagster_shared.libraries import DagsterLibraryRegistry
|
|
2
2
|
|
|
3
|
-
from .event_log import PostgresEventLogStorage
|
|
4
|
-
from .run_storage import PostgresRunStorage
|
|
5
|
-
from .schedule_storage import PostgresScheduleStorage
|
|
6
|
-
from .storage import DagsterPostgresStorage
|
|
7
|
-
from .version import __version__
|
|
3
|
+
from dagster_postgres.event_log import PostgresEventLogStorage
|
|
4
|
+
from dagster_postgres.run_storage import PostgresRunStorage
|
|
5
|
+
from dagster_postgres.schedule_storage import PostgresScheduleStorage
|
|
6
|
+
from dagster_postgres.storage import DagsterPostgresStorage
|
|
7
|
+
from dagster_postgres.version import __version__
|
|
8
8
|
|
|
9
9
|
DagsterLibraryRegistry.register("dagster-postgres", __version__)
|
|
10
10
|
__all__ = [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
from .event_log import PostgresEventLogStorage as PostgresEventLogStorage
|
|
1
|
+
from dagster_postgres.event_log.event_log import PostgresEventLogStorage as PostgresEventLogStorage
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
1
2
|
from contextlib import contextmanager
|
|
2
|
-
from typing import Any, ContextManager,
|
|
3
|
+
from typing import Any, ContextManager, Optional, cast # noqa: UP035
|
|
3
4
|
|
|
4
5
|
import dagster._check as check
|
|
5
6
|
import sqlalchemy as db
|
|
@@ -30,15 +31,16 @@ from dagster._core.storage.sql import (
|
|
|
30
31
|
)
|
|
31
32
|
from dagster._core.storage.sqlalchemy_compat import db_select
|
|
32
33
|
from dagster._serdes import ConfigurableClass, ConfigurableClassData, deserialize_value
|
|
34
|
+
from sqlalchemy import event
|
|
33
35
|
from sqlalchemy.engine import Connection
|
|
34
36
|
|
|
35
|
-
from
|
|
37
|
+
from dagster_postgres.utils import (
|
|
36
38
|
create_pg_connection,
|
|
37
39
|
pg_alembic_config,
|
|
38
|
-
pg_statement_timeout,
|
|
39
40
|
pg_url_from_config,
|
|
40
41
|
retry_pg_connection_fn,
|
|
41
42
|
retry_pg_creation_fn,
|
|
43
|
+
set_pg_statement_timeout,
|
|
42
44
|
)
|
|
43
45
|
|
|
44
46
|
CHANNEL_NAME = "run_events"
|
|
@@ -110,20 +112,24 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
|
|
|
110
112
|
SqlEventLogStorageMetadata.create_all(conn)
|
|
111
113
|
stamp_alembic_rev(pg_alembic_config(__file__), conn)
|
|
112
114
|
|
|
113
|
-
def optimize_for_webserver(
|
|
115
|
+
def optimize_for_webserver(
|
|
116
|
+
self, statement_timeout: int, pool_recycle: int, max_overflow: int
|
|
117
|
+
) -> None:
|
|
114
118
|
# When running in dagster-webserver, hold an open connection and set statement_timeout
|
|
119
|
+
kwargs = {
|
|
120
|
+
"isolation_level": "AUTOCOMMIT",
|
|
121
|
+
"pool_size": 1,
|
|
122
|
+
"pool_recycle": pool_recycle,
|
|
123
|
+
"max_overflow": max_overflow,
|
|
124
|
+
}
|
|
115
125
|
existing_options = self._engine.url.query.get("options")
|
|
116
|
-
timeout_option = pg_statement_timeout(statement_timeout)
|
|
117
126
|
if existing_options:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
pool_size=1,
|
|
125
|
-
connect_args={"options": options},
|
|
126
|
-
pool_recycle=pool_recycle,
|
|
127
|
+
kwargs["connect_args"] = {"options": existing_options}
|
|
128
|
+
self._engine = create_engine(self.postgres_url, **kwargs)
|
|
129
|
+
event.listen(
|
|
130
|
+
self._engine,
|
|
131
|
+
"connect",
|
|
132
|
+
lambda connection, _: set_pg_statement_timeout(connection, statement_timeout),
|
|
127
133
|
)
|
|
128
134
|
|
|
129
135
|
def upgrade(self) -> None:
|
|
@@ -170,7 +176,6 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
|
|
|
170
176
|
event (EventLogEntry): The event to store.
|
|
171
177
|
"""
|
|
172
178
|
check.inst_param(event, "event", EventLogEntry)
|
|
173
|
-
|
|
174
179
|
insert_event_statement = self.prepare_insert_event(event) # from SqlEventLogStorage.py
|
|
175
180
|
with self._connect() as conn:
|
|
176
181
|
result = conn.execute(
|
|
@@ -206,25 +211,45 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
|
|
|
206
211
|
self.store_asset_check_event(event, event_id)
|
|
207
212
|
|
|
208
213
|
def store_event_batch(self, events: Sequence[EventLogEntry]) -> None:
|
|
214
|
+
from dagster import DagsterEventType
|
|
215
|
+
|
|
209
216
|
check.sequence_param(events, "event", of_type=EventLogEntry)
|
|
210
217
|
|
|
218
|
+
event_types = {event.get_dagster_event().event_type for event in events}
|
|
219
|
+
|
|
211
220
|
check.invariant(
|
|
212
|
-
all(
|
|
221
|
+
all(event_type in BATCH_WRITABLE_EVENTS for event_type in event_types),
|
|
213
222
|
f"{BATCH_WRITABLE_EVENTS} are the only currently supported events for batch writes.",
|
|
214
223
|
)
|
|
224
|
+
events = [
|
|
225
|
+
event
|
|
226
|
+
for event in events
|
|
227
|
+
if not event.get_dagster_event().is_asset_failed_to_materialize
|
|
228
|
+
]
|
|
229
|
+
if len(events) == 0:
|
|
230
|
+
return
|
|
215
231
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
232
|
+
if event_types == {DagsterEventType.ASSET_MATERIALIZATION} or event_types == {
|
|
233
|
+
DagsterEventType.ASSET_OBSERVATION
|
|
234
|
+
}:
|
|
235
|
+
insert_event_statement = self.prepare_insert_event_batch(events)
|
|
236
|
+
with self._connect() as conn:
|
|
237
|
+
result = conn.execute(
|
|
238
|
+
insert_event_statement.returning(SqlEventLogStorageTable.c.id)
|
|
239
|
+
)
|
|
240
|
+
event_ids = [cast("int", row[0]) for row in result.fetchall()]
|
|
220
241
|
|
|
221
|
-
|
|
222
|
-
|
|
242
|
+
# We only update the asset table with the last event
|
|
243
|
+
self.store_asset_event(events[-1], event_ids[-1])
|
|
223
244
|
|
|
224
|
-
|
|
225
|
-
|
|
245
|
+
if any(event_id is None for event_id in event_ids):
|
|
246
|
+
raise DagsterInvariantViolationError(
|
|
247
|
+
"Cannot store asset event tags for null event id."
|
|
248
|
+
)
|
|
226
249
|
|
|
227
|
-
|
|
250
|
+
self.store_asset_event_tags(events, event_ids)
|
|
251
|
+
else:
|
|
252
|
+
return super().store_event_batch(events)
|
|
228
253
|
|
|
229
254
|
def store_asset_event(self, event: EventLogEntry, event_id: int) -> None:
|
|
230
255
|
check.inst_param(event, "event", EventLogEntry)
|
|
@@ -323,13 +348,11 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
|
|
|
323
348
|
|
|
324
349
|
def has_secondary_index(self, name: str) -> bool:
|
|
325
350
|
if name not in self._secondary_index_cache:
|
|
326
|
-
self._secondary_index_cache[name] = super(
|
|
327
|
-
PostgresEventLogStorage, self
|
|
328
|
-
).has_secondary_index(name)
|
|
351
|
+
self._secondary_index_cache[name] = super().has_secondary_index(name)
|
|
329
352
|
return self._secondary_index_cache[name]
|
|
330
353
|
|
|
331
354
|
def enable_secondary_index(self, name: str) -> None:
|
|
332
|
-
super(
|
|
355
|
+
super().enable_secondary_index(name)
|
|
333
356
|
if name in self._secondary_index_cache:
|
|
334
357
|
del self._secondary_index_cache[name]
|
|
335
358
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
from .run_storage import PostgresRunStorage as PostgresRunStorage
|
|
1
|
+
from dagster_postgres.run_storage.run_storage import PostgresRunStorage as PostgresRunStorage
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import zlib
|
|
2
|
-
from
|
|
2
|
+
from collections.abc import Mapping
|
|
3
|
+
from typing import ContextManager, Optional # noqa: UP035
|
|
3
4
|
|
|
4
5
|
import dagster._check as check
|
|
5
6
|
import sqlalchemy as db
|
|
@@ -24,16 +25,17 @@ from dagster._core.storage.sql import (
|
|
|
24
25
|
)
|
|
25
26
|
from dagster._daemon.types import DaemonHeartbeat
|
|
26
27
|
from dagster._serdes import ConfigurableClass, ConfigurableClassData, serialize_value
|
|
27
|
-
from dagster.
|
|
28
|
+
from dagster._time import datetime_from_timestamp
|
|
29
|
+
from sqlalchemy import event
|
|
28
30
|
from sqlalchemy.engine import Connection
|
|
29
31
|
|
|
30
|
-
from
|
|
32
|
+
from dagster_postgres.utils import (
|
|
31
33
|
create_pg_connection,
|
|
32
34
|
pg_alembic_config,
|
|
33
|
-
pg_statement_timeout,
|
|
34
35
|
pg_url_from_config,
|
|
35
36
|
retry_pg_connection_fn,
|
|
36
37
|
retry_pg_creation_fn,
|
|
38
|
+
set_pg_statement_timeout,
|
|
37
39
|
)
|
|
38
40
|
|
|
39
41
|
|
|
@@ -106,20 +108,24 @@ class PostgresRunStorage(SqlRunStorage, ConfigurableClass):
|
|
|
106
108
|
# This revision may be shared by any other dagster storage classes using the same DB
|
|
107
109
|
stamp_alembic_rev(pg_alembic_config(__file__), conn)
|
|
108
110
|
|
|
109
|
-
def optimize_for_webserver(
|
|
110
|
-
|
|
111
|
+
def optimize_for_webserver(
|
|
112
|
+
self, statement_timeout: int, pool_recycle: int, max_overflow: int
|
|
113
|
+
) -> None:
|
|
114
|
+
# When running in dagster-webserver, hold an open connection and set statement_timeout
|
|
115
|
+
kwargs = {
|
|
116
|
+
"isolation_level": "AUTOCOMMIT",
|
|
117
|
+
"pool_size": 1,
|
|
118
|
+
"pool_recycle": pool_recycle,
|
|
119
|
+
"max_overflow": max_overflow,
|
|
120
|
+
}
|
|
111
121
|
existing_options = self._engine.url.query.get("options")
|
|
112
|
-
timeout_option = pg_statement_timeout(statement_timeout)
|
|
113
122
|
if existing_options:
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
pool_size=1,
|
|
121
|
-
connect_args={"options": options},
|
|
122
|
-
pool_recycle=pool_recycle,
|
|
123
|
+
kwargs["connect_args"] = {"options": existing_options}
|
|
124
|
+
self._engine = create_engine(self.postgres_url, **kwargs)
|
|
125
|
+
event.listen(
|
|
126
|
+
self._engine,
|
|
127
|
+
"connect",
|
|
128
|
+
lambda connection, _: set_pg_statement_timeout(connection, statement_timeout),
|
|
123
129
|
)
|
|
124
130
|
|
|
125
131
|
@property
|
|
@@ -131,7 +137,7 @@ class PostgresRunStorage(SqlRunStorage, ConfigurableClass):
|
|
|
131
137
|
return pg_config()
|
|
132
138
|
|
|
133
139
|
@classmethod
|
|
134
|
-
def from_config_value(
|
|
140
|
+
def from_config_value( # pyright: ignore[reportIncompatibleMethodOverride]
|
|
135
141
|
cls, inst_data: Optional[ConfigurableClassData], config_value: PostgresStorageConfig
|
|
136
142
|
):
|
|
137
143
|
return PostgresRunStorage(
|
|
@@ -162,13 +168,11 @@ class PostgresRunStorage(SqlRunStorage, ConfigurableClass):
|
|
|
162
168
|
|
|
163
169
|
def has_built_index(self, migration_name: str) -> bool:
|
|
164
170
|
if migration_name not in self._index_migration_cache:
|
|
165
|
-
self._index_migration_cache[migration_name] = super(
|
|
166
|
-
PostgresRunStorage, self
|
|
167
|
-
).has_built_index(migration_name)
|
|
171
|
+
self._index_migration_cache[migration_name] = super().has_built_index(migration_name)
|
|
168
172
|
return self._index_migration_cache[migration_name]
|
|
169
173
|
|
|
170
174
|
def mark_index_built(self, migration_name: str) -> None:
|
|
171
|
-
super(
|
|
175
|
+
super().mark_index_built(migration_name)
|
|
172
176
|
if migration_name in self._index_migration_cache:
|
|
173
177
|
del self._index_migration_cache[migration_name]
|
|
174
178
|
|
|
@@ -178,7 +182,7 @@ class PostgresRunStorage(SqlRunStorage, ConfigurableClass):
|
|
|
178
182
|
conn.execute(
|
|
179
183
|
db_dialects.postgresql.insert(DaemonHeartbeatsTable)
|
|
180
184
|
.values(
|
|
181
|
-
timestamp=
|
|
185
|
+
timestamp=datetime_from_timestamp(daemon_heartbeat.timestamp),
|
|
182
186
|
daemon_type=daemon_heartbeat.daemon_type,
|
|
183
187
|
daemon_id=daemon_heartbeat.daemon_id,
|
|
184
188
|
body=serialize_value(daemon_heartbeat),
|
|
@@ -186,7 +190,7 @@ class PostgresRunStorage(SqlRunStorage, ConfigurableClass):
|
|
|
186
190
|
.on_conflict_do_update(
|
|
187
191
|
index_elements=[DaemonHeartbeatsTable.c.daemon_type],
|
|
188
192
|
set_={
|
|
189
|
-
"timestamp":
|
|
193
|
+
"timestamp": datetime_from_timestamp(daemon_heartbeat.timestamp),
|
|
190
194
|
"daemon_id": daemon_heartbeat.daemon_id,
|
|
191
195
|
"body": serialize_value(daemon_heartbeat),
|
|
192
196
|
},
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
from
|
|
1
|
+
from collections.abc import Sequence
|
|
2
|
+
from typing import ContextManager, Optional # noqa: UP035
|
|
2
3
|
|
|
3
4
|
import dagster._check as check
|
|
4
|
-
import pendulum
|
|
5
5
|
import sqlalchemy as db
|
|
6
6
|
import sqlalchemy.dialects as db_dialects
|
|
7
7
|
import sqlalchemy.pool as db_pool
|
|
8
8
|
from dagster._config.config_schema import UserConfigSchema
|
|
9
|
-
from dagster._core.definitions.
|
|
10
|
-
|
|
9
|
+
from dagster._core.definitions.asset_key import EntityKey
|
|
10
|
+
from dagster._core.definitions.declarative_automation.serialized_objects import (
|
|
11
|
+
AutomationConditionEvaluationWithRunIds,
|
|
11
12
|
)
|
|
12
13
|
from dagster._core.scheduler.instigation import InstigatorState
|
|
13
14
|
from dagster._core.storage.config import PostgresStorageConfig, pg_config
|
|
@@ -24,15 +25,17 @@ from dagster._core.storage.sql import (
|
|
|
24
25
|
stamp_alembic_rev,
|
|
25
26
|
)
|
|
26
27
|
from dagster._serdes import ConfigurableClass, ConfigurableClassData, serialize_value
|
|
28
|
+
from dagster._time import get_current_datetime
|
|
29
|
+
from sqlalchemy import event
|
|
27
30
|
from sqlalchemy.engine import Connection
|
|
28
31
|
|
|
29
|
-
from
|
|
32
|
+
from dagster_postgres.utils import (
|
|
30
33
|
create_pg_connection,
|
|
31
34
|
pg_alembic_config,
|
|
32
|
-
pg_statement_timeout,
|
|
33
35
|
pg_url_from_config,
|
|
34
36
|
retry_pg_connection_fn,
|
|
35
37
|
retry_pg_creation_fn,
|
|
38
|
+
set_pg_statement_timeout,
|
|
36
39
|
)
|
|
37
40
|
|
|
38
41
|
|
|
@@ -101,20 +104,24 @@ class PostgresScheduleStorage(SqlScheduleStorage, ConfigurableClass):
|
|
|
101
104
|
self.migrate()
|
|
102
105
|
self.optimize()
|
|
103
106
|
|
|
104
|
-
def optimize_for_webserver(
|
|
107
|
+
def optimize_for_webserver(
|
|
108
|
+
self, statement_timeout: int, pool_recycle: int, max_overflow: int
|
|
109
|
+
) -> None:
|
|
105
110
|
# When running in dagster-webserver, hold an open connection and set statement_timeout
|
|
111
|
+
kwargs = {
|
|
112
|
+
"isolation_level": "AUTOCOMMIT",
|
|
113
|
+
"pool_size": 1,
|
|
114
|
+
"pool_recycle": pool_recycle,
|
|
115
|
+
"max_overflow": max_overflow,
|
|
116
|
+
}
|
|
106
117
|
existing_options = self._engine.url.query.get("options")
|
|
107
|
-
timeout_option = pg_statement_timeout(statement_timeout)
|
|
108
118
|
if existing_options:
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
pool_size=1,
|
|
116
|
-
connect_args={"options": options},
|
|
117
|
-
pool_recycle=pool_recycle,
|
|
119
|
+
kwargs["connect_args"] = {"options": existing_options}
|
|
120
|
+
self._engine = create_engine(self.postgres_url, **kwargs)
|
|
121
|
+
event.listen(
|
|
122
|
+
self._engine,
|
|
123
|
+
"connect",
|
|
124
|
+
lambda connection, _: set_pg_statement_timeout(connection, statement_timeout),
|
|
118
125
|
)
|
|
119
126
|
|
|
120
127
|
@property
|
|
@@ -126,7 +133,7 @@ class PostgresScheduleStorage(SqlScheduleStorage, ConfigurableClass):
|
|
|
126
133
|
return pg_config()
|
|
127
134
|
|
|
128
135
|
@classmethod
|
|
129
|
-
def from_config_value(
|
|
136
|
+
def from_config_value( # pyright: ignore[reportIncompatibleMethodOverride]
|
|
130
137
|
cls, inst_data: Optional[ConfigurableClassData], config_value: PostgresStorageConfig
|
|
131
138
|
) -> "PostgresScheduleStorage":
|
|
132
139
|
return PostgresScheduleStorage(
|
|
@@ -173,7 +180,7 @@ class PostgresScheduleStorage(SqlScheduleStorage, ConfigurableClass):
|
|
|
173
180
|
"status": state.status.value,
|
|
174
181
|
"instigator_type": state.instigator_type.value,
|
|
175
182
|
"instigator_body": serialize_value(state),
|
|
176
|
-
"update_timestamp":
|
|
183
|
+
"update_timestamp": get_current_datetime(),
|
|
177
184
|
},
|
|
178
185
|
)
|
|
179
186
|
)
|
|
@@ -181,7 +188,7 @@ class PostgresScheduleStorage(SqlScheduleStorage, ConfigurableClass):
|
|
|
181
188
|
def add_auto_materialize_asset_evaluations(
|
|
182
189
|
self,
|
|
183
190
|
evaluation_id: int,
|
|
184
|
-
asset_evaluations: Sequence[
|
|
191
|
+
asset_evaluations: Sequence[AutomationConditionEvaluationWithRunIds[EntityKey]],
|
|
185
192
|
):
|
|
186
193
|
if not asset_evaluations:
|
|
187
194
|
return
|
|
@@ -190,7 +197,7 @@ class PostgresScheduleStorage(SqlScheduleStorage, ConfigurableClass):
|
|
|
190
197
|
[
|
|
191
198
|
{
|
|
192
199
|
"evaluation_id": evaluation_id,
|
|
193
|
-
"asset_key": evaluation.
|
|
200
|
+
"asset_key": evaluation.key.to_db_string(),
|
|
194
201
|
"asset_evaluation_body": serialize_value(evaluation),
|
|
195
202
|
"num_requested": evaluation.num_requested,
|
|
196
203
|
}
|
dagster_postgres/storage.py
CHANGED
|
@@ -9,10 +9,10 @@ from dagster._core.storage.runs import RunStorage
|
|
|
9
9
|
from dagster._core.storage.schedules import ScheduleStorage
|
|
10
10
|
from dagster._serdes import ConfigurableClass, ConfigurableClassData
|
|
11
11
|
|
|
12
|
-
from .event_log import PostgresEventLogStorage
|
|
13
|
-
from .run_storage import PostgresRunStorage
|
|
14
|
-
from .schedule_storage import PostgresScheduleStorage
|
|
15
|
-
from .utils import pg_url_from_config
|
|
12
|
+
from dagster_postgres.event_log import PostgresEventLogStorage
|
|
13
|
+
from dagster_postgres.run_storage import PostgresRunStorage
|
|
14
|
+
from dagster_postgres.schedule_storage import PostgresScheduleStorage
|
|
15
|
+
from dagster_postgres.utils import pg_url_from_config
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
class DagsterPostgresStorage(DagsterStorage, ConfigurableClass):
|
|
@@ -59,7 +59,7 @@ class DagsterPostgresStorage(DagsterStorage, ConfigurableClass):
|
|
|
59
59
|
return pg_config()
|
|
60
60
|
|
|
61
61
|
@classmethod
|
|
62
|
-
def from_config_value(
|
|
62
|
+
def from_config_value( # pyright: ignore[reportIncompatibleMethodOverride]
|
|
63
63
|
cls, inst_data: Optional[ConfigurableClassData], config_value: PostgresStorageConfig
|
|
64
64
|
) -> "DagsterPostgresStorage":
|
|
65
65
|
return DagsterPostgresStorage(
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import tempfile
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from dagster._core.test_utils import instance_for_test
|
|
7
|
+
from dagster._utils.merger import merge_dicts
|
|
8
|
+
from dagster_test.fixtures import docker_compose_cm, network_name_from_yml
|
|
9
|
+
|
|
10
|
+
from dagster_postgres.utils import get_conn_string, wait_for_connection
|
|
11
|
+
|
|
12
|
+
compose_file = Path(__file__).parent / "docker-compose.yml"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture(scope="session")
|
|
16
|
+
def postgres_network():
|
|
17
|
+
yield network_name_from_yml(compose_file)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.fixture(scope="session")
|
|
21
|
+
def postgres_hostname():
|
|
22
|
+
with docker_compose_cm(docker_compose_yml=compose_file) as hostnames:
|
|
23
|
+
yield hostnames["postgres"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.fixture(scope="session")
|
|
27
|
+
def postgres_conn_str(postgres_hostname):
|
|
28
|
+
conn_str = get_conn_string(
|
|
29
|
+
username="test",
|
|
30
|
+
password="test",
|
|
31
|
+
hostname=postgres_hostname,
|
|
32
|
+
db_name="test",
|
|
33
|
+
params=dict(connect_timeout=5),
|
|
34
|
+
)
|
|
35
|
+
wait_for_connection(
|
|
36
|
+
conn_str,
|
|
37
|
+
retry_limit=10,
|
|
38
|
+
retry_wait=3,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
yield conn_str
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@pytest.fixture
|
|
45
|
+
def postgres_instance(postgres_conn_str):
|
|
46
|
+
@contextmanager
|
|
47
|
+
def _instance(overrides=None):
|
|
48
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
|
49
|
+
with instance_for_test(
|
|
50
|
+
temp_dir=temp_dir,
|
|
51
|
+
overrides=merge_dicts(
|
|
52
|
+
{
|
|
53
|
+
"run_storage": {
|
|
54
|
+
"module": "dagster_postgres.run_storage.run_storage",
|
|
55
|
+
"class": "PostgresRunStorage",
|
|
56
|
+
"config": {"postgres_url": postgres_conn_str},
|
|
57
|
+
},
|
|
58
|
+
"event_log_storage": {
|
|
59
|
+
"module": "dagster_postgres.event_log.event_log",
|
|
60
|
+
"class": "PostgresEventLogStorage",
|
|
61
|
+
"config": {"postgres_url": postgres_conn_str},
|
|
62
|
+
},
|
|
63
|
+
"schedule_storage": {
|
|
64
|
+
"module": "dagster_postgres.schedule_storage.schedule_storage",
|
|
65
|
+
"class": "PostgresScheduleStorage",
|
|
66
|
+
"config": {"postgres_url": postgres_conn_str},
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
overrides if overrides else {},
|
|
70
|
+
),
|
|
71
|
+
) as instance:
|
|
72
|
+
instance.wipe()
|
|
73
|
+
yield instance
|
|
74
|
+
|
|
75
|
+
return _instance
|
dagster_postgres/utils.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import time
|
|
3
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
3
4
|
from contextlib import contextmanager
|
|
4
|
-
from typing import Any,
|
|
5
|
+
from typing import Any, Optional, TypeVar
|
|
5
6
|
from urllib.parse import quote, urlencode
|
|
6
7
|
|
|
7
8
|
import alembic.config
|
|
@@ -169,6 +170,8 @@ def create_pg_connection(
|
|
|
169
170
|
conn.close()
|
|
170
171
|
|
|
171
172
|
|
|
172
|
-
def
|
|
173
|
+
def set_pg_statement_timeout(conn: psycopg2.extensions.connection, millis: int):
|
|
173
174
|
check.int_param(millis, "millis")
|
|
174
|
-
|
|
175
|
+
with conn:
|
|
176
|
+
with conn.cursor() as curs:
|
|
177
|
+
curs.execute(f"SET statement_timeout = {millis};")
|
dagster_postgres/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.28.11"
|
|
@@ -1,20 +1,28 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-postgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.28.11
|
|
4
4
|
Summary: A Dagster integration for postgres
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-postgres
|
|
6
6
|
Author: Dagster Labs
|
|
7
7
|
Author-email: hello@dagsterlabs.com
|
|
8
8
|
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
14
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
|
-
Requires-Python: >=3.
|
|
16
|
+
Requires-Python: >=3.10,<3.15
|
|
17
17
|
License-File: LICENSE
|
|
18
|
-
Requires-Dist: dagster
|
|
18
|
+
Requires-Dist: dagster==1.12.11
|
|
19
19
|
Requires-Dist: psycopg2-binary
|
|
20
|
-
|
|
20
|
+
Dynamic: author
|
|
21
|
+
Dynamic: author-email
|
|
22
|
+
Dynamic: classifier
|
|
23
|
+
Dynamic: home-page
|
|
24
|
+
Dynamic: license
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
Dynamic: requires-dist
|
|
27
|
+
Dynamic: requires-python
|
|
28
|
+
Dynamic: summary
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
dagster_postgres/__init__.py,sha256=rZfjhBDGC_SIGnKuiSCzi7D0xD36mAnchDLRL-n1Tmk,561
|
|
2
|
+
dagster_postgres/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
3
|
+
dagster_postgres/storage.py,sha256=CiPA773j0BQvPtCEz5vVqKjap9h0ODAxR_R_CvbI-bI,4310
|
|
4
|
+
dagster_postgres/utils.py,sha256=BzkzfQ9t9U9FMCs_3NazAKW7stK0Wrz_nH0KK4NQfi8,5930
|
|
5
|
+
dagster_postgres/version.py,sha256=MlSqB7SmZ8R1F0K_PtG4rMWRGGllC7hEsX-reHAF_8A,24
|
|
6
|
+
dagster_postgres/alembic/alembic.ini,sha256=GovyDEhu_6HvkWV6txqjdDBOe4BseSM0YDWGxXM5_cA,986
|
|
7
|
+
dagster_postgres/event_log/__init__.py,sha256=wRcUR-StRNrPCqpEzi0MRY8b-r_TEWV17OsEynFqlLs,100
|
|
8
|
+
dagster_postgres/event_log/event_log.py,sha256=w8_ZsvDb0vt7qDVSS912jmNRpLDI6F0KQrwribyAmwM,16145
|
|
9
|
+
dagster_postgres/run_storage/__init__.py,sha256=oW_546mJ5K-e-RF0Ou7r-4fHWxFthHgPPhWxklsVK1g,94
|
|
10
|
+
dagster_postgres/run_storage/run_storage.py,sha256=VMXT5sb5n04ViWOYj2eOoZqOk78hqL38J6CuikM79No,9757
|
|
11
|
+
dagster_postgres/schedule_storage/__init__.py,sha256=-jW-1S4Xf5Ew-cz-DjKjU5sVs9EEly_2ELMLOXTewv0,123
|
|
12
|
+
dagster_postgres/schedule_storage/schedule_storage.py,sha256=iPKcvnj1eEeG5LH73GoFvoS3hmE8MFddiFyf7QoKZ3g,8826
|
|
13
|
+
dagster_postgres/test_fixtures/__init__.py,sha256=ZIcbGYa_CMmVF0YkuiX4uI3wRjBke_2VE8QXKcji0Y4,2447
|
|
14
|
+
dagster_postgres/test_fixtures/docker-compose.yml,sha256=hp2VTnENYK6CL2Yae3IsktssahPFQiHyhpNcG36CivM,208
|
|
15
|
+
dagster_postgres-0.28.11.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
|
|
16
|
+
dagster_postgres-0.28.11.dist-info/METADATA,sha256=971I43x8Ltg7CRFX9AILtVaxOlw1PK7fsQ0jwuGdEYo,947
|
|
17
|
+
dagster_postgres-0.28.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
18
|
+
dagster_postgres-0.28.11.dist-info/top_level.txt,sha256=lScMtAEKDX1yIv2tGa1nzntBa0HEStfWPfCwD8FWlHk,17
|
|
19
|
+
dagster_postgres-0.28.11.dist-info/RECORD,,
|
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright
|
|
189
|
+
Copyright 2025 Dagster Labs, Inc.
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
dagster_postgres/__init__.py,sha256=LEcXwiGeuZgIvfDg0PVcRuKjUmg0t9Uslx5nLZ8Yg08,480
|
|
2
|
-
dagster_postgres/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
3
|
-
dagster_postgres/storage.py,sha256=uo264Cr0it0KzduaGD5_54YmjAPng5Vy5AhPsffPPEU,4193
|
|
4
|
-
dagster_postgres/utils.py,sha256=b_KukmD2puOJdvTFy0OUZtM5sGWyE1Fq6nP9hmUmRCc,5798
|
|
5
|
-
dagster_postgres/version.py,sha256=cC_HEsX6d06QmpjNLfRezzJvL501PPwiQYXWDM2_2UI,23
|
|
6
|
-
dagster_postgres/alembic/alembic.ini,sha256=GovyDEhu_6HvkWV6txqjdDBOe4BseSM0YDWGxXM5_cA,986
|
|
7
|
-
dagster_postgres/event_log/__init__.py,sha256=pQ7nZw5Fp60xVd_JPsZ61n6G5V4d0y8MfUGNkHvgbik,74
|
|
8
|
-
dagster_postgres/event_log/event_log.py,sha256=w8QheUMubekLwynbsMI3FZCsolKplYMrd1aoGWny6BU,15390
|
|
9
|
-
dagster_postgres/run_storage/__init__.py,sha256=4bI2C2kkZO9glXH8jIUViYBRSeMjdDHiVQVi_k1HkYw,66
|
|
10
|
-
dagster_postgres/run_storage/run_storage.py,sha256=KqsJ_vOJyDL6dfO3kgah-10ZbbTPJFXLbTSmvDoGaww,9589
|
|
11
|
-
dagster_postgres/schedule_storage/__init__.py,sha256=GKJlFZYh27fUxJDMQwNJUwibbTnip1ledbcokczutmk,81
|
|
12
|
-
dagster_postgres/schedule_storage/schedule_storage.py,sha256=NbcPd3ia45su1_v9d65ZLUUAW0m53-KBQO8zvGdhvZI,8458
|
|
13
|
-
dagster_postgres-0.23.7.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
|
|
14
|
-
dagster_postgres-0.23.7.dist-info/METADATA,sha256=tOsQTETMefyuaNUKDNyEaHHm0eSmqNeQzg_jPlQVF0c,762
|
|
15
|
-
dagster_postgres-0.23.7.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
16
|
-
dagster_postgres-0.23.7.dist-info/top_level.txt,sha256=lScMtAEKDX1yIv2tGa1nzntBa0HEStfWPfCwD8FWlHk,17
|
|
17
|
-
dagster_postgres-0.23.7.dist-info/RECORD,,
|
|
File without changes
|