dagster-postgres 0.26.7rc0__py3-none-any.whl → 0.26.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dagster-postgres might be problematic. Click here for more details.
- dagster_postgres/event_log/event_log.py +8 -2
- dagster_postgres/test_fixtures/__init__.py +75 -0
- dagster_postgres/test_fixtures/docker-compose.yml +10 -0
- dagster_postgres/version.py +1 -1
- {dagster_postgres-0.26.7rc0.dist-info → dagster_postgres-0.26.11.dist-info}/METADATA +2 -2
- {dagster_postgres-0.26.7rc0.dist-info → dagster_postgres-0.26.11.dist-info}/RECORD +9 -7
- {dagster_postgres-0.26.7rc0.dist-info → dagster_postgres-0.26.11.dist-info}/LICENSE +0 -0
- {dagster_postgres-0.26.7rc0.dist-info → dagster_postgres-0.26.11.dist-info}/WHEEL +0 -0
- {dagster_postgres-0.26.7rc0.dist-info → dagster_postgres-0.26.11.dist-info}/top_level.txt +0 -0
|
@@ -176,7 +176,6 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
|
|
|
176
176
|
event (EventLogEntry): The event to store.
|
|
177
177
|
"""
|
|
178
178
|
check.inst_param(event, "event", EventLogEntry)
|
|
179
|
-
|
|
180
179
|
insert_event_statement = self.prepare_insert_event(event) # from SqlEventLogStorage.py
|
|
181
180
|
with self._connect() as conn:
|
|
182
181
|
result = conn.execute(
|
|
@@ -218,11 +217,18 @@ class PostgresEventLogStorage(SqlEventLogStorage, ConfigurableClass):
|
|
|
218
217
|
all(event.get_dagster_event().event_type in BATCH_WRITABLE_EVENTS for event in events),
|
|
219
218
|
f"{BATCH_WRITABLE_EVENTS} are the only currently supported events for batch writes.",
|
|
220
219
|
)
|
|
220
|
+
events = [
|
|
221
|
+
event
|
|
222
|
+
for event in events
|
|
223
|
+
if not event.get_dagster_event().is_asset_failed_to_materialize
|
|
224
|
+
]
|
|
225
|
+
if len(events) == 0:
|
|
226
|
+
return
|
|
221
227
|
|
|
222
228
|
insert_event_statement = self.prepare_insert_event_batch(events)
|
|
223
229
|
with self._connect() as conn:
|
|
224
230
|
result = conn.execute(insert_event_statement.returning(SqlEventLogStorageTable.c.id))
|
|
225
|
-
event_ids = [cast(int, row[0]) for row in result.fetchall()]
|
|
231
|
+
event_ids = [cast("int", row[0]) for row in result.fetchall()]
|
|
226
232
|
|
|
227
233
|
# We only update the asset table with the last event
|
|
228
234
|
self.store_asset_event(events[-1], event_ids[-1])
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import tempfile
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from dagster._core.test_utils import instance_for_test
|
|
7
|
+
from dagster._utils.merger import merge_dicts
|
|
8
|
+
from dagster_test.fixtures import docker_compose_cm, network_name_from_yml
|
|
9
|
+
|
|
10
|
+
from dagster_postgres.utils import get_conn_string, wait_for_connection
|
|
11
|
+
|
|
12
|
+
compose_file = Path(__file__).parent / "docker-compose.yml"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture(scope="session")
|
|
16
|
+
def postgres_network():
|
|
17
|
+
yield network_name_from_yml(compose_file)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.fixture(scope="session")
|
|
21
|
+
def postgres_hostname():
|
|
22
|
+
with docker_compose_cm(docker_compose_yml=compose_file) as hostnames:
|
|
23
|
+
yield hostnames["postgres"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.fixture(scope="session")
|
|
27
|
+
def postgres_conn_str(postgres_hostname):
|
|
28
|
+
conn_str = get_conn_string(
|
|
29
|
+
username="test",
|
|
30
|
+
password="test",
|
|
31
|
+
hostname=postgres_hostname,
|
|
32
|
+
db_name="test",
|
|
33
|
+
params=dict(connect_timeout=5),
|
|
34
|
+
)
|
|
35
|
+
wait_for_connection(
|
|
36
|
+
conn_str,
|
|
37
|
+
retry_limit=10,
|
|
38
|
+
retry_wait=3,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
yield conn_str
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@pytest.fixture
|
|
45
|
+
def postgres_instance(postgres_conn_str):
|
|
46
|
+
@contextmanager
|
|
47
|
+
def _instance(overrides=None):
|
|
48
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
|
49
|
+
with instance_for_test(
|
|
50
|
+
temp_dir=temp_dir,
|
|
51
|
+
overrides=merge_dicts(
|
|
52
|
+
{
|
|
53
|
+
"run_storage": {
|
|
54
|
+
"module": "dagster_postgres.run_storage.run_storage",
|
|
55
|
+
"class": "PostgresRunStorage",
|
|
56
|
+
"config": {"postgres_url": postgres_conn_str},
|
|
57
|
+
},
|
|
58
|
+
"event_log_storage": {
|
|
59
|
+
"module": "dagster_postgres.event_log.event_log",
|
|
60
|
+
"class": "PostgresEventLogStorage",
|
|
61
|
+
"config": {"postgres_url": postgres_conn_str},
|
|
62
|
+
},
|
|
63
|
+
"schedule_storage": {
|
|
64
|
+
"module": "dagster_postgres.schedule_storage.schedule_storage",
|
|
65
|
+
"class": "PostgresScheduleStorage",
|
|
66
|
+
"config": {"postgres_url": postgres_conn_str},
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
overrides if overrides else {},
|
|
70
|
+
),
|
|
71
|
+
) as instance:
|
|
72
|
+
instance.wipe()
|
|
73
|
+
yield instance
|
|
74
|
+
|
|
75
|
+
return _instance
|
dagster_postgres/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.26.
|
|
1
|
+
__version__ = "0.26.11"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dagster-postgres
|
|
3
|
-
Version: 0.26.
|
|
3
|
+
Version: 0.26.11
|
|
4
4
|
Summary: A Dagster integration for postgres
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-postgres
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -14,6 +14,6 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Requires-Python: >=3.9,<3.13
|
|
16
16
|
License-File: LICENSE
|
|
17
|
-
Requires-Dist: dagster ==1.10.
|
|
17
|
+
Requires-Dist: dagster ==1.10.11
|
|
18
18
|
Requires-Dist: psycopg2-binary
|
|
19
19
|
|
|
@@ -2,16 +2,18 @@ dagster_postgres/__init__.py,sha256=rZfjhBDGC_SIGnKuiSCzi7D0xD36mAnchDLRL-n1Tmk,
|
|
|
2
2
|
dagster_postgres/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
3
3
|
dagster_postgres/storage.py,sha256=CiPA773j0BQvPtCEz5vVqKjap9h0ODAxR_R_CvbI-bI,4310
|
|
4
4
|
dagster_postgres/utils.py,sha256=AjuUYZ3y4Q3oaa-6I2zd6SNwl66FE8cJ_e2rIKrI9IQ,5930
|
|
5
|
-
dagster_postgres/version.py,sha256=
|
|
5
|
+
dagster_postgres/version.py,sha256=KbFltDKZs6Fz9BEm-IuAjy0mNlt5Il5YMdB41Jptyqg,24
|
|
6
6
|
dagster_postgres/alembic/alembic.ini,sha256=GovyDEhu_6HvkWV6txqjdDBOe4BseSM0YDWGxXM5_cA,986
|
|
7
7
|
dagster_postgres/event_log/__init__.py,sha256=wRcUR-StRNrPCqpEzi0MRY8b-r_TEWV17OsEynFqlLs,100
|
|
8
|
-
dagster_postgres/event_log/event_log.py,sha256=
|
|
8
|
+
dagster_postgres/event_log/event_log.py,sha256=fPmO1w7ZweU9P_Kw366APG_aKSSRp74-L21sYspCAYc,15709
|
|
9
9
|
dagster_postgres/run_storage/__init__.py,sha256=oW_546mJ5K-e-RF0Ou7r-4fHWxFthHgPPhWxklsVK1g,94
|
|
10
10
|
dagster_postgres/run_storage/run_storage.py,sha256=VMXT5sb5n04ViWOYj2eOoZqOk78hqL38J6CuikM79No,9757
|
|
11
11
|
dagster_postgres/schedule_storage/__init__.py,sha256=-jW-1S4Xf5Ew-cz-DjKjU5sVs9EEly_2ELMLOXTewv0,123
|
|
12
12
|
dagster_postgres/schedule_storage/schedule_storage.py,sha256=iPKcvnj1eEeG5LH73GoFvoS3hmE8MFddiFyf7QoKZ3g,8826
|
|
13
|
-
dagster_postgres
|
|
14
|
-
dagster_postgres-
|
|
15
|
-
dagster_postgres-0.26.
|
|
16
|
-
dagster_postgres-0.26.
|
|
17
|
-
dagster_postgres-0.26.
|
|
13
|
+
dagster_postgres/test_fixtures/__init__.py,sha256=ZIcbGYa_CMmVF0YkuiX4uI3wRjBke_2VE8QXKcji0Y4,2447
|
|
14
|
+
dagster_postgres/test_fixtures/docker-compose.yml,sha256=hp2VTnENYK6CL2Yae3IsktssahPFQiHyhpNcG36CivM,208
|
|
15
|
+
dagster_postgres-0.26.11.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
|
|
16
|
+
dagster_postgres-0.26.11.dist-info/METADATA,sha256=k4TVSasY33p_dKio-dvrg_W1RlZzVfjLPxEOe44LZxY,715
|
|
17
|
+
dagster_postgres-0.26.11.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
18
|
+
dagster_postgres-0.26.11.dist-info/top_level.txt,sha256=lScMtAEKDX1yIv2tGa1nzntBa0HEStfWPfCwD8FWlHk,17
|
|
19
|
+
dagster_postgres-0.26.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|