pytest-duckdb 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_duckdb/__init__.py +3 -0
- pytest_duckdb/fixtures.py +50 -0
- pytest_duckdb/plugin.py +254 -0
- pytest_duckdb/snapshot.py +274 -0
- pytest_duckdb-0.1.0.dist-info/METADATA +111 -0
- pytest_duckdb-0.1.0.dist-info/RECORD +9 -0
- pytest_duckdb-0.1.0.dist-info/WHEEL +4 -0
- pytest_duckdb-0.1.0.dist-info/entry_points.txt +2 -0
- pytest_duckdb-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""CSV fixture loading — reads CSV files from a directory into DuckDB tables."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import duckdb
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def load_csvs(conn: duckdb.DuckDBPyConnection, fixtures_dir: str) -> None:
|
|
12
|
+
"""Load all CSV files from *fixtures_dir* as DuckDB tables.
|
|
13
|
+
|
|
14
|
+
Each CSV file becomes a table named after its stem (filename without extension).
|
|
15
|
+
Uses ``read_csv_auto`` for schema inference. Non-existent directories are
|
|
16
|
+
silently ignored so that tests without fixtures work out of the box.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
conn: An open DuckDB connection.
|
|
20
|
+
fixtures_dir: Absolute or project-relative path to the CSV directory.
|
|
21
|
+
"""
|
|
22
|
+
path = Path(fixtures_dir)
|
|
23
|
+
if not path.is_dir():
|
|
24
|
+
return
|
|
25
|
+
|
|
26
|
+
csv_files = sorted(path.glob("*.csv"))
|
|
27
|
+
for csv_file in csv_files:
|
|
28
|
+
table_name = csv_file.stem
|
|
29
|
+
conn.execute(
|
|
30
|
+
f'CREATE TABLE "{table_name}" AS '
|
|
31
|
+
f"SELECT * FROM read_csv_auto('{csv_file}')"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def read_sql_file(path: str) -> str:
|
|
36
|
+
"""Read and return the contents of a SQL file.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
path: Absolute or relative path to the ``.sql`` file.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
The file contents as a string.
|
|
43
|
+
|
|
44
|
+
Raises:
|
|
45
|
+
FileNotFoundError: If the file does not exist.
|
|
46
|
+
"""
|
|
47
|
+
if not os.path.isfile(path):
|
|
48
|
+
raise FileNotFoundError(f"SQL file not found: {path}")
|
|
49
|
+
with open(path) as fh:
|
|
50
|
+
return fh.read()
|
pytest_duckdb/plugin.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""pytest plugin registration — fixtures, markers, and CLI options.
|
|
2
|
+
|
|
3
|
+
Provides three fixtures:
|
|
4
|
+
|
|
5
|
+
* ``duckdb_session`` – function-scoped in-memory DuckDB with CSV fixtures
|
|
6
|
+
auto-loaded.
|
|
7
|
+
* ``sql_result`` – executes a ``.sql`` file referenced via ``@pytest.mark.sql``.
|
|
8
|
+
* ``sql_snapshot`` – callable fixture for Parquet snapshot testing.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Callable, Optional
|
|
16
|
+
|
|
17
|
+
import pytest
|
|
18
|
+
|
|
19
|
+
from .fixtures import load_csvs, read_sql_file
|
|
20
|
+
from .snapshot import compare, save
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def pytest_addoption(parser: pytest.Parser) -> None:
|
|
24
|
+
"""Register CLI options and ini-file keys."""
|
|
25
|
+
parser.addini(
|
|
26
|
+
"duckdb_fixtures_dir",
|
|
27
|
+
type="string",
|
|
28
|
+
default="tests/fixtures",
|
|
29
|
+
help="Directory containing CSV fixture files",
|
|
30
|
+
)
|
|
31
|
+
parser.addini(
|
|
32
|
+
"duckdb_snapshots_dir",
|
|
33
|
+
type="string",
|
|
34
|
+
default="tests/__snapshots__",
|
|
35
|
+
help="Directory for Parquet snapshot files",
|
|
36
|
+
)
|
|
37
|
+
group = parser.getgroup("duckdb")
|
|
38
|
+
group.addoption(
|
|
39
|
+
"--snapshot-update",
|
|
40
|
+
action="store_true",
|
|
41
|
+
default=False,
|
|
42
|
+
help="Regenerate all Parquet snapshots instead of comparing",
|
|
43
|
+
)
|
|
44
|
+
group.addoption(
|
|
45
|
+
"--duckdb-fixtures-dir",
|
|
46
|
+
action="store",
|
|
47
|
+
default=None,
|
|
48
|
+
metavar="PATH",
|
|
49
|
+
help="Directory containing CSV fixture files (default: tests/fixtures)",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def pytest_configure(config: pytest.Config) -> None:
|
|
54
|
+
"""Register the ``sql`` marker and expand docstring."""
|
|
55
|
+
config.addinivalue_line(
|
|
56
|
+
"markers",
|
|
57
|
+
"sql(path): mark a test to run a SQL file; "
|
|
58
|
+
"the path is resolved relative to the project root.",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# Persist resolved config options.
|
|
62
|
+
config.duckdb_fixtures_dir = _resolve_option(
|
|
63
|
+
config,
|
|
64
|
+
cli_name="--duckdb-fixtures-dir",
|
|
65
|
+
ini_name="duckdb_fixtures_dir",
|
|
66
|
+
default="tests/fixtures",
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
config.duckdb_snapshots_dir = _resolve_ini_option(
|
|
70
|
+
config,
|
|
71
|
+
ini_name="duckdb_snapshots_dir",
|
|
72
|
+
default="tests/__snapshots__",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# ---------------------------------------------------------------------------
|
|
77
|
+
# Fixtures
|
|
78
|
+
# ---------------------------------------------------------------------------
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.fixture(scope="function")
|
|
82
|
+
def duckdb_session(request: pytest.FixtureRequest):
|
|
83
|
+
"""Function-scoped DuckDB in-memory session with auto-loaded CSV fixtures.
|
|
84
|
+
|
|
85
|
+
CSVs in the configured fixtures directory are loaded as tables named
|
|
86
|
+
after the file stem. If the directory does not exist the session is
|
|
87
|
+
returned empty with no error.
|
|
88
|
+
|
|
89
|
+
Yields:
|
|
90
|
+
``duckdb.DuckDBPyConnection`` -- close on teardown.
|
|
91
|
+
"""
|
|
92
|
+
import duckdb
|
|
93
|
+
|
|
94
|
+
conn = duckdb.connect(":memory:")
|
|
95
|
+
fixtures_dir = _resolve_path(
|
|
96
|
+
request.config.duckdb_fixtures_dir,
|
|
97
|
+
request.config.rootpath,
|
|
98
|
+
)
|
|
99
|
+
load_csvs(conn, fixtures_dir)
|
|
100
|
+
yield conn
|
|
101
|
+
conn.close()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@pytest.fixture(scope="function")
|
|
105
|
+
def sql_result(
|
|
106
|
+
duckdb_session, # noqa: ANN001, ANN201
|
|
107
|
+
request: pytest.FixtureRequest,
|
|
108
|
+
):
|
|
109
|
+
"""Fixture that executes a SQL file referenced by ``@pytest.mark.sql``.
|
|
110
|
+
|
|
111
|
+
Usage::
|
|
112
|
+
|
|
113
|
+
@pytest.mark.sql("tests/queries/my_query.sql")
|
|
114
|
+
def test_something(sql_result):
|
|
115
|
+
assert len(sql_result) > 0
|
|
116
|
+
|
|
117
|
+
Returns a ``pandas.DataFrame`` if pandas is installed, otherwise a
|
|
118
|
+
:class:`~pytest_duckdb.snapshot.QueryResult` with ``.columns``.
|
|
119
|
+
|
|
120
|
+
Raises:
|
|
121
|
+
FileNotFoundError: The referenced SQL file does not exist.
|
|
122
|
+
"""
|
|
123
|
+
marker = request.node.get_closest_marker("sql")
|
|
124
|
+
if marker is None:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
sql_path = _resolve_marker_path(marker, request)
|
|
128
|
+
query = read_sql_file(sql_path)
|
|
129
|
+
result = duckdb_session.execute(query)
|
|
130
|
+
|
|
131
|
+
from .snapshot import _cursor_to_result
|
|
132
|
+
|
|
133
|
+
return _cursor_to_result(result)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@pytest.fixture(scope="function")
|
|
137
|
+
def sql_snapshot(
|
|
138
|
+
duckdb_session, # noqa: ANN001, ANN201
|
|
139
|
+
request: pytest.FixtureRequest,
|
|
140
|
+
) -> Callable[..., None]:
|
|
141
|
+
"""Callable fixture for Parquet snapshot testing.
|
|
142
|
+
|
|
143
|
+
Usage::
|
|
144
|
+
|
|
145
|
+
def test_query(sql_snapshot):
|
|
146
|
+
result = duckdb_session.execute("SELECT ...").fetchdf()
|
|
147
|
+
sql_snapshot(result)
|
|
148
|
+
|
|
149
|
+
**First run:** saves the result to ``{snapshots_dir}/{module}_{test}.parquet``
|
|
150
|
+
and passes.
|
|
151
|
+
|
|
152
|
+
**Subsequent runs:** loads the Parquet snapshot and asserts equality. Raises
|
|
153
|
+
``AssertionError`` with a clear diff on mismatch.
|
|
154
|
+
|
|
155
|
+
Pass ``--snapshot-update`` on the CLI to regenerate all snapshots.
|
|
156
|
+
"""
|
|
157
|
+
snapshots_dir = _resolve_path(
|
|
158
|
+
request.config.duckdb_snapshots_dir,
|
|
159
|
+
request.config.rootpath,
|
|
160
|
+
)
|
|
161
|
+
update = request.config.getoption("--snapshot-update", default=False)
|
|
162
|
+
|
|
163
|
+
# Build a deterministic snapshot file name.
|
|
164
|
+
module_name = _module_name(request.node)
|
|
165
|
+
test_name = _test_name(request.node)
|
|
166
|
+
snapshot_path = os.path.join(
|
|
167
|
+
snapshots_dir, f"{module_name}_{test_name}.parquet",
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def _snapshot(result: Any, name: Optional[str] = None) -> None:
|
|
171
|
+
"""Save or compare *result* against the Parquet snapshot.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
result: DataFrame or list-of-tuples to snapshot.
|
|
175
|
+
name: Optional override for the snapshot file name stem.
|
|
176
|
+
|
|
177
|
+
Raises:
|
|
178
|
+
AssertionError: On mismatch (when not updating).
|
|
179
|
+
"""
|
|
180
|
+
nonlocal snapshot_path
|
|
181
|
+
if name is not None:
|
|
182
|
+
snapshot_path = os.path.join(
|
|
183
|
+
snapshots_dir, f"{module_name}_{name}.parquet",
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
if update or not os.path.isfile(snapshot_path):
|
|
187
|
+
save(duckdb_session, result, snapshot_path)
|
|
188
|
+
else:
|
|
189
|
+
compare(duckdb_session, result, snapshot_path)
|
|
190
|
+
|
|
191
|
+
return _snapshot
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# ---------------------------------------------------------------------------
|
|
195
|
+
# Internal helpers
|
|
196
|
+
# ---------------------------------------------------------------------------
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _resolve_option(
|
|
200
|
+
config: pytest.Config,
|
|
201
|
+
cli_name: str,
|
|
202
|
+
ini_name: str,
|
|
203
|
+
default: str,
|
|
204
|
+
) -> str:
|
|
205
|
+
"""Resolve a config value from CLI → ini → default."""
|
|
206
|
+
cli_val = config.getoption(cli_name.replace("--", "").replace("-", "_"))
|
|
207
|
+
if cli_val is not None:
|
|
208
|
+
return cli_val
|
|
209
|
+
ini_val = config.getini(ini_name)
|
|
210
|
+
if ini_val:
|
|
211
|
+
return ini_val
|
|
212
|
+
return default
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _resolve_ini_option(
|
|
216
|
+
config: pytest.Config,
|
|
217
|
+
ini_name: str,
|
|
218
|
+
default: str,
|
|
219
|
+
) -> str:
|
|
220
|
+
"""Resolve a config value from ini → default."""
|
|
221
|
+
ini_val = config.getini(ini_name)
|
|
222
|
+
if ini_val:
|
|
223
|
+
return ini_val
|
|
224
|
+
return default
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _resolve_path(path_str: str, root: Path) -> str:
|
|
228
|
+
"""Resolve a possibly-relative path against the project root."""
|
|
229
|
+
if not os.path.isabs(path_str):
|
|
230
|
+
return os.path.join(str(root), path_str)
|
|
231
|
+
return path_str
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _resolve_marker_path(marker: pytest.Mark, request: pytest.FixtureRequest) -> str:
|
|
235
|
+
"""Resolve the path from a ``@pytest.mark.sql`` marker."""
|
|
236
|
+
path = marker.args[0] if marker.args else marker.kwargs.get("path", "")
|
|
237
|
+
if not os.path.isabs(path):
|
|
238
|
+
return os.path.join(str(request.config.rootpath), path)
|
|
239
|
+
return path
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _module_name(node: pytest.Item) -> str:
|
|
243
|
+
"""Return a sanitised module name for snapshot file naming."""
|
|
244
|
+
# node.path gives the file path, e.g. tests/test_fixtures.py
|
|
245
|
+
fspath = getattr(node, "path", None) or getattr(node, "fspath", None)
|
|
246
|
+
if fspath:
|
|
247
|
+
stem = Path(str(fspath)).stem # e.g. test_fixtures
|
|
248
|
+
return stem
|
|
249
|
+
return "unknown_module"
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _test_name(node: pytest.Item) -> str:
|
|
253
|
+
"""Return a sanitised test function name."""
|
|
254
|
+
return node.name.replace("[", "_").replace("]", "_").replace("/", "_")
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Parquet snapshot save / load / compare.
|
|
2
|
+
|
|
3
|
+
Snapshots provide deterministic regression testing for SQL queries. The first
|
|
4
|
+
run saves the result as a Parquet file. Subsequent runs compare against the
|
|
5
|
+
stored snapshot and raise ``AssertionError`` with a clear diff on mismatch.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, List, Sequence
|
|
13
|
+
|
|
14
|
+
import duckdb
|
|
15
|
+
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
# Public helpers
|
|
18
|
+
# ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def save(conn: duckdb.DuckDBPyConnection, result: Any, path: str) -> None:
|
|
22
|
+
"""Save *result* (DataFrame or list-of-tuples) to a Parquet file.
|
|
23
|
+
|
|
24
|
+
The parent directory is created if it does not exist.
|
|
25
|
+
"""
|
|
26
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
_register_temp(conn, result, "__snapshot_to_save__")
|
|
28
|
+
conn.execute(f"COPY __snapshot_to_save__ TO '{path}' (FORMAT PARQUET)")
|
|
29
|
+
conn.execute("DROP TABLE IF EXISTS __snapshot_to_save__")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def load(conn: duckdb.DuckDBPyConnection, path: str) -> Any:
|
|
33
|
+
"""Load a Parquet snapshot as a pandas DataFrame (if available) or
|
|
34
|
+
:class:`QueryResult`."""
|
|
35
|
+
if not os.path.isfile(path):
|
|
36
|
+
raise FileNotFoundError(f"Snapshot file not found: {path}")
|
|
37
|
+
cur = conn.execute(f"SELECT * FROM read_parquet('{path}')")
|
|
38
|
+
return _cursor_to_result(cur)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def compare(
|
|
42
|
+
conn: duckdb.DuckDBPyConnection,
|
|
43
|
+
actual: Any,
|
|
44
|
+
path: str,
|
|
45
|
+
) -> None:
|
|
46
|
+
"""Compare *actual* against the Parquet snapshot at *path*.
|
|
47
|
+
|
|
48
|
+
Raises ``AssertionError`` with a human-readable diff if they differ.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
conn: DuckDB connection used for comparison queries.
|
|
52
|
+
actual: DataFrame or list-of-tuples (the current query result).
|
|
53
|
+
path: Path to the stored Parquet snapshot.
|
|
54
|
+
|
|
55
|
+
Raises:
|
|
56
|
+
FileNotFoundError: The snapshot has not been created yet.
|
|
57
|
+
AssertionError: The result does not match the snapshot.
|
|
58
|
+
"""
|
|
59
|
+
if not os.path.isfile(path):
|
|
60
|
+
raise FileNotFoundError(
|
|
61
|
+
f"Snapshot does not exist yet — run without --snapshot-update "
|
|
62
|
+
f"to create it first (expected: {path})"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
_register_temp(conn, actual, "__snapshot_actual__")
|
|
66
|
+
|
|
67
|
+
# Load the snapshot into a temp table for SQL-based comparison.
|
|
68
|
+
conn.execute(
|
|
69
|
+
"CREATE TEMP TABLE __snapshot_expected__ AS "
|
|
70
|
+
f"SELECT * FROM read_parquet('{path}')"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
# 1. Column / type matching.
|
|
75
|
+
actual_cols = _describe(conn, "__snapshot_actual__")
|
|
76
|
+
expected_cols = _describe(conn, "__snapshot_expected__")
|
|
77
|
+
|
|
78
|
+
if actual_cols != expected_cols:
|
|
79
|
+
_raise_schema_diff(actual_cols, expected_cols)
|
|
80
|
+
|
|
81
|
+
# 2. Row-level comparison via EXCEPT.
|
|
82
|
+
differing = conn.execute(
|
|
83
|
+
"SELECT * FROM __snapshot_actual__ "
|
|
84
|
+
"EXCEPT "
|
|
85
|
+
"SELECT * FROM __snapshot_expected__"
|
|
86
|
+
).fetchall()
|
|
87
|
+
|
|
88
|
+
missing = conn.execute(
|
|
89
|
+
"SELECT * FROM __snapshot_expected__ "
|
|
90
|
+
"EXCEPT "
|
|
91
|
+
"SELECT * FROM __snapshot_actual__"
|
|
92
|
+
).fetchall()
|
|
93
|
+
|
|
94
|
+
if differing or missing:
|
|
95
|
+
col_names = [c[0] for c in expected_cols] if expected_cols else []
|
|
96
|
+
_raise_row_diff(differing, missing, col_names)
|
|
97
|
+
|
|
98
|
+
finally:
|
|
99
|
+
conn.execute("DROP TABLE IF EXISTS __snapshot_actual__")
|
|
100
|
+
conn.execute("DROP TABLE IF EXISTS __snapshot_expected__")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# ---------------------------------------------------------------------------
|
|
104
|
+
# Result types
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class QueryResult:
|
|
109
|
+
"""Wrapper around a list of result rows with a ``.columns`` attribute.
|
|
110
|
+
|
|
111
|
+
Used when pandas is not installed so that consumers can access column
|
|
112
|
+
metadata in a uniform way.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
__slots__ = ("_rows", "columns")
|
|
116
|
+
|
|
117
|
+
def __init__(self, rows: Sequence[tuple], columns: Sequence[str]) -> None:
|
|
118
|
+
self._rows = list(rows)
|
|
119
|
+
self.columns = tuple(columns)
|
|
120
|
+
|
|
121
|
+
def __iter__(self): # noqa: ANN201
|
|
122
|
+
return iter(self._rows)
|
|
123
|
+
|
|
124
|
+
def __len__(self) -> int:
|
|
125
|
+
return len(self._rows)
|
|
126
|
+
|
|
127
|
+
def __getitem__(self, idx: int) -> tuple:
|
|
128
|
+
return self._rows[idx]
|
|
129
|
+
|
|
130
|
+
def __repr__(self) -> str: # noqa: D105
|
|
131
|
+
return f"QueryResult(rows={len(self)}, columns={self.columns})"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ---------------------------------------------------------------------------
|
|
135
|
+
# Internal helpers
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _register_temp(conn: duckdb.DuckDBPyConnection, result: Any, name: str) -> None:
|
|
140
|
+
"""Register *result* as a temporary DuckDB table called *name*."""
|
|
141
|
+
# If it has a `columns` attribute it's a QueryResult (list-of-tuples).
|
|
142
|
+
if hasattr(result, "_rows"):
|
|
143
|
+
rows = list(result)
|
|
144
|
+
if not rows:
|
|
145
|
+
columns = list(result.columns)
|
|
146
|
+
# Create an empty table with VARCHAR columns.
|
|
147
|
+
col_defs = ",\n".join(f'"{c}" VARCHAR' for c in columns)
|
|
148
|
+
conn.execute(f"CREATE TEMP TABLE {name} ({col_defs})")
|
|
149
|
+
else:
|
|
150
|
+
_register_tuple_list(conn, rows, list(result.columns), name)
|
|
151
|
+
return
|
|
152
|
+
|
|
153
|
+
# Check for pandas DataFrame.
|
|
154
|
+
try:
|
|
155
|
+
import pandas as pd
|
|
156
|
+
|
|
157
|
+
if isinstance(result, pd.DataFrame):
|
|
158
|
+
conn.register(name, result)
|
|
159
|
+
return
|
|
160
|
+
except ImportError:
|
|
161
|
+
pass
|
|
162
|
+
|
|
163
|
+
# Fallback: treat as list-of-tuples with no column metadata.
|
|
164
|
+
if isinstance(result, (list, tuple)):
|
|
165
|
+
rows = list(result)
|
|
166
|
+
if rows:
|
|
167
|
+
cols = [f"col{i}" for i in range(len(rows[0]))]
|
|
168
|
+
else:
|
|
169
|
+
cols = []
|
|
170
|
+
_register_tuple_list(conn, rows, cols, name)
|
|
171
|
+
return
|
|
172
|
+
|
|
173
|
+
# DuckDB-compatible type (e.g. a relation).
|
|
174
|
+
conn.execute(f"CREATE TEMP TABLE {name} AS SELECT * FROM result")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _register_tuple_list(
|
|
178
|
+
conn: duckdb.DuckDBPyConnection,
|
|
179
|
+
rows: list,
|
|
180
|
+
columns: List[str],
|
|
181
|
+
name: str,
|
|
182
|
+
) -> None:
|
|
183
|
+
"""Create a temp table from a list of Python tuples, preserving types."""
|
|
184
|
+
col_list = ", ".join(f'"{c}"' for c in columns)
|
|
185
|
+
quoted = ", ".join("?" for _ in columns)
|
|
186
|
+
head = rows[0]
|
|
187
|
+
|
|
188
|
+
# Let DuckDB infer types from the first row.
|
|
189
|
+
placeholders = "(" + ", ".join(
|
|
190
|
+
_py_value_repr(v) for v in head
|
|
191
|
+
) + ")"
|
|
192
|
+
conn.execute(
|
|
193
|
+
f"CREATE TEMP TABLE {name} AS "
|
|
194
|
+
f"SELECT * FROM (VALUES {placeholders}) AS t ({col_list}) WHERE FALSE"
|
|
195
|
+
)
|
|
196
|
+
conn.executemany(
|
|
197
|
+
f"INSERT INTO {name} ({col_list}) VALUES ({quoted})",
|
|
198
|
+
rows,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _py_value_repr(val: Any) -> str:
|
|
203
|
+
"""Return a DuckDB-compatible literal for a Python value."""
|
|
204
|
+
if val is None:
|
|
205
|
+
return "NULL"
|
|
206
|
+
if isinstance(val, bool):
|
|
207
|
+
return "TRUE" if val else "FALSE"
|
|
208
|
+
if isinstance(val, (int, float)):
|
|
209
|
+
return str(val)
|
|
210
|
+
if isinstance(val, str):
|
|
211
|
+
escaped = val.replace("'", "''")
|
|
212
|
+
return f"'{escaped}'"
|
|
213
|
+
return f"'{val}'"
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _cursor_to_result(cur: duckdb.DuckDBPyConnection) -> Any:
|
|
217
|
+
"""Convert a DuckDB cursor to a DataFrame (preferred) or QueryResult."""
|
|
218
|
+
columns = [desc[0] for desc in cur.description] if cur.description else []
|
|
219
|
+
rows = cur.fetchall()
|
|
220
|
+
try:
|
|
221
|
+
import pandas as pd
|
|
222
|
+
|
|
223
|
+
return pd.DataFrame(rows, columns=columns)
|
|
224
|
+
except ImportError:
|
|
225
|
+
return QueryResult(rows, columns)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _describe(conn: duckdb.DuckDBPyConnection, table: str) -> List[tuple]:
|
|
229
|
+
"""Return ``[(column_name, data_type), ...]`` for *table*."""
|
|
230
|
+
return conn.execute(f"DESCRIBE {table}").fetchall()
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _raise_schema_diff(
|
|
234
|
+
actual: List[tuple],
|
|
235
|
+
expected: List[tuple],
|
|
236
|
+
) -> None:
|
|
237
|
+
"""Raise ``AssertionError`` with a schema diff message."""
|
|
238
|
+
actual_map = {r[0]: r[1] for r in actual}
|
|
239
|
+
expected_map = {r[0]: r[1] for r in expected}
|
|
240
|
+
lines: list[str] = []
|
|
241
|
+
|
|
242
|
+
for col in sorted(set(list(actual_map) + list(expected_map))):
|
|
243
|
+
a_type = actual_map.get(col, "<missing>")
|
|
244
|
+
e_type = expected_map.get(col, "<missing>")
|
|
245
|
+
if a_type != e_type:
|
|
246
|
+
lines.append(f" {col}: expected={e_type} actual={a_type}")
|
|
247
|
+
|
|
248
|
+
msg = "Schema mismatch:\n" + "\n".join(lines)
|
|
249
|
+
raise AssertionError(msg)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _raise_row_diff(
|
|
253
|
+
differing: List[tuple],
|
|
254
|
+
missing: List[tuple],
|
|
255
|
+
col_names: List[str],
|
|
256
|
+
) -> None:
|
|
257
|
+
"""Raise ``AssertionError`` with a row-level diff message."""
|
|
258
|
+
lines: list[str] = []
|
|
259
|
+
if differing:
|
|
260
|
+
lines.append(f"Rows in actual but not expected ({len(differing)}):")
|
|
261
|
+
for row in differing[:10]:
|
|
262
|
+
lines.append(f" {row}")
|
|
263
|
+
if len(differing) > 10:
|
|
264
|
+
lines.append(f" ... and {len(differing) - 10} more")
|
|
265
|
+
if missing:
|
|
266
|
+
lines.append(f"Rows in expected but not actual ({len(missing)}):")
|
|
267
|
+
for row in missing[:10]:
|
|
268
|
+
lines.append(f" {row}")
|
|
269
|
+
if len(missing) > 10:
|
|
270
|
+
lines.append(f" ... and {len(missing) - 10} more")
|
|
271
|
+
if col_names:
|
|
272
|
+
lines.insert(0, f"Columns: {col_names}")
|
|
273
|
+
msg = "Snapshot mismatch:\n" + "\n".join(lines)
|
|
274
|
+
raise AssertionError(msg)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pytest-duckdb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: pytest plugin for SQL pipeline testing with DuckDB — load fixtures, run queries, snapshot results
|
|
5
|
+
Project-URL: Homepage, https://github.com/AnnasMazhar/pytest-duckdb
|
|
6
|
+
Project-URL: Issues, https://github.com/AnnasMazhar/pytest-duckdb/issues
|
|
7
|
+
Project-URL: Source, https://github.com/AnnasMazhar/pytest-duckdb
|
|
8
|
+
Author-email: Annas Mazhar <annas.mazhar10@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: data-engineering,duckdb,pytest,snapshot,sql,testing
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Framework :: Pytest
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Database
|
|
23
|
+
Classifier: Topic :: Software Development :: Testing
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Requires-Dist: duckdb>=0.9.0
|
|
26
|
+
Requires-Dist: pytest>=7.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: coverage>=7.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: pandas>=2.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# pytest-duckdb
|
|
35
|
+
|
|
36
|
+
[](https://github.com/AnnasMazhar/pytest-duckdb/actions/workflows/ci.yml)
|
|
37
|
+
[](https://pypi.org/project/pytest-duckdb/)
|
|
38
|
+
[](https://pypi.org/project/pytest-duckdb/)
|
|
39
|
+
|
|
40
|
+
SQL pipeline testing made trivial. Load fixture CSVs as tables, run your queries, snapshot the results.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install pytest-duckdb
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Quick Start
|
|
49
|
+
|
|
50
|
+
Place CSV files in `tests/fixtures/`:
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
tests/
|
|
54
|
+
fixtures/
|
|
55
|
+
orders.csv
|
|
56
|
+
customers.csv
|
|
57
|
+
test_queries.py
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
They become tables automatically:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
def test_revenue_query(duckdb_session, sql_snapshot):
|
|
64
|
+
result = duckdb_session.execute(
|
|
65
|
+
"SELECT * FROM orders WHERE amount > 100"
|
|
66
|
+
).fetchall()
|
|
67
|
+
|
|
68
|
+
# First run saves the snapshot, subsequent runs diff against it
|
|
69
|
+
sql_snapshot(result)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Features
|
|
73
|
+
|
|
74
|
+
- **Zero config** — fixture CSVs in `tests/fixtures/` auto-load as tables
|
|
75
|
+
- **Snapshot testing** — first run saves output, subsequent runs diff
|
|
76
|
+
- **Raw SQL files** — test `.sql` files directly without dbt
|
|
77
|
+
- **Ephemeral** — each test gets a fresh in-memory DuckDB
|
|
78
|
+
- **Fast** — DuckDB is embedded, no Docker, no network
|
|
79
|
+
|
|
80
|
+
## Fixture Loading
|
|
81
|
+
|
|
82
|
+
CSVs loaded by file stem become table names. `orders.csv` → `orders` table.
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
def test_join(duckdb_session):
|
|
86
|
+
result = duckdb_session.execute("""
|
|
87
|
+
SELECT c.name, o.amount
|
|
88
|
+
FROM orders o JOIN customers c ON o.customer_id = c.id
|
|
89
|
+
""").fetchall()
|
|
90
|
+
assert len(result) > 0
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## SQL File Testing
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
@pytest.mark.sql("queries/revenue_by_month.sql")
|
|
97
|
+
def test_revenue_by_month(sql_result, sql_snapshot):
|
|
98
|
+
sql_snapshot(sql_result)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Snapshot Update
|
|
102
|
+
|
|
103
|
+
Regenerate all snapshots when your expected output changes:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
pytest --snapshot-update
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## License
|
|
110
|
+
|
|
111
|
+
MIT
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
pytest_duckdb/__init__.py,sha256=4OGvmHcZdLQ1gI2k2wFuJpNJRojabdEZ-D-K7OIKL0I,79
|
|
2
|
+
pytest_duckdb/fixtures.py,sha256=kOaTUlAZBqYKnbDL5ENoeKbtdzVYJmj45n-ly_MMObw,1444
|
|
3
|
+
pytest_duckdb/plugin.py,sha256=sjWPHno3G6MU2b17qGTpZBDDav7m_-_CN9KXbLrsA9Y,7663
|
|
4
|
+
pytest_duckdb/snapshot.py,sha256=UReiv5Z8oXeHciLsEpZ-eBY5JtOAbAUCBNaopQVkdfI,9034
|
|
5
|
+
pytest_duckdb-0.1.0.dist-info/METADATA,sha256=3-dQpOKenzLanbPyemLZgyiouZlqRQ5PCACpY6IkV4w,3361
|
|
6
|
+
pytest_duckdb-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
7
|
+
pytest_duckdb-0.1.0.dist-info/entry_points.txt,sha256=m6c8cR8sma92kzR8ZYc6Uq8Z99egktJge8Kq1G23jZo,41
|
|
8
|
+
pytest_duckdb-0.1.0.dist-info/licenses/LICENSE,sha256=o83nYDkVVPK53SMcq6N6vVcAdhD0SCtkjo4yyDrygk0,1069
|
|
9
|
+
pytest_duckdb-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Annas Mazhar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|