PostBOUND 0.22.0__tar.gz → 0.22.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {postbound-0.22.0 → postbound-0.22.2}/PKG-INFO +1 -1
- {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/PKG-INFO +1 -1
- {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/SOURCES.txt +1 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/db/_db.py +52 -41
- {postbound-0.22.0 → postbound-0.22.2}/postbound/db/_stats.py +27 -10
- {postbound-0.22.0 → postbound-0.22.2}/postbound/duckdb/_duckdb.py +34 -4
- {postbound-0.22.0 → postbound-0.22.2}/postbound/qal/_formatter.py +29 -14
- {postbound-0.22.0 → postbound-0.22.2}/pyproject.toml +1 -1
- postbound-0.22.2/tests/test_duckdb.py +156 -0
- {postbound-0.22.0 → postbound-0.22.2}/LICENSE.txt +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/dependency_links.txt +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/requires.txt +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/top_level.txt +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/README.md +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/__init__.pyi +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/_base.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/_core.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/_hints.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/_ipython.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/_pipelines.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/_qep.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/_stages.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/bench.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/db/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/db/_cache.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/duckdb/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/mysql.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/__init__.pyi +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_cardinalities.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_dynprog.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_enumeration.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_helpers.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_native.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_randomized.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/parser.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_config.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_ctl.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_explain.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_pg.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/qal/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/qal/_qal.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/relalg.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/train/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/train/_train.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/transform.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/__init__.pyi +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/_df.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/_errors.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/collections.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/dicts.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/jsonize.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/logging.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/misc.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/num.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/nx.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/proc.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/stats.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/system.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/util/typing.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/validation.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/__init__.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/__init__.pyi +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/fdl.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/graphs.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/optimizer.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/plots.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/trees.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/postbound/workloads.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/setup.cfg +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/tests/test_benchmarking.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/tests/test_core.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/tests/test_docs.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/tests/test_postgres.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/tests/test_qal.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/tests/test_relalg.py +0 -0
- {postbound-0.22.0 → postbound-0.22.2}/tests/test_util.py +0 -0
|
@@ -18,6 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import bisect
|
|
20
20
|
import collections
|
|
21
|
+
import itertools
|
|
21
22
|
import textwrap
|
|
22
23
|
from abc import ABC, abstractmethod
|
|
23
24
|
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
@@ -1610,13 +1611,16 @@ class Histogram[T: _HistElem]:
|
|
|
1610
1611
|
|
|
1611
1612
|
Currently, we only provide two high-level methods for cardinality estimation: `frequency_below` and `frequency_above`.
|
|
1612
1613
|
The former estimates the number of elements that are less than or equal to a given value, while the latter estimates
|
|
1613
|
-
the number of elements that are greater than a given value.
|
|
1614
|
-
|
|
1614
|
+
the number of elements that are greater than a given value. Both estimates are based on the cumulative frequencies of
|
|
1615
|
+
the buckets, so equi-depth and arbitrary other bucketizations are handled alike. If the queried value lies on a
|
|
1616
|
+
bucket bound, both estimates are exact. Otherwise, the behavior depends on the selected `bucket_interpolation`
|
|
1617
|
+
strategy:
|
|
1615
1618
|
|
|
1616
|
-
- "bound" is a conservative strategy. It *bounds* the frequency
|
|
1617
|
-
|
|
1619
|
+
- "bound" is a conservative strategy. It *bounds* the frequency by counting the entire bucket that contains the
|
|
1620
|
+
queried value, i.e. it always provides an *upper bound* on the true frequency.
|
|
1618
1621
|
- "approx-uni" is a more aggressive strategy. It assumes that the values within each bucket are uniformly distributed
|
|
1619
|
-
and uses a simple linear interpolation to estimate
|
|
1622
|
+
and uses a simple linear interpolation to estimate which share of the bucket that contains the queried value lies
|
|
1623
|
+
below the value. With this strategy, ``frequency_below(v) + frequency_above(v) == n_rows`` holds for every value.
|
|
1620
1624
|
|
|
1621
1625
|
Histograms can be iterated over to access the individual (bucket bound, frequency) pairs. In addition, we provide
|
|
1622
1626
|
*__getitem__* access to get the i-th bucket bound and its frequency.
|
|
@@ -1634,6 +1638,12 @@ class Histogram[T: _HistElem]:
|
|
|
1634
1638
|
bucket_interpolation : Literal["approx-uni", "bound"], optional
|
|
1635
1639
|
The strategy to estimate the frequency of values that fall into a bucket, but are not exactly on the bucket bounds.
|
|
1636
1640
|
The default is "bound".
|
|
1641
|
+
|
|
1642
|
+
Raises
|
|
1643
|
+
------
|
|
1644
|
+
ValueError
|
|
1645
|
+
If `bounds` is empty, if `bounds` and `frequencies` differ in length, or if `bucket_interpolation` is not a
|
|
1646
|
+
supported strategy.
|
|
1637
1647
|
"""
|
|
1638
1648
|
|
|
1639
1649
|
def __init__(
|
|
@@ -1657,7 +1667,8 @@ class Histogram[T: _HistElem]:
|
|
|
1657
1667
|
self._n_rows = sum(self._frequencies)
|
|
1658
1668
|
self._lower = lower
|
|
1659
1669
|
self._freq_per_bucket = self._n_rows // len(self._bounds)
|
|
1660
|
-
self.
|
|
1670
|
+
self._cumulative_frequencies = [0, *itertools.accumulate(self._frequencies)]
|
|
1671
|
+
self.bucket_interpolation = bucket_interpolation
|
|
1661
1672
|
|
|
1662
1673
|
@property
|
|
1663
1674
|
def bounds(self) -> Sequence[T]:
|
|
@@ -1707,22 +1718,34 @@ class Histogram[T: _HistElem]:
|
|
|
1707
1718
|
)
|
|
1708
1719
|
self._bucket_interpolation = strategy
|
|
1709
1720
|
|
|
1710
|
-
def frequency_below(self, value: T) ->
|
|
1721
|
+
def frequency_below(self, value: T) -> float:
|
|
1711
1722
|
"""Estimate the frequency of values that are less than or equal to a given value.
|
|
1712
1723
|
|
|
1713
1724
|
If the value does not align exactly with the bucket bounds, the behavior depends on the selected
|
|
1714
|
-
`bucket_interpolation` strategy. See the class documentation for details.
|
|
1725
|
+
`bucket_interpolation` strategy. See the class documentation for details. Interpolated estimates are not
|
|
1726
|
+
rounded and can therefore be fractional.
|
|
1727
|
+
|
|
1728
|
+
Raises
|
|
1729
|
+
------
|
|
1730
|
+
ValueError
|
|
1731
|
+
If the "approx-uni" strategy has to interpolate within a bucket, but the value is not numeric or temporal.
|
|
1715
1732
|
"""
|
|
1716
|
-
|
|
1717
|
-
if upper_idx == 0:
|
|
1733
|
+
if value < self._lower:
|
|
1718
1734
|
return 0
|
|
1719
|
-
elif upper_idx == len(self._bounds):
|
|
1720
|
-
return self._n_rows
|
|
1721
1735
|
|
|
1722
1736
|
if self._bucket_interpolation == "bound":
|
|
1723
|
-
|
|
1737
|
+
# value lies in bucket containing_idx (or on its upper bound), i.e. all later buckets are entirely > value
|
|
1738
|
+
containing_idx = bisect.bisect_left(self._bounds, value)
|
|
1739
|
+
if containing_idx == len(self._bounds):
|
|
1740
|
+
return self._n_rows
|
|
1741
|
+
return self._cumulative_frequencies[containing_idx + 1]
|
|
1724
1742
|
|
|
1725
|
-
|
|
1743
|
+
# value lies in bucket upper_idx, i.e. all buckets before upper_idx are entirely <= value
|
|
1744
|
+
upper_idx = bisect.bisect_right(self._bounds, value)
|
|
1745
|
+
if upper_idx == len(self._bounds):
|
|
1746
|
+
return self._n_rows
|
|
1747
|
+
|
|
1748
|
+
complete_buckets = self._cumulative_frequencies[upper_idx]
|
|
1726
1749
|
|
|
1727
1750
|
if not isinstance(value, (int, float, date, datetime)):
|
|
1728
1751
|
raise ValueError(
|
|
@@ -1731,44 +1754,32 @@ class Histogram[T: _HistElem]:
|
|
|
1731
1754
|
f"for histograms of type {type(value).__name__}."
|
|
1732
1755
|
)
|
|
1733
1756
|
|
|
1734
|
-
|
|
1757
|
+
lower_bound = self._bounds[upper_idx - 1] if upper_idx > 0 else self._lower
|
|
1735
1758
|
upper_bound = self._bounds[upper_idx]
|
|
1736
|
-
lower_bound = self._bounds[lower_idx]
|
|
1737
1759
|
in_bucket_frac = (value - lower_bound) / (upper_bound - lower_bound)
|
|
1738
|
-
return
|
|
1760
|
+
return complete_buckets + in_bucket_frac * self._frequencies[upper_idx]
|
|
1739
1761
|
|
|
1740
|
-
def frequency_above(self, value: T) ->
|
|
1762
|
+
def frequency_above(self, value: T) -> float:
|
|
1741
1763
|
"""Estimate the frequency of values that are greater than a given value.
|
|
1742
1764
|
|
|
1743
1765
|
Due to the way we model histograms, we cannot directly estimate the frequency of values greater or equal to a given
|
|
1744
1766
|
value, but only the frequency of values that are strictly greater than it.
|
|
1745
1767
|
|
|
1746
1768
|
If the value does not align exactly with the bucket bounds, the behavior depends on the selected
|
|
1747
|
-
`bucket_interpolation` strategy. See the class documentation for details.
|
|
1748
|
-
|
|
1749
|
-
lower_idx = bisect.bisect_left(self._bounds, value)
|
|
1750
|
-
if lower_idx == 0:
|
|
1751
|
-
return self._n_rows
|
|
1752
|
-
elif lower_idx == len(self._bounds):
|
|
1753
|
-
return 0
|
|
1754
|
-
|
|
1755
|
-
if self._bucket_interpolation == "bound":
|
|
1756
|
-
return lower_idx * self._freq_per_bucket
|
|
1769
|
+
`bucket_interpolation` strategy. See the class documentation for details. Interpolated estimates are not
|
|
1770
|
+
rounded and can therefore be fractional.
|
|
1757
1771
|
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
)
|
|
1772
|
+
Raises
|
|
1773
|
+
------
|
|
1774
|
+
ValueError
|
|
1775
|
+
If the "approx-uni" strategy has to interpolate within a bucket, but the value is not numeric or temporal.
|
|
1776
|
+
"""
|
|
1777
|
+
if self._bucket_interpolation == "approx-uni":
|
|
1778
|
+
return self._n_rows - self.frequency_below(value)
|
|
1766
1779
|
|
|
1767
|
-
upper_idx
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
in_bucket_frac = (value - lower_bound) / (upper_bound - lower_bound)
|
|
1771
|
-
return (upper_idx + in_bucket_frac) * self._freq_per_bucket
|
|
1780
|
+
# all buckets before upper_idx are entirely <= value, but the bucket containing value might hold larger values
|
|
1781
|
+
upper_idx = bisect.bisect_right(self._bounds, value)
|
|
1782
|
+
return self._n_rows - self._cumulative_frequencies[upper_idx]
|
|
1772
1783
|
|
|
1773
1784
|
def __len__(self) -> int:
|
|
1774
1785
|
return len(self._bounds)
|
|
@@ -31,12 +31,16 @@ from ._db import Database, Histogram, HistogramApproximation, MostCommonValues,
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
def _infer_histogram_bounds[T](
|
|
34
|
-
frequencies: Sequence[tuple[T, int]], *, n_bins: int
|
|
34
|
+
frequencies: Sequence[tuple[T, int]], *, n_bins: int
|
|
35
35
|
) -> tuple[T, Sequence[T], Sequence[int]]:
|
|
36
|
-
"""Infer the bucket bounds and frequencies for a histogram from a list of (value, frequency) pairs.
|
|
36
|
+
"""Infer the bucket bounds and frequencies for a histogram from a list of (value, frequency) pairs.
|
|
37
|
+
|
|
38
|
+
The pairs must be sorted by value and must not contain NULL values.
|
|
39
|
+
"""
|
|
37
40
|
if not frequencies:
|
|
38
41
|
raise ValueError("Cannot infer histogram bounds from empty frequency list")
|
|
39
42
|
|
|
43
|
+
n_rows = sum(freq for _, freq in frequencies)
|
|
40
44
|
bucket_size = n_rows // n_bins
|
|
41
45
|
|
|
42
46
|
bounds: list[T] = []
|
|
@@ -50,6 +54,11 @@ def _infer_histogram_bounds[T](
|
|
|
50
54
|
buckets.append(cumulative_freq)
|
|
51
55
|
cumulative_freq = 0
|
|
52
56
|
|
|
57
|
+
if cumulative_freq:
|
|
58
|
+
# the remaining values did not fill an entire bucket, but they still need to be part of the histogram
|
|
59
|
+
bounds.append(frequencies[-1][0])
|
|
60
|
+
buckets.append(cumulative_freq)
|
|
61
|
+
|
|
53
62
|
return frequencies[0][0], bounds, buckets
|
|
54
63
|
|
|
55
64
|
|
|
@@ -188,7 +197,7 @@ class PreciseStatistics(StatisticsCatalog):
|
|
|
188
197
|
limit_clause = Limit(limit=k) if k is not None and k > 0 else None
|
|
189
198
|
sql = as_query(select_clause, from_clause, group_clause, order_clause, limit_clause)
|
|
190
199
|
|
|
191
|
-
result_set = self._db.execute_query(sql)
|
|
200
|
+
result_set = self._db.execute_query(sql, raw=True)
|
|
192
201
|
return MostCommonValues(result_set)
|
|
193
202
|
|
|
194
203
|
def histogram(
|
|
@@ -198,10 +207,18 @@ class PreciseStatistics(StatisticsCatalog):
|
|
|
198
207
|
|
|
199
208
|
In addition to `StatisticsCatalog.histogram`, this implementation allows to customize the number of buckets via
|
|
200
209
|
`n_bins`. Since the histogram has to be constructed from scratch, `n_bins` is required and passing *None*
|
|
201
|
-
raises a `ValueError`.
|
|
210
|
+
raises a `ValueError`. NULL values are not part of the histogram (use `null_frac` for them).
|
|
211
|
+
|
|
212
|
+
Since a value is never split across buckets, the histogram is only approximately equi-depth: a very frequent
|
|
213
|
+
value fills its bucket beyond the target size, and fewer than `n_bins` buckets might be created.
|
|
214
|
+
|
|
215
|
+
Raises
|
|
216
|
+
------
|
|
217
|
+
ValueError
|
|
218
|
+
If `n_bins` is *None* or not positive, or if the column does not contain any non-NULL values.
|
|
202
219
|
"""
|
|
203
|
-
if n_bins is None:
|
|
204
|
-
raise ValueError("n_bins must be
|
|
220
|
+
if n_bins is None or n_bins < 1:
|
|
221
|
+
raise ValueError(f"n_bins must be a positive number for emulated histogram, not {n_bins}")
|
|
205
222
|
|
|
206
223
|
if not ColumnReference.assert_bound(column):
|
|
207
224
|
raise UnboundColumnError(column)
|
|
@@ -210,13 +227,13 @@ class PreciseStatistics(StatisticsCatalog):
|
|
|
210
227
|
|
|
211
228
|
select_clause = Select([Projection.column(column), Projection.create_count(column, target_name="n")])
|
|
212
229
|
from_clause = From.create_for(column.table)
|
|
230
|
+
where_clause = Where(as_predicate(column, "IS NOT NULL"))
|
|
213
231
|
group_clause = GroupBy.create_for(column)
|
|
214
232
|
order_clause = OrderBy.create_for(column)
|
|
215
|
-
sql = as_query(select_clause, from_clause, group_clause, order_clause)
|
|
233
|
+
sql = as_query(select_clause, from_clause, where_clause, group_clause, order_clause)
|
|
216
234
|
|
|
217
|
-
result_set = self._db.execute_query(sql)
|
|
218
|
-
|
|
219
|
-
lo, bounds, buckets = _infer_histogram_bounds(result_set, n_bins=n_bins, n_rows=int(n_rows))
|
|
235
|
+
result_set = self._db.execute_query(sql, raw=True)
|
|
236
|
+
lo, bounds, buckets = _infer_histogram_bounds(result_set, n_bins=n_bins)
|
|
220
237
|
|
|
221
238
|
return Histogram(
|
|
222
239
|
bounds,
|
|
@@ -500,11 +500,41 @@ class DuckDBStatistics(StatisticsCatalog):
|
|
|
500
500
|
|
|
501
501
|
def _bind_node_to_table(scan_node: dict, *, query: SqlQuery) -> TableReference | None:
|
|
502
502
|
tables = query.tables()
|
|
503
|
-
relname = scan_node.get("extra_info", {}).get("Table", "")
|
|
503
|
+
relname: str = scan_node.get("extra_info", {}).get("Table", "")
|
|
504
504
|
if not relname:
|
|
505
505
|
return None
|
|
506
506
|
|
|
507
|
-
|
|
507
|
+
# DuckDB reports fully qualified table names in the scan node, while the query might reference just the table name
|
|
508
|
+
# or a schema-qualified name. We try our best to match these differences.
|
|
509
|
+
# The code below is pretty ugly, but works for now ¯\_(ツ)_/¯
|
|
510
|
+
n_comps = relname.count(".")
|
|
511
|
+
match n_comps:
|
|
512
|
+
case 0:
|
|
513
|
+
candidate_tables = [tab for tab in tables if tab.full_name == relname]
|
|
514
|
+
case 1:
|
|
515
|
+
schema, name = relname.split(".")
|
|
516
|
+
needle = TableReference(name, schema=schema)
|
|
517
|
+
candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
|
|
518
|
+
if not candidate_tables:
|
|
519
|
+
needle = TableReference(name)
|
|
520
|
+
candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
|
|
521
|
+
case 2:
|
|
522
|
+
cat, schema, name = relname.split(".")
|
|
523
|
+
needle = TableReference(name, schema=schema, catalog=cat)
|
|
524
|
+
candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
|
|
525
|
+
if not candidate_tables:
|
|
526
|
+
needle = TableReference(name, schema=schema)
|
|
527
|
+
candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
|
|
528
|
+
if not candidate_tables:
|
|
529
|
+
needle = TableReference(name)
|
|
530
|
+
candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
|
|
531
|
+
case _:
|
|
532
|
+
warnings.warn(
|
|
533
|
+
f"Unexpected table format '{relname}'. Only binding based on final name component", stacklevel=2
|
|
534
|
+
)
|
|
535
|
+
needle = relname.split(".")[-1]
|
|
536
|
+
candidate_tables = [tab for tab in tables if tab.full_name == needle]
|
|
537
|
+
|
|
508
538
|
if not candidate_tables:
|
|
509
539
|
return None
|
|
510
540
|
elif len(candidate_tables) == 1:
|
|
@@ -655,7 +685,7 @@ class DuckDBOptimizer(OptimizerInterface):
|
|
|
655
685
|
|
|
656
686
|
raw_explain = result_set[1]
|
|
657
687
|
parsed = json.loads(raw_explain)
|
|
658
|
-
return parse_duckdb_plan(parsed
|
|
688
|
+
return parse_duckdb_plan(parsed, query=query)
|
|
659
689
|
|
|
660
690
|
def parse_plan(self, plan: ResultSet, *, query: SqlQuery | None = None) -> QueryPlan | None:
|
|
661
691
|
# Similar to the Postgres implementation of parse_plan(), we try to be graceful
|
|
@@ -814,7 +844,7 @@ class DuckDBHintService(HintService):
|
|
|
814
844
|
|
|
815
845
|
def format_query(self, query: SqlQuery) -> str:
|
|
816
846
|
# DuckDB uses the Postgres SQL dialect, so this part is easy..
|
|
817
|
-
return qal.format_quick(query, flavor="
|
|
847
|
+
return qal.format_quick(query, flavor="duckdb")
|
|
818
848
|
|
|
819
849
|
def supports_hint(self, hint: PhysicalOperator | HintType) -> bool:
|
|
820
850
|
return hint in {
|
|
@@ -59,7 +59,7 @@ from ._qal import (
|
|
|
59
59
|
DefaultIndent = 2
|
|
60
60
|
"""The default amount of whitespace that is used to indent specific parts of the SQL query."""
|
|
61
61
|
|
|
62
|
-
SqlDialect = Literal["vanilla", "postgres"]
|
|
62
|
+
SqlDialect = Literal["vanilla", "postgres", "duckdb"]
|
|
63
63
|
"""The different flavors of SQL syntax that are supported by the formatter."""
|
|
64
64
|
|
|
65
65
|
|
|
@@ -156,16 +156,31 @@ class FormattingCaseExpression(CaseExpression):
|
|
|
156
156
|
|
|
157
157
|
|
|
158
158
|
def _quick_format_explain(explain_clause: Explain, *, flavor: SqlDialect) -> list[str]:
|
|
159
|
-
|
|
160
|
-
|
|
159
|
+
match flavor:
|
|
160
|
+
case "vanilla":
|
|
161
|
+
return [str(explain_clause)]
|
|
162
|
+
|
|
163
|
+
case "postgres":
|
|
164
|
+
options = ["SETTINGS", "SUMMARY", "VERBOSE"]
|
|
165
|
+
if explain_clause.analyze:
|
|
166
|
+
options.append("ANALYZE")
|
|
167
|
+
options.append("FORMAT JSON")
|
|
161
168
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
options.append("ANALYZE")
|
|
165
|
-
options.append("FORMAT JSON")
|
|
169
|
+
args = ", ".join(options)
|
|
170
|
+
return [f"EXPLAIN ({args})"]
|
|
166
171
|
|
|
167
|
-
|
|
168
|
-
|
|
172
|
+
case "duckdb":
|
|
173
|
+
options = []
|
|
174
|
+
if explain_clause.analyze:
|
|
175
|
+
options.append("ANALYZE")
|
|
176
|
+
options.append("FORMAT JSON")
|
|
177
|
+
|
|
178
|
+
args = ", ".join(options)
|
|
179
|
+
return [f"EXPLAIN ({args})"]
|
|
180
|
+
|
|
181
|
+
case _:
|
|
182
|
+
warnings.warn("Unknown SQL flavor for EXPLAIN clauses. Falling back to naive formatting", stacklevel=2)
|
|
183
|
+
return [str(explain_clause)]
|
|
169
184
|
|
|
170
185
|
|
|
171
186
|
def _quick_format_cte(cte_clause: CommonTableExpression, *, flavor: SqlDialect) -> list[str]:
|
|
@@ -547,7 +562,7 @@ def _quick_format_limit(limit_clause: Limit, *, flavor: SqlDialect) -> list[str]
|
|
|
547
562
|
else:
|
|
548
563
|
return []
|
|
549
564
|
|
|
550
|
-
case "postgres" if limit_clause.fetch_direction in {"first", "next"}:
|
|
565
|
+
case "postgres" | "duckdb" if limit_clause.fetch_direction in {"first", "next"}:
|
|
551
566
|
if limit_clause.limit and limit_clause.offset:
|
|
552
567
|
return [
|
|
553
568
|
f"LIMIT {limit_clause.limit}",
|
|
@@ -559,7 +574,7 @@ def _quick_format_limit(limit_clause: Limit, *, flavor: SqlDialect) -> list[str]
|
|
|
559
574
|
return [f"OFFSET {limit_clause.offset}"]
|
|
560
575
|
return []
|
|
561
576
|
|
|
562
|
-
case "postgres" if limit_clause.fetch_direction in {"prior", "last"}:
|
|
577
|
+
case "postgres" | "duckdb" if limit_clause.fetch_direction in {"prior", "last"}:
|
|
563
578
|
warnings.warn(
|
|
564
579
|
"Postgres does not support FETCH PRIOR and FETCH LAST. Falling back to naive formatting", stacklevel=2
|
|
565
580
|
)
|
|
@@ -673,9 +688,9 @@ def _expression_prettifier[T: SqlExpression](
|
|
|
673
688
|
indentation=indentation,
|
|
674
689
|
)
|
|
675
690
|
return (
|
|
676
|
-
|
|
677
|
-
if flavor == "
|
|
678
|
-
else
|
|
691
|
+
_PostgresCastExpression(replaced_cast, typ, type_params=params, array_type=array)
|
|
692
|
+
if flavor == "postgres"
|
|
693
|
+
else target(replaced_cast, typ, type_params=params)
|
|
679
694
|
)
|
|
680
695
|
|
|
681
696
|
case MathExpression(op, lhs, rhs):
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "PostBOUND"
|
|
7
|
-
version = "0.22.
|
|
7
|
+
version = "0.22.2"
|
|
8
8
|
authors = [{ name = "Rico Bergmann", email = "rico.bergmann1@tu-dresden.de" }]
|
|
9
9
|
description = "PostBOUND is a research framework to prototype and benchmark database query optimizers"
|
|
10
10
|
readme = "README.md"
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""Tests for `postbound.duckdb` -- the DuckDB backend, run against a real in-memory DuckDB (via quacklab).
|
|
2
|
+
|
|
3
|
+
All tests here are tier 1 (`embedded`): they need a real DuckDB engine, but no server or database file, because
|
|
4
|
+
the behaviours under test are about whether DuckDB *accepts* the SQL that PostBOUND generates. A double would
|
|
5
|
+
share the backend's assumptions about DuckDB's dialect and could only confirm a bug, not catch it. The synthetic
|
|
6
|
+
schema is created per test, so nothing is downloaded or written to disk.
|
|
7
|
+
|
|
8
|
+
`PreciseStatistics.histogram` is run here as well, because its SQL and the shape of the result set (in particular
|
|
9
|
+
how NULLs are grouped and sorted) are decided by the engine. Its decision logic is covered offline in
|
|
10
|
+
`tests/unit/test_db_stats.py`, which scripts the result sets observed here.
|
|
11
|
+
|
|
12
|
+
The pure formatting part of the EXPLAIN regression (the *duckdb* flavor of `format_quick`) is covered offline in
|
|
13
|
+
`tests/unit/test_qal_formatter.py`; the parsing of EXPLAIN output and the scan-to-table binding in
|
|
14
|
+
`tests/unit/test_duckdb.py`.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from collections.abc import Iterator
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import pytest
|
|
23
|
+
|
|
24
|
+
from postbound import ColumnReference, ScanOperator, TableReference, parser, transform
|
|
25
|
+
from postbound.db import Histogram, PreciseStatistics
|
|
26
|
+
from postbound.duckdb import DuckDBDatabase
|
|
27
|
+
|
|
28
|
+
pytestmark = pytest.mark.embedded
|
|
29
|
+
|
|
30
|
+
# -- fixtures -----------------------------------------------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def parse(sql: str):
|
|
34
|
+
return parser.parse_query(sql, bind_columns=False)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
FILTER_QUERY = parse("SELECT * FROM r WHERE r.a = 1")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@pytest.fixture
|
|
41
|
+
def duckdb_instance() -> Iterator[DuckDBDatabase]:
|
|
42
|
+
"""An in-memory DuckDB with a tiny table `r(a, b)`. The constructor does not register it on the pool."""
|
|
43
|
+
instance = DuckDBDatabase(Path(":memory:"))
|
|
44
|
+
instance.execute_query("CREATE TABLE r (a INTEGER, b INTEGER)")
|
|
45
|
+
instance.execute_query("INSERT INTO r VALUES (1, 10), (2, 20), (3, 30)")
|
|
46
|
+
yield instance
|
|
47
|
+
instance.close()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def create_column(instance: DuckDBDatabase, values: list[int | None]) -> ColumnReference:
|
|
51
|
+
"""Fills a fresh table `h(v)` with `values` and returns the reference to its column."""
|
|
52
|
+
instance.execute_query("CREATE TABLE h (v INTEGER)")
|
|
53
|
+
rows = ", ".join("(NULL)" if value is None else f"({value})" for value in values)
|
|
54
|
+
instance.execute_query(f"INSERT INTO h VALUES {rows}")
|
|
55
|
+
return ColumnReference("v", TableReference("h"))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# -- PreciseStatistics.histogram ----------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_precise_histogram_builds_equi_depth_buckets_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
62
|
+
column = create_column(duckdb_instance, [1, 2, 3, 4, 5, 6, 7, 8])
|
|
63
|
+
|
|
64
|
+
hist = PreciseStatistics(duckdb_instance).histogram(column, n_bins=4)
|
|
65
|
+
|
|
66
|
+
assert hist == Histogram([2, 4, 6, 8], [2, 2, 2, 2], lower=1, bucket_interpolation="approx-uni")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# -- regression tests --------------------------------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_hint_service_formats_explain_queries_without_postgres_only_options(duckdb_instance: DuckDBDatabase) -> None:
|
|
73
|
+
"""Regression guard for the v0.22.1 DuckDB EXPLAIN fix: `DuckDBHintService.format_query` used to call
|
|
74
|
+
`format_quick(..., flavor="postgres")`. Since v0.22.0 that flavor writes `EXPLAIN (SETTINGS, SUMMARY,
|
|
75
|
+
VERBOSE, FORMAT JSON)`, which DuckDB does not support, so the backend could not run any EXPLAIN query.
|
|
76
|
+
"""
|
|
77
|
+
explain_query = transform.as_explain(FILTER_QUERY)
|
|
78
|
+
|
|
79
|
+
formatted = duckdb_instance.hinting().format_query(explain_query)
|
|
80
|
+
|
|
81
|
+
header = formatted.splitlines()[0]
|
|
82
|
+
assert header == "EXPLAIN (FORMAT JSON)"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_query_plan_of_a_parsed_query_runs_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
86
|
+
"""Regression guard for the v0.22.1 DuckDB EXPLAIN fix: `DuckDBOptimizer.query_plan` wraps a `SqlQuery` in an
|
|
87
|
+
EXPLAIN and formats it through the hint service, which used the Postgres flavor and thereby emitted the
|
|
88
|
+
Postgres-only `SETTINGS`/`SUMMARY`/`VERBOSE` options. DuckDB rejected the statement (`NotImplementedException: Unimplemented explain type: settings`)
|
|
89
|
+
before planning even started. (Plain-string queries took a separate hand-written `EXPLAIN (FORMAT JSON)` path
|
|
90
|
+
and were unaffected, which is why this test passes a parsed query.)
|
|
91
|
+
"""
|
|
92
|
+
plan = duckdb_instance.optimizer().query_plan(FILTER_QUERY)
|
|
93
|
+
|
|
94
|
+
assert plan.operator == ScanOperator.SequentialScan
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_execute_query_runs_a_parsed_explain_query_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
98
|
+
"""Regression guard for the v0.22.1 DuckDB EXPLAIN fix, on the `DuckDBDatabase.execute_query` path: it formats
|
|
99
|
+
every `SqlQuery` through the hint service as well, so executing an `EXPLAIN` / `EXPLAIN ANALYZE` query object
|
|
100
|
+
directly failed with the same `NotImplementedException` as `query_plan`.
|
|
101
|
+
"""
|
|
102
|
+
explain_queries = [transform.as_explain(FILTER_QUERY), transform.as_explain_analyze(FILTER_QUERY)]
|
|
103
|
+
|
|
104
|
+
result_sets = [duckdb_instance.execute_query(query, raw=True) for query in explain_queries]
|
|
105
|
+
|
|
106
|
+
assert [result_set[0][0] for result_set in result_sets] == ["physical_plan", "analyzed_plan"]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def test_analyze_plan_returns_the_measured_plan(duckdb_instance: DuckDBDatabase) -> None:
|
|
110
|
+
"""Regression guard for the v0.22.1 DuckDB `analyze_plan` fix: `DuckDBOptimizer.analyze_plan` passed `parsed[0]`
|
|
111
|
+
to `parse_duckdb_plan`, assuming the JSON *list* that plain `EXPLAIN (FORMAT JSON)` returns. With ANALYZE,
|
|
112
|
+
DuckDB returns a single JSON *object* (a profiling root with the plan below `children`), so indexing it with `0`
|
|
113
|
+
raised a `KeyError` and no query could be analyzed.
|
|
114
|
+
"""
|
|
115
|
+
plan = duckdb_instance.optimizer().analyze_plan(FILTER_QUERY)
|
|
116
|
+
|
|
117
|
+
assert plan.operator == ScanOperator.SequentialScan
|
|
118
|
+
assert plan.actual_cardinality == 1
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_query_plan_binds_scan_nodes_to_their_tables(duckdb_instance: DuckDBDatabase) -> None:
|
|
122
|
+
"""Regression guard for the v0.22.1 DuckDB table-binding fix, end to end: `_bind_node_to_table` compared the
|
|
123
|
+
`Table` entry of a scan node with `TableReference.full_name` by plain string equality, but DuckDB reports fully
|
|
124
|
+
qualified names (`memory.main.r` here) while the query references just `r`. Every scan node stayed unbound, so
|
|
125
|
+
`query_plan` returned plans without any tables. The offline tests in `tests/unit/test_duckdb.py` pin the captured
|
|
126
|
+
output; this one guards against DuckDB changing the reported name format underneath them.
|
|
127
|
+
"""
|
|
128
|
+
plan = duckdb_instance.optimizer().query_plan(FILTER_QUERY)
|
|
129
|
+
|
|
130
|
+
assert plan.base_table == TableReference("r")
|
|
131
|
+
assert plan.tables() == {TableReference("r")}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_precise_histogram_excludes_nulls_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
135
|
+
"""Regression guard for the unreleased histogram fixes: DuckDB returns the NULL group of the histogram's GROUP BY
|
|
136
|
+
query last as ``(None, 0)``. With fewer rows than bins, `_infer_histogram_bounds` turned it into the upper bound of
|
|
137
|
+
the histogram, and every estimate at or above the largest value raised a `TypeError` comparing with *None*.
|
|
138
|
+
"""
|
|
139
|
+
column = create_column(duckdb_instance, [1, 1, 2, 3, None, None])
|
|
140
|
+
|
|
141
|
+
hist = PreciseStatistics(duckdb_instance).histogram(column)
|
|
142
|
+
|
|
143
|
+
assert list(hist) == [(1, 2), (2, 1), (3, 1)]
|
|
144
|
+
assert hist.frequency_below(5) == 4
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_precise_histogram_of_a_single_distinct_value_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
148
|
+
"""Regression guard for the unreleased histogram fixes: the one-row result set was simplified to a plain tuple
|
|
149
|
+
before the bounds were inferred, which crashed. See `test_histogram_of_a_column_with_a_single_distinct_value` in
|
|
150
|
+
`tests/unit/test_db_stats.py`.
|
|
151
|
+
"""
|
|
152
|
+
column = create_column(duckdb_instance, [7, 7])
|
|
153
|
+
|
|
154
|
+
hist = PreciseStatistics(duckdb_instance).histogram(column)
|
|
155
|
+
|
|
156
|
+
assert list(hist) == [(7, 2)]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|