PostBOUND 0.22.1__tar.gz → 0.22.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {postbound-0.22.1 → postbound-0.22.2}/PKG-INFO +1 -1
- {postbound-0.22.1 → postbound-0.22.2}/PostBOUND.egg-info/PKG-INFO +1 -1
- {postbound-0.22.1 → postbound-0.22.2}/postbound/db/_db.py +52 -41
- {postbound-0.22.1 → postbound-0.22.2}/postbound/db/_stats.py +27 -10
- {postbound-0.22.1 → postbound-0.22.2}/pyproject.toml +1 -1
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_duckdb.py +50 -1
- {postbound-0.22.1 → postbound-0.22.2}/LICENSE.txt +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/PostBOUND.egg-info/SOURCES.txt +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/PostBOUND.egg-info/dependency_links.txt +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/PostBOUND.egg-info/requires.txt +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/PostBOUND.egg-info/top_level.txt +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/README.md +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/__init__.pyi +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/_base.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/_core.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/_hints.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/_ipython.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/_pipelines.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/_qep.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/_stages.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/bench.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/db/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/db/_cache.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/duckdb/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/duckdb/_duckdb.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/mysql.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/__init__.pyi +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/_cardinalities.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/_dynprog.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/_enumeration.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/_helpers.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/_native.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/opt/_randomized.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/parser.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/postgres/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/postgres/_config.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/postgres/_ctl.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/postgres/_explain.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/postgres/_pg.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/qal/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/qal/_formatter.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/qal/_qal.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/relalg.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/train/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/train/_train.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/transform.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/__init__.pyi +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/_df.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/_errors.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/collections.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/dicts.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/jsonize.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/logging.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/misc.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/num.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/nx.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/proc.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/stats.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/system.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/util/typing.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/validation.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/vis/__init__.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/vis/__init__.pyi +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/vis/fdl.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/vis/graphs.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/vis/optimizer.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/vis/plots.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/vis/trees.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/postbound/workloads.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/setup.cfg +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_benchmarking.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_core.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_docs.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_postgres.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_qal.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_relalg.py +0 -0
- {postbound-0.22.1 → postbound-0.22.2}/tests/test_util.py +0 -0
|
@@ -18,6 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import bisect
|
|
20
20
|
import collections
|
|
21
|
+
import itertools
|
|
21
22
|
import textwrap
|
|
22
23
|
from abc import ABC, abstractmethod
|
|
23
24
|
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
@@ -1610,13 +1611,16 @@ class Histogram[T: _HistElem]:
|
|
|
1610
1611
|
|
|
1611
1612
|
Currently, we only provide two high-level methods for cardinality estimation: `frequency_below` and `frequency_above`.
|
|
1612
1613
|
The former estimates the number of elements that are less than or equal to a given value, while the latter estimates
|
|
1613
|
-
the number of elements that are greater than a given value.
|
|
1614
|
-
|
|
1614
|
+
the number of elements that are greater than a given value. Both estimates are based on the cumulative frequencies of
|
|
1615
|
+
the buckets, so equi-depth and arbitrary other bucketizations are handled alike. If the queried value lies on a
|
|
1616
|
+
bucket bound, both estimates are exact. Otherwise, the behavior depends on the selected `bucket_interpolation`
|
|
1617
|
+
strategy:
|
|
1615
1618
|
|
|
1616
|
-
- "bound" is a conservative strategy. It *bounds* the frequency
|
|
1617
|
-
|
|
1619
|
+
- "bound" is a conservative strategy. It *bounds* the frequency by counting the entire bucket that contains the
|
|
1620
|
+
queried value, i.e. it always provides an *upper bound* on the true frequency.
|
|
1618
1621
|
- "approx-uni" is a more aggressive strategy. It assumes that the values within each bucket are uniformly distributed
|
|
1619
|
-
and uses a simple linear interpolation to estimate
|
|
1622
|
+
and uses a simple linear interpolation to estimate which share of the bucket that contains the queried value lies
|
|
1623
|
+
below the value. With this strategy, ``frequency_below(v) + frequency_above(v) == n_rows`` holds for every value.
|
|
1620
1624
|
|
|
1621
1625
|
Histograms can be iterated over to access the individual (bucket bound, frequency) pairs. In addition, we provide
|
|
1622
1626
|
*__getitem__* access to get the i-th bucket bound and its frequency.
|
|
@@ -1634,6 +1638,12 @@ class Histogram[T: _HistElem]:
|
|
|
1634
1638
|
bucket_interpolation : Literal["approx-uni", "bound"], optional
|
|
1635
1639
|
The strategy to estimate the frequency of values that fall into a bucket, but are not exactly on the bucket bounds.
|
|
1636
1640
|
The default is "bound".
|
|
1641
|
+
|
|
1642
|
+
Raises
|
|
1643
|
+
------
|
|
1644
|
+
ValueError
|
|
1645
|
+
If `bounds` is empty, if `bounds` and `frequencies` differ in length, or if `bucket_interpolation` is not a
|
|
1646
|
+
supported strategy.
|
|
1637
1647
|
"""
|
|
1638
1648
|
|
|
1639
1649
|
def __init__(
|
|
@@ -1657,7 +1667,8 @@ class Histogram[T: _HistElem]:
|
|
|
1657
1667
|
self._n_rows = sum(self._frequencies)
|
|
1658
1668
|
self._lower = lower
|
|
1659
1669
|
self._freq_per_bucket = self._n_rows // len(self._bounds)
|
|
1660
|
-
self.
|
|
1670
|
+
self._cumulative_frequencies = [0, *itertools.accumulate(self._frequencies)]
|
|
1671
|
+
self.bucket_interpolation = bucket_interpolation
|
|
1661
1672
|
|
|
1662
1673
|
@property
|
|
1663
1674
|
def bounds(self) -> Sequence[T]:
|
|
@@ -1707,22 +1718,34 @@ class Histogram[T: _HistElem]:
|
|
|
1707
1718
|
)
|
|
1708
1719
|
self._bucket_interpolation = strategy
|
|
1709
1720
|
|
|
1710
|
-
def frequency_below(self, value: T) ->
|
|
1721
|
+
def frequency_below(self, value: T) -> float:
|
|
1711
1722
|
"""Estimate the frequency of values that are less than or equal to a given value.
|
|
1712
1723
|
|
|
1713
1724
|
If the value does not align exactly with the bucket bounds, the behavior depends on the selected
|
|
1714
|
-
`bucket_interpolation` strategy. See the class documentation for details.
|
|
1725
|
+
`bucket_interpolation` strategy. See the class documentation for details. Interpolated estimates are not
|
|
1726
|
+
rounded and can therefore be fractional.
|
|
1727
|
+
|
|
1728
|
+
Raises
|
|
1729
|
+
------
|
|
1730
|
+
ValueError
|
|
1731
|
+
If the "approx-uni" strategy has to interpolate within a bucket, but the value is not numeric or temporal.
|
|
1715
1732
|
"""
|
|
1716
|
-
|
|
1717
|
-
if upper_idx == 0:
|
|
1733
|
+
if value < self._lower:
|
|
1718
1734
|
return 0
|
|
1719
|
-
elif upper_idx == len(self._bounds):
|
|
1720
|
-
return self._n_rows
|
|
1721
1735
|
|
|
1722
1736
|
if self._bucket_interpolation == "bound":
|
|
1723
|
-
|
|
1737
|
+
# value lies in bucket containing_idx (or on its upper bound), i.e. all later buckets are entirely > value
|
|
1738
|
+
containing_idx = bisect.bisect_left(self._bounds, value)
|
|
1739
|
+
if containing_idx == len(self._bounds):
|
|
1740
|
+
return self._n_rows
|
|
1741
|
+
return self._cumulative_frequencies[containing_idx + 1]
|
|
1724
1742
|
|
|
1725
|
-
|
|
1743
|
+
# value lies in bucket upper_idx, i.e. all buckets before upper_idx are entirely <= value
|
|
1744
|
+
upper_idx = bisect.bisect_right(self._bounds, value)
|
|
1745
|
+
if upper_idx == len(self._bounds):
|
|
1746
|
+
return self._n_rows
|
|
1747
|
+
|
|
1748
|
+
complete_buckets = self._cumulative_frequencies[upper_idx]
|
|
1726
1749
|
|
|
1727
1750
|
if not isinstance(value, (int, float, date, datetime)):
|
|
1728
1751
|
raise ValueError(
|
|
@@ -1731,44 +1754,32 @@ class Histogram[T: _HistElem]:
|
|
|
1731
1754
|
f"for histograms of type {type(value).__name__}."
|
|
1732
1755
|
)
|
|
1733
1756
|
|
|
1734
|
-
|
|
1757
|
+
lower_bound = self._bounds[upper_idx - 1] if upper_idx > 0 else self._lower
|
|
1735
1758
|
upper_bound = self._bounds[upper_idx]
|
|
1736
|
-
lower_bound = self._bounds[lower_idx]
|
|
1737
1759
|
in_bucket_frac = (value - lower_bound) / (upper_bound - lower_bound)
|
|
1738
|
-
return
|
|
1760
|
+
return complete_buckets + in_bucket_frac * self._frequencies[upper_idx]
|
|
1739
1761
|
|
|
1740
|
-
def frequency_above(self, value: T) ->
|
|
1762
|
+
def frequency_above(self, value: T) -> float:
|
|
1741
1763
|
"""Estimate the frequency of values that are greater than a given value.
|
|
1742
1764
|
|
|
1743
1765
|
Due to the way we model histograms, we cannot directly estimate the frequency of values greater or equal to a given
|
|
1744
1766
|
value, but only the frequency of values that are strictly greater than it.
|
|
1745
1767
|
|
|
1746
1768
|
If the value does not align exactly with the bucket bounds, the behavior depends on the selected
|
|
1747
|
-
`bucket_interpolation` strategy. See the class documentation for details.
|
|
1748
|
-
|
|
1749
|
-
lower_idx = bisect.bisect_left(self._bounds, value)
|
|
1750
|
-
if lower_idx == 0:
|
|
1751
|
-
return self._n_rows
|
|
1752
|
-
elif lower_idx == len(self._bounds):
|
|
1753
|
-
return 0
|
|
1754
|
-
|
|
1755
|
-
if self._bucket_interpolation == "bound":
|
|
1756
|
-
return lower_idx * self._freq_per_bucket
|
|
1769
|
+
`bucket_interpolation` strategy. See the class documentation for details. Interpolated estimates are not
|
|
1770
|
+
rounded and can therefore be fractional.
|
|
1757
1771
|
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
)
|
|
1772
|
+
Raises
|
|
1773
|
+
------
|
|
1774
|
+
ValueError
|
|
1775
|
+
If the "approx-uni" strategy has to interpolate within a bucket, but the value is not numeric or temporal.
|
|
1776
|
+
"""
|
|
1777
|
+
if self._bucket_interpolation == "approx-uni":
|
|
1778
|
+
return self._n_rows - self.frequency_below(value)
|
|
1766
1779
|
|
|
1767
|
-
upper_idx
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
in_bucket_frac = (value - lower_bound) / (upper_bound - lower_bound)
|
|
1771
|
-
return (upper_idx + in_bucket_frac) * self._freq_per_bucket
|
|
1780
|
+
# all buckets before upper_idx are entirely <= value, but the bucket containing value might hold larger values
|
|
1781
|
+
upper_idx = bisect.bisect_right(self._bounds, value)
|
|
1782
|
+
return self._n_rows - self._cumulative_frequencies[upper_idx]
|
|
1772
1783
|
|
|
1773
1784
|
def __len__(self) -> int:
|
|
1774
1785
|
return len(self._bounds)
|
|
@@ -31,12 +31,16 @@ from ._db import Database, Histogram, HistogramApproximation, MostCommonValues,
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
def _infer_histogram_bounds[T](
|
|
34
|
-
frequencies: Sequence[tuple[T, int]], *, n_bins: int
|
|
34
|
+
frequencies: Sequence[tuple[T, int]], *, n_bins: int
|
|
35
35
|
) -> tuple[T, Sequence[T], Sequence[int]]:
|
|
36
|
-
"""Infer the bucket bounds and frequencies for a histogram from a list of (value, frequency) pairs.
|
|
36
|
+
"""Infer the bucket bounds and frequencies for a histogram from a list of (value, frequency) pairs.
|
|
37
|
+
|
|
38
|
+
The pairs must be sorted by value and must not contain NULL values.
|
|
39
|
+
"""
|
|
37
40
|
if not frequencies:
|
|
38
41
|
raise ValueError("Cannot infer histogram bounds from empty frequency list")
|
|
39
42
|
|
|
43
|
+
n_rows = sum(freq for _, freq in frequencies)
|
|
40
44
|
bucket_size = n_rows // n_bins
|
|
41
45
|
|
|
42
46
|
bounds: list[T] = []
|
|
@@ -50,6 +54,11 @@ def _infer_histogram_bounds[T](
|
|
|
50
54
|
buckets.append(cumulative_freq)
|
|
51
55
|
cumulative_freq = 0
|
|
52
56
|
|
|
57
|
+
if cumulative_freq:
|
|
58
|
+
# the remaining values did not fill an entire bucket, but they still need to be part of the histogram
|
|
59
|
+
bounds.append(frequencies[-1][0])
|
|
60
|
+
buckets.append(cumulative_freq)
|
|
61
|
+
|
|
53
62
|
return frequencies[0][0], bounds, buckets
|
|
54
63
|
|
|
55
64
|
|
|
@@ -188,7 +197,7 @@ class PreciseStatistics(StatisticsCatalog):
|
|
|
188
197
|
limit_clause = Limit(limit=k) if k is not None and k > 0 else None
|
|
189
198
|
sql = as_query(select_clause, from_clause, group_clause, order_clause, limit_clause)
|
|
190
199
|
|
|
191
|
-
result_set = self._db.execute_query(sql)
|
|
200
|
+
result_set = self._db.execute_query(sql, raw=True)
|
|
192
201
|
return MostCommonValues(result_set)
|
|
193
202
|
|
|
194
203
|
def histogram(
|
|
@@ -198,10 +207,18 @@ class PreciseStatistics(StatisticsCatalog):
|
|
|
198
207
|
|
|
199
208
|
In addition to `StatisticsCatalog.histogram`, this implementation allows to customize the number of buckets via
|
|
200
209
|
`n_bins`. Since the histogram has to be constructed from scratch, `n_bins` is required and passing *None*
|
|
201
|
-
raises a `ValueError`.
|
|
210
|
+
raises a `ValueError`. NULL values are not part of the histogram (use `null_frac` for them).
|
|
211
|
+
|
|
212
|
+
Since a value is never split across buckets, the histogram is only approximately equi-depth: a very frequent
|
|
213
|
+
value fills its bucket beyond the target size, and fewer than `n_bins` buckets might be created.
|
|
214
|
+
|
|
215
|
+
Raises
|
|
216
|
+
------
|
|
217
|
+
ValueError
|
|
218
|
+
If `n_bins` is *None* or not positive, or if the column does not contain any non-NULL values.
|
|
202
219
|
"""
|
|
203
|
-
if n_bins is None:
|
|
204
|
-
raise ValueError("n_bins must be
|
|
220
|
+
if n_bins is None or n_bins < 1:
|
|
221
|
+
raise ValueError(f"n_bins must be a positive number for emulated histogram, not {n_bins}")
|
|
205
222
|
|
|
206
223
|
if not ColumnReference.assert_bound(column):
|
|
207
224
|
raise UnboundColumnError(column)
|
|
@@ -210,13 +227,13 @@ class PreciseStatistics(StatisticsCatalog):
|
|
|
210
227
|
|
|
211
228
|
select_clause = Select([Projection.column(column), Projection.create_count(column, target_name="n")])
|
|
212
229
|
from_clause = From.create_for(column.table)
|
|
230
|
+
where_clause = Where(as_predicate(column, "IS NOT NULL"))
|
|
213
231
|
group_clause = GroupBy.create_for(column)
|
|
214
232
|
order_clause = OrderBy.create_for(column)
|
|
215
|
-
sql = as_query(select_clause, from_clause, group_clause, order_clause)
|
|
233
|
+
sql = as_query(select_clause, from_clause, where_clause, group_clause, order_clause)
|
|
216
234
|
|
|
217
|
-
result_set = self._db.execute_query(sql)
|
|
218
|
-
|
|
219
|
-
lo, bounds, buckets = _infer_histogram_bounds(result_set, n_bins=n_bins, n_rows=int(n_rows))
|
|
235
|
+
result_set = self._db.execute_query(sql, raw=True)
|
|
236
|
+
lo, bounds, buckets = _infer_histogram_bounds(result_set, n_bins=n_bins)
|
|
220
237
|
|
|
221
238
|
return Histogram(
|
|
222
239
|
bounds,
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "PostBOUND"
|
|
7
|
-
version = "0.22.
|
|
7
|
+
version = "0.22.2"
|
|
8
8
|
authors = [{ name = "Rico Bergmann", email = "rico.bergmann1@tu-dresden.de" }]
|
|
9
9
|
description = "PostBOUND is a research framework to prototype and benchmark database query optimizers"
|
|
10
10
|
readme = "README.md"
|
|
@@ -5,6 +5,10 @@ the behaviours under test are about whether DuckDB *accepts* the SQL that PostBO
|
|
|
5
5
|
share the backend's assumptions about DuckDB's dialect and could only confirm a bug, not catch it. The synthetic
|
|
6
6
|
schema is created per test, so nothing is downloaded or written to disk.
|
|
7
7
|
|
|
8
|
+
`PreciseStatistics.histogram` is run here as well, because its SQL and the shape of the result set (in particular
|
|
9
|
+
how NULLs are grouped and sorted) are decided by the engine. Its decision logic is covered offline in
|
|
10
|
+
`tests/unit/test_db_stats.py`, which scripts the result sets observed here.
|
|
11
|
+
|
|
8
12
|
The pure formatting part of the EXPLAIN regression (the *duckdb* flavor of `format_quick`) is covered offline in
|
|
9
13
|
`tests/unit/test_qal_formatter.py`; the parsing of EXPLAIN output and the scan-to-table binding in
|
|
10
14
|
`tests/unit/test_duckdb.py`.
|
|
@@ -17,7 +21,8 @@ from pathlib import Path
|
|
|
17
21
|
|
|
18
22
|
import pytest
|
|
19
23
|
|
|
20
|
-
from postbound import ScanOperator, TableReference, parser, transform
|
|
24
|
+
from postbound import ColumnReference, ScanOperator, TableReference, parser, transform
|
|
25
|
+
from postbound.db import Histogram, PreciseStatistics
|
|
21
26
|
from postbound.duckdb import DuckDBDatabase
|
|
22
27
|
|
|
23
28
|
pytestmark = pytest.mark.embedded
|
|
@@ -42,6 +47,25 @@ def duckdb_instance() -> Iterator[DuckDBDatabase]:
|
|
|
42
47
|
instance.close()
|
|
43
48
|
|
|
44
49
|
|
|
50
|
+
def create_column(instance: DuckDBDatabase, values: list[int | None]) -> ColumnReference:
|
|
51
|
+
"""Fills a fresh table `h(v)` with `values` and returns the reference to its column."""
|
|
52
|
+
instance.execute_query("CREATE TABLE h (v INTEGER)")
|
|
53
|
+
rows = ", ".join("(NULL)" if value is None else f"({value})" for value in values)
|
|
54
|
+
instance.execute_query(f"INSERT INTO h VALUES {rows}")
|
|
55
|
+
return ColumnReference("v", TableReference("h"))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# -- PreciseStatistics.histogram ----------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_precise_histogram_builds_equi_depth_buckets_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
62
|
+
column = create_column(duckdb_instance, [1, 2, 3, 4, 5, 6, 7, 8])
|
|
63
|
+
|
|
64
|
+
hist = PreciseStatistics(duckdb_instance).histogram(column, n_bins=4)
|
|
65
|
+
|
|
66
|
+
assert hist == Histogram([2, 4, 6, 8], [2, 2, 2, 2], lower=1, bucket_interpolation="approx-uni")
|
|
67
|
+
|
|
68
|
+
|
|
45
69
|
# -- regression tests --------------------------------------------------------------------------------------
|
|
46
70
|
|
|
47
71
|
|
|
@@ -105,3 +129,28 @@ def test_query_plan_binds_scan_nodes_to_their_tables(duckdb_instance: DuckDBData
|
|
|
105
129
|
|
|
106
130
|
assert plan.base_table == TableReference("r")
|
|
107
131
|
assert plan.tables() == {TableReference("r")}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_precise_histogram_excludes_nulls_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
135
|
+
"""Regression guard for the unreleased histogram fixes: DuckDB returns the NULL group of the histogram's GROUP BY
|
|
136
|
+
query last as ``(None, 0)``. With fewer rows than bins, `_infer_histogram_bounds` turned it into the upper bound of
|
|
137
|
+
the histogram, and every estimate at or above the largest value raised a `TypeError` comparing with *None*.
|
|
138
|
+
"""
|
|
139
|
+
column = create_column(duckdb_instance, [1, 1, 2, 3, None, None])
|
|
140
|
+
|
|
141
|
+
hist = PreciseStatistics(duckdb_instance).histogram(column)
|
|
142
|
+
|
|
143
|
+
assert list(hist) == [(1, 2), (2, 1), (3, 1)]
|
|
144
|
+
assert hist.frequency_below(5) == 4
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_precise_histogram_of_a_single_distinct_value_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
|
|
148
|
+
"""Regression guard for the unreleased histogram fixes: the one-row result set was simplified to a plain tuple
|
|
149
|
+
before the bounds were inferred, which crashed. See `test_histogram_of_a_column_with_a_single_distinct_value` in
|
|
150
|
+
`tests/unit/test_db_stats.py`.
|
|
151
|
+
"""
|
|
152
|
+
column = create_column(duckdb_instance, [7, 7])
|
|
153
|
+
|
|
154
|
+
hist = PreciseStatistics(duckdb_instance).histogram(column)
|
|
155
|
+
|
|
156
|
+
assert list(hist) == [(7, 2)]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|