PostBOUND 0.22.0__tar.gz → 0.22.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {postbound-0.22.0 → postbound-0.22.2}/PKG-INFO +1 -1
  2. {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/PKG-INFO +1 -1
  3. {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/SOURCES.txt +1 -0
  4. {postbound-0.22.0 → postbound-0.22.2}/postbound/db/_db.py +52 -41
  5. {postbound-0.22.0 → postbound-0.22.2}/postbound/db/_stats.py +27 -10
  6. {postbound-0.22.0 → postbound-0.22.2}/postbound/duckdb/_duckdb.py +34 -4
  7. {postbound-0.22.0 → postbound-0.22.2}/postbound/qal/_formatter.py +29 -14
  8. {postbound-0.22.0 → postbound-0.22.2}/pyproject.toml +1 -1
  9. postbound-0.22.2/tests/test_duckdb.py +156 -0
  10. {postbound-0.22.0 → postbound-0.22.2}/LICENSE.txt +0 -0
  11. {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/dependency_links.txt +0 -0
  12. {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/requires.txt +0 -0
  13. {postbound-0.22.0 → postbound-0.22.2}/PostBOUND.egg-info/top_level.txt +0 -0
  14. {postbound-0.22.0 → postbound-0.22.2}/README.md +0 -0
  15. {postbound-0.22.0 → postbound-0.22.2}/postbound/__init__.py +0 -0
  16. {postbound-0.22.0 → postbound-0.22.2}/postbound/__init__.pyi +0 -0
  17. {postbound-0.22.0 → postbound-0.22.2}/postbound/_base.py +0 -0
  18. {postbound-0.22.0 → postbound-0.22.2}/postbound/_core.py +0 -0
  19. {postbound-0.22.0 → postbound-0.22.2}/postbound/_hints.py +0 -0
  20. {postbound-0.22.0 → postbound-0.22.2}/postbound/_ipython.py +0 -0
  21. {postbound-0.22.0 → postbound-0.22.2}/postbound/_pipelines.py +0 -0
  22. {postbound-0.22.0 → postbound-0.22.2}/postbound/_qep.py +0 -0
  23. {postbound-0.22.0 → postbound-0.22.2}/postbound/_stages.py +0 -0
  24. {postbound-0.22.0 → postbound-0.22.2}/postbound/bench.py +0 -0
  25. {postbound-0.22.0 → postbound-0.22.2}/postbound/db/__init__.py +0 -0
  26. {postbound-0.22.0 → postbound-0.22.2}/postbound/db/_cache.py +0 -0
  27. {postbound-0.22.0 → postbound-0.22.2}/postbound/duckdb/__init__.py +0 -0
  28. {postbound-0.22.0 → postbound-0.22.2}/postbound/mysql.py +0 -0
  29. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/__init__.py +0 -0
  30. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/__init__.pyi +0 -0
  31. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_cardinalities.py +0 -0
  32. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_dynprog.py +0 -0
  33. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_enumeration.py +0 -0
  34. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_helpers.py +0 -0
  35. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_native.py +0 -0
  36. {postbound-0.22.0 → postbound-0.22.2}/postbound/opt/_randomized.py +0 -0
  37. {postbound-0.22.0 → postbound-0.22.2}/postbound/parser.py +0 -0
  38. {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/__init__.py +0 -0
  39. {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_config.py +0 -0
  40. {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_ctl.py +0 -0
  41. {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_explain.py +0 -0
  42. {postbound-0.22.0 → postbound-0.22.2}/postbound/postgres/_pg.py +0 -0
  43. {postbound-0.22.0 → postbound-0.22.2}/postbound/qal/__init__.py +0 -0
  44. {postbound-0.22.0 → postbound-0.22.2}/postbound/qal/_qal.py +0 -0
  45. {postbound-0.22.0 → postbound-0.22.2}/postbound/relalg.py +0 -0
  46. {postbound-0.22.0 → postbound-0.22.2}/postbound/train/__init__.py +0 -0
  47. {postbound-0.22.0 → postbound-0.22.2}/postbound/train/_train.py +0 -0
  48. {postbound-0.22.0 → postbound-0.22.2}/postbound/transform.py +0 -0
  49. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/__init__.py +0 -0
  50. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/__init__.pyi +0 -0
  51. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/_df.py +0 -0
  52. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/_errors.py +0 -0
  53. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/collections.py +0 -0
  54. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/dicts.py +0 -0
  55. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/jsonize.py +0 -0
  56. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/logging.py +0 -0
  57. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/misc.py +0 -0
  58. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/num.py +0 -0
  59. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/nx.py +0 -0
  60. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/proc.py +0 -0
  61. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/stats.py +0 -0
  62. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/system.py +0 -0
  63. {postbound-0.22.0 → postbound-0.22.2}/postbound/util/typing.py +0 -0
  64. {postbound-0.22.0 → postbound-0.22.2}/postbound/validation.py +0 -0
  65. {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/__init__.py +0 -0
  66. {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/__init__.pyi +0 -0
  67. {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/fdl.py +0 -0
  68. {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/graphs.py +0 -0
  69. {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/optimizer.py +0 -0
  70. {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/plots.py +0 -0
  71. {postbound-0.22.0 → postbound-0.22.2}/postbound/vis/trees.py +0 -0
  72. {postbound-0.22.0 → postbound-0.22.2}/postbound/workloads.py +0 -0
  73. {postbound-0.22.0 → postbound-0.22.2}/setup.cfg +0 -0
  74. {postbound-0.22.0 → postbound-0.22.2}/tests/test_benchmarking.py +0 -0
  75. {postbound-0.22.0 → postbound-0.22.2}/tests/test_core.py +0 -0
  76. {postbound-0.22.0 → postbound-0.22.2}/tests/test_docs.py +0 -0
  77. {postbound-0.22.0 → postbound-0.22.2}/tests/test_postgres.py +0 -0
  78. {postbound-0.22.0 → postbound-0.22.2}/tests/test_qal.py +0 -0
  79. {postbound-0.22.0 → postbound-0.22.2}/tests/test_relalg.py +0 -0
  80. {postbound-0.22.0 → postbound-0.22.2}/tests/test_util.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: PostBOUND
3
- Version: 0.22.0
3
+ Version: 0.22.2
4
4
  Summary: PostBOUND is a research framework to prototype and benchmark database query optimizers
5
5
  Author-email: Rico Bergmann <rico.bergmann1@tu-dresden.de>
6
6
  License-Expression: Apache-2.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: PostBOUND
3
- Version: 0.22.0
3
+ Version: 0.22.2
4
4
  Summary: PostBOUND is a research framework to prototype and benchmark database query optimizers
5
5
  Author-email: Rico Bergmann <rico.bergmann1@tu-dresden.de>
6
6
  License-Expression: Apache-2.0
@@ -71,6 +71,7 @@ postbound/vis/trees.py
71
71
  tests/test_benchmarking.py
72
72
  tests/test_core.py
73
73
  tests/test_docs.py
74
+ tests/test_duckdb.py
74
75
  tests/test_postgres.py
75
76
  tests/test_qal.py
76
77
  tests/test_relalg.py
@@ -18,6 +18,7 @@ from __future__ import annotations
18
18
 
19
19
  import bisect
20
20
  import collections
21
+ import itertools
21
22
  import textwrap
22
23
  from abc import ABC, abstractmethod
23
24
  from collections.abc import Iterable, Iterator, Mapping, Sequence
@@ -1610,13 +1611,16 @@ class Histogram[T: _HistElem]:
1610
1611
 
1611
1612
  Currently, we only provide two high-level methods for cardinality estimation: `frequency_below` and `frequency_above`.
1612
1613
  The former estimates the number of elements that are less than or equal to a given value, while the latter estimates
1613
- the number of elements that are greater than a given value. If the queried value does not align perfectly with the
1614
- bucket bounds, the behavior depends on the selected `bucket_interpolation` strategy:
1614
+ the number of elements that are greater than a given value. Both estimates are based on the cumulative frequencies of
1615
+ the buckets, so equi-depth and arbitrary other bucketizations are handled alike. If the queried value lies on a
1616
+ bucket bound, both estimates are exact. Otherwise, the behavior depends on the selected `bucket_interpolation`
1617
+ strategy:
1615
1618
 
1616
- - "bound" is a conservative strategy. It *bounds* the frequency of the queried value by the frequency of the entire
1617
- bucket that contains it.
1619
+ - "bound" is a conservative strategy. It *bounds* the frequency by counting the entire bucket that contains the
1620
+ queried value, i.e. it always provides an *upper bound* on the true frequency.
1618
1621
  - "approx-uni" is a more aggressive strategy. It assumes that the values within each bucket are uniformly distributed
1619
- and uses a simple linear interpolation to estimate the frequency of the queried value.
1622
+ and uses a simple linear interpolation to estimate which share of the bucket that contains the queried value lies
1623
+ below the value. With this strategy, ``frequency_below(v) + frequency_above(v) == n_rows`` holds for every value.
1620
1624
 
1621
1625
  Histograms can be iterated over to access the individual (bucket bound, frequency) pairs. In addition, we provide
1622
1626
  *__getitem__* access to get the i-th bucket bound and its frequency.
@@ -1634,6 +1638,12 @@ class Histogram[T: _HistElem]:
1634
1638
  bucket_interpolation : Literal["approx-uni", "bound"], optional
1635
1639
  The strategy to estimate the frequency of values that fall into a bucket, but are not exactly on the bucket bounds.
1636
1640
  The default is "bound".
1641
+
1642
+ Raises
1643
+ ------
1644
+ ValueError
1645
+ If `bounds` is empty, if `bounds` and `frequencies` differ in length, or if `bucket_interpolation` is not a
1646
+ supported strategy.
1637
1647
  """
1638
1648
 
1639
1649
  def __init__(
@@ -1657,7 +1667,8 @@ class Histogram[T: _HistElem]:
1657
1667
  self._n_rows = sum(self._frequencies)
1658
1668
  self._lower = lower
1659
1669
  self._freq_per_bucket = self._n_rows // len(self._bounds)
1660
- self._bucket_interpolation = bucket_interpolation
1670
+ self._cumulative_frequencies = [0, *itertools.accumulate(self._frequencies)]
1671
+ self.bucket_interpolation = bucket_interpolation
1661
1672
 
1662
1673
  @property
1663
1674
  def bounds(self) -> Sequence[T]:
@@ -1707,22 +1718,34 @@ class Histogram[T: _HistElem]:
1707
1718
  )
1708
1719
  self._bucket_interpolation = strategy
1709
1720
 
1710
- def frequency_below(self, value: T) -> int:
1721
+ def frequency_below(self, value: T) -> float:
1711
1722
  """Estimate the frequency of values that are less than or equal to a given value.
1712
1723
 
1713
1724
  If the value does not align exactly with the bucket bounds, the behavior depends on the selected
1714
- `bucket_interpolation` strategy. See the class documentation for details.
1725
+ `bucket_interpolation` strategy. See the class documentation for details. Interpolated estimates are not
1726
+ rounded and can therefore be fractional.
1727
+
1728
+ Raises
1729
+ ------
1730
+ ValueError
1731
+ If the "approx-uni" strategy has to interpolate within a bucket, but the value is not numeric or temporal.
1715
1732
  """
1716
- upper_idx = bisect.bisect_right(self._bounds, value)
1717
- if upper_idx == 0:
1733
+ if value < self._lower:
1718
1734
  return 0
1719
- elif upper_idx == len(self._bounds):
1720
- return self._n_rows
1721
1735
 
1722
1736
  if self._bucket_interpolation == "bound":
1723
- return upper_idx * self._freq_per_bucket
1737
+ # value lies in bucket containing_idx (or on its upper bound), i.e. all later buckets are entirely > value
1738
+ containing_idx = bisect.bisect_left(self._bounds, value)
1739
+ if containing_idx == len(self._bounds):
1740
+ return self._n_rows
1741
+ return self._cumulative_frequencies[containing_idx + 1]
1724
1742
 
1725
- assert self._bucket_interpolation == "approx-uni"
1743
+ # value lies in bucket upper_idx, i.e. all buckets before upper_idx are entirely <= value
1744
+ upper_idx = bisect.bisect_right(self._bounds, value)
1745
+ if upper_idx == len(self._bounds):
1746
+ return self._n_rows
1747
+
1748
+ complete_buckets = self._cumulative_frequencies[upper_idx]
1726
1749
 
1727
1750
  if not isinstance(value, (int, float, date, datetime)):
1728
1751
  raise ValueError(
@@ -1731,44 +1754,32 @@ class Histogram[T: _HistElem]:
1731
1754
  f"for histograms of type {type(value).__name__}."
1732
1755
  )
1733
1756
 
1734
- lower_idx = upper_idx - 1
1757
+ lower_bound = self._bounds[upper_idx - 1] if upper_idx > 0 else self._lower
1735
1758
  upper_bound = self._bounds[upper_idx]
1736
- lower_bound = self._bounds[lower_idx]
1737
1759
  in_bucket_frac = (value - lower_bound) / (upper_bound - lower_bound)
1738
- return (lower_idx + in_bucket_frac) * self._freq_per_bucket
1760
+ return complete_buckets + in_bucket_frac * self._frequencies[upper_idx]
1739
1761
 
1740
- def frequency_above(self, value: T) -> int:
1762
+ def frequency_above(self, value: T) -> float:
1741
1763
  """Estimate the frequency of values that are greater than a given value.
1742
1764
 
1743
1765
  Due to the way we model histograms, we cannot directly estimate the frequency of values greater or equal to a given
1744
1766
  value, but only the frequency of values that are strictly greater than it.
1745
1767
 
1746
1768
  If the value does not align exactly with the bucket bounds, the behavior depends on the selected
1747
- `bucket_interpolation` strategy. See the class documentation for details.
1748
- """
1749
- lower_idx = bisect.bisect_left(self._bounds, value)
1750
- if lower_idx == 0:
1751
- return self._n_rows
1752
- elif lower_idx == len(self._bounds):
1753
- return 0
1754
-
1755
- if self._bucket_interpolation == "bound":
1756
- return lower_idx * self._freq_per_bucket
1769
+ `bucket_interpolation` strategy. See the class documentation for details. Interpolated estimates are not
1770
+ rounded and can therefore be fractional.
1757
1771
 
1758
- assert self._bucket_interpolation == "approx-uni"
1759
-
1760
- if not isinstance(value, (int, float, date, datetime)):
1761
- raise ValueError(
1762
- "Cannot estimate frequency within bucket. "
1763
- f"Strategy '{self._bucket_interpolation}' is unknown or unsupported "
1764
- f"for histograms of type {type(value).__name__}."
1765
- )
1772
+ Raises
1773
+ ------
1774
+ ValueError
1775
+ If the "approx-uni" strategy has to interpolate within a bucket, but the value is not numeric or temporal.
1776
+ """
1777
+ if self._bucket_interpolation == "approx-uni":
1778
+ return self._n_rows - self.frequency_below(value)
1766
1779
 
1767
- upper_idx = lower_idx + 1
1768
- upper_bound = self._bounds[upper_idx]
1769
- lower_bound = self._bounds[lower_idx]
1770
- in_bucket_frac = (value - lower_bound) / (upper_bound - lower_bound)
1771
- return (upper_idx + in_bucket_frac) * self._freq_per_bucket
1780
+ # all buckets before upper_idx are entirely <= value, but the bucket containing value might hold larger values
1781
+ upper_idx = bisect.bisect_right(self._bounds, value)
1782
+ return self._n_rows - self._cumulative_frequencies[upper_idx]
1772
1783
 
1773
1784
  def __len__(self) -> int:
1774
1785
  return len(self._bounds)
@@ -31,12 +31,16 @@ from ._db import Database, Histogram, HistogramApproximation, MostCommonValues,
31
31
 
32
32
 
33
33
  def _infer_histogram_bounds[T](
34
- frequencies: Sequence[tuple[T, int]], *, n_bins: int, n_rows: int
34
+ frequencies: Sequence[tuple[T, int]], *, n_bins: int
35
35
  ) -> tuple[T, Sequence[T], Sequence[int]]:
36
- """Infer the bucket bounds and frequencies for a histogram from a list of (value, frequency) pairs."""
36
+ """Infer the bucket bounds and frequencies for a histogram from a list of (value, frequency) pairs.
37
+
38
+ The pairs must be sorted by value and must not contain NULL values.
39
+ """
37
40
  if not frequencies:
38
41
  raise ValueError("Cannot infer histogram bounds from empty frequency list")
39
42
 
43
+ n_rows = sum(freq for _, freq in frequencies)
40
44
  bucket_size = n_rows // n_bins
41
45
 
42
46
  bounds: list[T] = []
@@ -50,6 +54,11 @@ def _infer_histogram_bounds[T](
50
54
  buckets.append(cumulative_freq)
51
55
  cumulative_freq = 0
52
56
 
57
+ if cumulative_freq:
58
+ # the remaining values did not fill an entire bucket, but they still need to be part of the histogram
59
+ bounds.append(frequencies[-1][0])
60
+ buckets.append(cumulative_freq)
61
+
53
62
  return frequencies[0][0], bounds, buckets
54
63
 
55
64
 
@@ -188,7 +197,7 @@ class PreciseStatistics(StatisticsCatalog):
188
197
  limit_clause = Limit(limit=k) if k is not None and k > 0 else None
189
198
  sql = as_query(select_clause, from_clause, group_clause, order_clause, limit_clause)
190
199
 
191
- result_set = self._db.execute_query(sql)
200
+ result_set = self._db.execute_query(sql, raw=True)
192
201
  return MostCommonValues(result_set)
193
202
 
194
203
  def histogram(
@@ -198,10 +207,18 @@ class PreciseStatistics(StatisticsCatalog):
198
207
 
199
208
  In addition to `StatisticsCatalog.histogram`, this implementation allows to customize the number of buckets via
200
209
  `n_bins`. Since the histogram has to be constructed from scratch, `n_bins` is required and passing *None*
201
- raises a `ValueError`.
210
+ raises a `ValueError`. NULL values are not part of the histogram (use `null_frac` for them).
211
+
212
+ Since a value is never split across buckets, the histogram is only approximately equi-depth: a very frequent
213
+ value fills its bucket beyond the target size, and fewer than `n_bins` buckets might be created.
214
+
215
+ Raises
216
+ ------
217
+ ValueError
218
+ If `n_bins` is *None* or not positive, or if the column does not contain any non-NULL values.
202
219
  """
203
- if n_bins is None:
204
- raise ValueError("n_bins must be set for emulated histogram")
220
+ if n_bins is None or n_bins < 1:
221
+ raise ValueError(f"n_bins must be a positive number for emulated histogram, not {n_bins}")
205
222
 
206
223
  if not ColumnReference.assert_bound(column):
207
224
  raise UnboundColumnError(column)
@@ -210,13 +227,13 @@ class PreciseStatistics(StatisticsCatalog):
210
227
 
211
228
  select_clause = Select([Projection.column(column), Projection.create_count(column, target_name="n")])
212
229
  from_clause = From.create_for(column.table)
230
+ where_clause = Where(as_predicate(column, "IS NOT NULL"))
213
231
  group_clause = GroupBy.create_for(column)
214
232
  order_clause = OrderBy.create_for(column)
215
- sql = as_query(select_clause, from_clause, group_clause, order_clause)
233
+ sql = as_query(select_clause, from_clause, where_clause, group_clause, order_clause)
216
234
 
217
- result_set = self._db.execute_query(sql)
218
- n_rows = self.total_rows(column.table)
219
- lo, bounds, buckets = _infer_histogram_bounds(result_set, n_bins=n_bins, n_rows=int(n_rows))
235
+ result_set = self._db.execute_query(sql, raw=True)
236
+ lo, bounds, buckets = _infer_histogram_bounds(result_set, n_bins=n_bins)
220
237
 
221
238
  return Histogram(
222
239
  bounds,
@@ -500,11 +500,41 @@ class DuckDBStatistics(StatisticsCatalog):
500
500
 
501
501
  def _bind_node_to_table(scan_node: dict, *, query: SqlQuery) -> TableReference | None:
502
502
  tables = query.tables()
503
- relname = scan_node.get("extra_info", {}).get("Table", "")
503
+ relname: str = scan_node.get("extra_info", {}).get("Table", "")
504
504
  if not relname:
505
505
  return None
506
506
 
507
- candidate_tables = [tab for tab in tables if tab.full_name == relname]
507
+ # DuckDB reports fully qualified table names in the scan node, while the query might reference just the table name
508
+ # or a schema-qualified name. We try our best to match these differences.
509
+ # The code below is pretty ugly, but works for now ¯\_(ツ)_/¯
510
+ n_comps = relname.count(".")
511
+ match n_comps:
512
+ case 0:
513
+ candidate_tables = [tab for tab in tables if tab.full_name == relname]
514
+ case 1:
515
+ schema, name = relname.split(".")
516
+ needle = TableReference(name, schema=schema)
517
+ candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
518
+ if not candidate_tables:
519
+ needle = TableReference(name)
520
+ candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
521
+ case 2:
522
+ cat, schema, name = relname.split(".")
523
+ needle = TableReference(name, schema=schema, catalog=cat)
524
+ candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
525
+ if not candidate_tables:
526
+ needle = TableReference(name, schema=schema)
527
+ candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
528
+ if not candidate_tables:
529
+ needle = TableReference(name)
530
+ candidate_tables = [tab for tab in tables if tab.drop_alias() == needle]
531
+ case _:
532
+ warnings.warn(
533
+ f"Unexpected table format '{relname}'. Only binding based on final name component", stacklevel=2
534
+ )
535
+ needle = relname.split(".")[-1]
536
+ candidate_tables = [tab for tab in tables if tab.full_name == needle]
537
+
508
538
  if not candidate_tables:
509
539
  return None
510
540
  elif len(candidate_tables) == 1:
@@ -655,7 +685,7 @@ class DuckDBOptimizer(OptimizerInterface):
655
685
 
656
686
  raw_explain = result_set[1]
657
687
  parsed = json.loads(raw_explain)
658
- return parse_duckdb_plan(parsed[0], query=query)
688
+ return parse_duckdb_plan(parsed, query=query)
659
689
 
660
690
  def parse_plan(self, plan: ResultSet, *, query: SqlQuery | None = None) -> QueryPlan | None:
661
691
  # Similar to the Postgres implementation of parse_plan(), we try to be graceful
@@ -814,7 +844,7 @@ class DuckDBHintService(HintService):
814
844
 
815
845
  def format_query(self, query: SqlQuery) -> str:
816
846
  # DuckDB uses the Postgres SQL dialect, so this part is easy..
817
- return qal.format_quick(query, flavor="postgres")
847
+ return qal.format_quick(query, flavor="duckdb")
818
848
 
819
849
  def supports_hint(self, hint: PhysicalOperator | HintType) -> bool:
820
850
  return hint in {
@@ -59,7 +59,7 @@ from ._qal import (
59
59
  DefaultIndent = 2
60
60
  """The default amount of whitespace that is used to indent specific parts of the SQL query."""
61
61
 
62
- SqlDialect = Literal["vanilla", "postgres"]
62
+ SqlDialect = Literal["vanilla", "postgres", "duckdb"]
63
63
  """The different flavors of SQL syntax that are supported by the formatter."""
64
64
 
65
65
 
@@ -156,16 +156,31 @@ class FormattingCaseExpression(CaseExpression):
156
156
 
157
157
 
158
158
  def _quick_format_explain(explain_clause: Explain, *, flavor: SqlDialect) -> list[str]:
159
- if flavor != "postgres":
160
- return [str(explain_clause)]
159
+ match flavor:
160
+ case "vanilla":
161
+ return [str(explain_clause)]
162
+
163
+ case "postgres":
164
+ options = ["SETTINGS", "SUMMARY", "VERBOSE"]
165
+ if explain_clause.analyze:
166
+ options.append("ANALYZE")
167
+ options.append("FORMAT JSON")
161
168
 
162
- options = ["SETTINGS", "SUMMARY", "VERBOSE"]
163
- if explain_clause.analyze:
164
- options.append("ANALYZE")
165
- options.append("FORMAT JSON")
169
+ args = ", ".join(options)
170
+ return [f"EXPLAIN ({args})"]
166
171
 
167
- args = ", ".join(options)
168
- return [f"EXPLAIN ({args})"]
172
+ case "duckdb":
173
+ options = []
174
+ if explain_clause.analyze:
175
+ options.append("ANALYZE")
176
+ options.append("FORMAT JSON")
177
+
178
+ args = ", ".join(options)
179
+ return [f"EXPLAIN ({args})"]
180
+
181
+ case _:
182
+ warnings.warn("Unknown SQL flavor for EXPLAIN clauses. Falling back to naive formatting", stacklevel=2)
183
+ return [str(explain_clause)]
169
184
 
170
185
 
171
186
  def _quick_format_cte(cte_clause: CommonTableExpression, *, flavor: SqlDialect) -> list[str]:
@@ -547,7 +562,7 @@ def _quick_format_limit(limit_clause: Limit, *, flavor: SqlDialect) -> list[str]
547
562
  else:
548
563
  return []
549
564
 
550
- case "postgres" if limit_clause.fetch_direction in {"first", "next"}:
565
+ case "postgres" | "duckdb" if limit_clause.fetch_direction in {"first", "next"}:
551
566
  if limit_clause.limit and limit_clause.offset:
552
567
  return [
553
568
  f"LIMIT {limit_clause.limit}",
@@ -559,7 +574,7 @@ def _quick_format_limit(limit_clause: Limit, *, flavor: SqlDialect) -> list[str]
559
574
  return [f"OFFSET {limit_clause.offset}"]
560
575
  return []
561
576
 
562
- case "postgres" if limit_clause.fetch_direction in {"prior", "last"}:
577
+ case "postgres" | "duckdb" if limit_clause.fetch_direction in {"prior", "last"}:
563
578
  warnings.warn(
564
579
  "Postgres does not support FETCH PRIOR and FETCH LAST. Falling back to naive formatting", stacklevel=2
565
580
  )
@@ -673,9 +688,9 @@ def _expression_prettifier[T: SqlExpression](
673
688
  indentation=indentation,
674
689
  )
675
690
  return (
676
- target(replaced_cast, typ, type_params=params)
677
- if flavor == "vanilla"
678
- else _PostgresCastExpression(replaced_cast, typ, type_params=params, array_type=array)
691
+ _PostgresCastExpression(replaced_cast, typ, type_params=params, array_type=array)
692
+ if flavor == "postgres"
693
+ else target(replaced_cast, typ, type_params=params)
679
694
  )
680
695
 
681
696
  case MathExpression(op, lhs, rhs):
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "PostBOUND"
7
- version = "0.22.0"
7
+ version = "0.22.2"
8
8
  authors = [{ name = "Rico Bergmann", email = "rico.bergmann1@tu-dresden.de" }]
9
9
  description = "PostBOUND is a research framework to prototype and benchmark database query optimizers"
10
10
  readme = "README.md"
@@ -0,0 +1,156 @@
1
+ """Tests for `postbound.duckdb` -- the DuckDB backend, run against a real in-memory DuckDB (via quacklab).
2
+
3
+ All tests here are tier 1 (`embedded`): they need a real DuckDB engine, but no server or database file, because
4
+ the behaviours under test are about whether DuckDB *accepts* the SQL that PostBOUND generates. A double would
5
+ share the backend's assumptions about DuckDB's dialect and could only confirm a bug, not catch it. The synthetic
6
+ schema is created per test, so nothing is downloaded or written to disk.
7
+
8
+ `PreciseStatistics.histogram` is run here as well, because its SQL and the shape of the result set (in particular
9
+ how NULLs are grouped and sorted) are decided by the engine. Its decision logic is covered offline in
10
+ `tests/unit/test_db_stats.py`, which scripts the result sets observed here.
11
+
12
+ The pure formatting part of the EXPLAIN regression (the *duckdb* flavor of `format_quick`) is covered offline in
13
+ `tests/unit/test_qal_formatter.py`; the parsing of EXPLAIN output and the scan-to-table binding in
14
+ `tests/unit/test_duckdb.py`.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from collections.abc import Iterator
20
+ from pathlib import Path
21
+
22
+ import pytest
23
+
24
+ from postbound import ColumnReference, ScanOperator, TableReference, parser, transform
25
+ from postbound.db import Histogram, PreciseStatistics
26
+ from postbound.duckdb import DuckDBDatabase
27
+
28
+ pytestmark = pytest.mark.embedded
29
+
30
+ # -- fixtures -----------------------------------------------------------------------------------------------
31
+
32
+
33
+ def parse(sql: str):
34
+ return parser.parse_query(sql, bind_columns=False)
35
+
36
+
37
+ FILTER_QUERY = parse("SELECT * FROM r WHERE r.a = 1")
38
+
39
+
40
+ @pytest.fixture
41
+ def duckdb_instance() -> Iterator[DuckDBDatabase]:
42
+ """An in-memory DuckDB with a tiny table `r(a, b)`. The constructor does not register it on the pool."""
43
+ instance = DuckDBDatabase(Path(":memory:"))
44
+ instance.execute_query("CREATE TABLE r (a INTEGER, b INTEGER)")
45
+ instance.execute_query("INSERT INTO r VALUES (1, 10), (2, 20), (3, 30)")
46
+ yield instance
47
+ instance.close()
48
+
49
+
50
+ def create_column(instance: DuckDBDatabase, values: list[int | None]) -> ColumnReference:
51
+ """Fills a fresh table `h(v)` with `values` and returns the reference to its column."""
52
+ instance.execute_query("CREATE TABLE h (v INTEGER)")
53
+ rows = ", ".join("(NULL)" if value is None else f"({value})" for value in values)
54
+ instance.execute_query(f"INSERT INTO h VALUES {rows}")
55
+ return ColumnReference("v", TableReference("h"))
56
+
57
+
58
+ # -- PreciseStatistics.histogram ----------------------------------------------------------------------------
59
+
60
+
61
+ def test_precise_histogram_builds_equi_depth_buckets_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
62
+ column = create_column(duckdb_instance, [1, 2, 3, 4, 5, 6, 7, 8])
63
+
64
+ hist = PreciseStatistics(duckdb_instance).histogram(column, n_bins=4)
65
+
66
+ assert hist == Histogram([2, 4, 6, 8], [2, 2, 2, 2], lower=1, bucket_interpolation="approx-uni")
67
+
68
+
69
+ # -- regression tests --------------------------------------------------------------------------------------
70
+
71
+
72
+ def test_hint_service_formats_explain_queries_without_postgres_only_options(duckdb_instance: DuckDBDatabase) -> None:
73
+ """Regression guard for the v0.22.1 DuckDB EXPLAIN fix: `DuckDBHintService.format_query` used to call
74
+ `format_quick(..., flavor="postgres")`. Since v0.22.0 that flavor writes `EXPLAIN (SETTINGS, SUMMARY,
75
+ VERBOSE, FORMAT JSON)`, which DuckDB does not support, so the backend could not run any EXPLAIN query.
76
+ """
77
+ explain_query = transform.as_explain(FILTER_QUERY)
78
+
79
+ formatted = duckdb_instance.hinting().format_query(explain_query)
80
+
81
+ header = formatted.splitlines()[0]
82
+ assert header == "EXPLAIN (FORMAT JSON)"
83
+
84
+
85
+ def test_query_plan_of_a_parsed_query_runs_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
86
+ """Regression guard for the v0.22.1 DuckDB EXPLAIN fix: `DuckDBOptimizer.query_plan` wraps a `SqlQuery` in an
87
+ EXPLAIN and formats it through the hint service, which used the Postgres flavor and thereby emitted the
88
+ Postgres-only `SETTINGS`/`SUMMARY`/`VERBOSE` options. DuckDB rejected the statement (`NotImplementedException: Unimplemented explain type: settings`)
89
+ before planning even started. (Plain-string queries took a separate hand-written `EXPLAIN (FORMAT JSON)` path
90
+ and were unaffected, which is why this test passes a parsed query.)
91
+ """
92
+ plan = duckdb_instance.optimizer().query_plan(FILTER_QUERY)
93
+
94
+ assert plan.operator == ScanOperator.SequentialScan
95
+
96
+
97
+ def test_execute_query_runs_a_parsed_explain_query_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
98
+ """Regression guard for the v0.22.1 DuckDB EXPLAIN fix, on the `DuckDBDatabase.execute_query` path: it formats
99
+ every `SqlQuery` through the hint service as well, so executing an `EXPLAIN` / `EXPLAIN ANALYZE` query object
100
+ directly failed with the same `NotImplementedException` as `query_plan`.
101
+ """
102
+ explain_queries = [transform.as_explain(FILTER_QUERY), transform.as_explain_analyze(FILTER_QUERY)]
103
+
104
+ result_sets = [duckdb_instance.execute_query(query, raw=True) for query in explain_queries]
105
+
106
+ assert [result_set[0][0] for result_set in result_sets] == ["physical_plan", "analyzed_plan"]
107
+
108
+
109
+ def test_analyze_plan_returns_the_measured_plan(duckdb_instance: DuckDBDatabase) -> None:
110
+ """Regression guard for the v0.22.1 DuckDB `analyze_plan` fix: `DuckDBOptimizer.analyze_plan` passed `parsed[0]`
111
+ to `parse_duckdb_plan`, assuming the JSON *list* that plain `EXPLAIN (FORMAT JSON)` returns. With ANALYZE,
112
+ DuckDB returns a single JSON *object* (a profiling root with the plan below `children`), so indexing it with `0`
113
+ raised a `KeyError` and no query could be analyzed.
114
+ """
115
+ plan = duckdb_instance.optimizer().analyze_plan(FILTER_QUERY)
116
+
117
+ assert plan.operator == ScanOperator.SequentialScan
118
+ assert plan.actual_cardinality == 1
119
+
120
+
121
+ def test_query_plan_binds_scan_nodes_to_their_tables(duckdb_instance: DuckDBDatabase) -> None:
122
+ """Regression guard for the v0.22.1 DuckDB table-binding fix, end to end: `_bind_node_to_table` compared the
123
+ `Table` entry of a scan node with `TableReference.full_name` by plain string equality, but DuckDB reports fully
124
+ qualified names (`memory.main.r` here) while the query references just `r`. Every scan node stayed unbound, so
125
+ `query_plan` returned plans without any tables. The offline tests in `tests/unit/test_duckdb.py` pin the captured
126
+ output; this one guards against DuckDB changing the reported name format underneath them.
127
+ """
128
+ plan = duckdb_instance.optimizer().query_plan(FILTER_QUERY)
129
+
130
+ assert plan.base_table == TableReference("r")
131
+ assert plan.tables() == {TableReference("r")}
132
+
133
+
134
+ def test_precise_histogram_excludes_nulls_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
135
+ """Regression guard for the unreleased histogram fixes: DuckDB returns the NULL group of the histogram's GROUP BY
136
+ query last as ``(None, 0)``. With fewer rows than bins, `_infer_histogram_bounds` turned it into the upper bound of
137
+ the histogram, and every estimate at or above the largest value raised a `TypeError` comparing with *None*.
138
+ """
139
+ column = create_column(duckdb_instance, [1, 1, 2, 3, None, None])
140
+
141
+ hist = PreciseStatistics(duckdb_instance).histogram(column)
142
+
143
+ assert list(hist) == [(1, 2), (2, 1), (3, 1)]
144
+ assert hist.frequency_below(5) == 4
145
+
146
+
147
+ def test_precise_histogram_of_a_single_distinct_value_on_duckdb(duckdb_instance: DuckDBDatabase) -> None:
148
+ """Regression guard for the unreleased histogram fixes: the one-row result set was simplified to a plain tuple
149
+ before the bounds were inferred, which crashed. See `test_histogram_of_a_column_with_a_single_distinct_value` in
150
+ `tests/unit/test_db_stats.py`.
151
+ """
152
+ column = create_column(duckdb_instance, [7, 7])
153
+
154
+ hist = PreciseStatistics(duckdb_instance).histogram(column)
155
+
156
+ assert list(hist) == [(7, 2)]
File without changes
File without changes
File without changes
File without changes
File without changes