PostBOUND 0.23.1__tar.gz → 0.23.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {postbound-0.23.1 → postbound-0.23.2}/PKG-INFO +1 -1
  2. {postbound-0.23.1 → postbound-0.23.2}/PostBOUND.egg-info/PKG-INFO +1 -1
  3. {postbound-0.23.1 → postbound-0.23.2}/postbound/db/_db.py +7 -6
  4. {postbound-0.23.1 → postbound-0.23.2}/postbound/duckdb/_duckdb.py +91 -53
  5. {postbound-0.23.1 → postbound-0.23.2}/postbound/parser.py +8 -5
  6. {postbound-0.23.1 → postbound-0.23.2}/pyproject.toml +1 -1
  7. {postbound-0.23.1 → postbound-0.23.2}/tests/test_duckdb.py +254 -3
  8. {postbound-0.23.1 → postbound-0.23.2}/tests/test_qal.py +25 -0
  9. {postbound-0.23.1 → postbound-0.23.2}/LICENSE.txt +0 -0
  10. {postbound-0.23.1 → postbound-0.23.2}/PostBOUND.egg-info/SOURCES.txt +0 -0
  11. {postbound-0.23.1 → postbound-0.23.2}/PostBOUND.egg-info/dependency_links.txt +0 -0
  12. {postbound-0.23.1 → postbound-0.23.2}/PostBOUND.egg-info/requires.txt +0 -0
  13. {postbound-0.23.1 → postbound-0.23.2}/PostBOUND.egg-info/top_level.txt +0 -0
  14. {postbound-0.23.1 → postbound-0.23.2}/README.md +0 -0
  15. {postbound-0.23.1 → postbound-0.23.2}/postbound/__init__.py +0 -0
  16. {postbound-0.23.1 → postbound-0.23.2}/postbound/__init__.pyi +0 -0
  17. {postbound-0.23.1 → postbound-0.23.2}/postbound/_base.py +0 -0
  18. {postbound-0.23.1 → postbound-0.23.2}/postbound/_core.py +0 -0
  19. {postbound-0.23.1 → postbound-0.23.2}/postbound/_hints.py +0 -0
  20. {postbound-0.23.1 → postbound-0.23.2}/postbound/_ipython.py +0 -0
  21. {postbound-0.23.1 → postbound-0.23.2}/postbound/_pipelines.py +0 -0
  22. {postbound-0.23.1 → postbound-0.23.2}/postbound/_qep.py +0 -0
  23. {postbound-0.23.1 → postbound-0.23.2}/postbound/_stages.py +0 -0
  24. {postbound-0.23.1 → postbound-0.23.2}/postbound/bench.py +0 -0
  25. {postbound-0.23.1 → postbound-0.23.2}/postbound/db/__init__.py +0 -0
  26. {postbound-0.23.1 → postbound-0.23.2}/postbound/db/_cache.py +0 -0
  27. {postbound-0.23.1 → postbound-0.23.2}/postbound/db/_stats.py +0 -0
  28. {postbound-0.23.1 → postbound-0.23.2}/postbound/duckdb/__init__.py +0 -0
  29. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/__init__.py +0 -0
  30. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/__init__.pyi +0 -0
  31. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/_cardinalities.py +0 -0
  32. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/_dynprog.py +0 -0
  33. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/_enumeration.py +0 -0
  34. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/_helpers.py +0 -0
  35. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/_native.py +0 -0
  36. {postbound-0.23.1 → postbound-0.23.2}/postbound/opt/_randomized.py +0 -0
  37. {postbound-0.23.1 → postbound-0.23.2}/postbound/postgres/__init__.py +0 -0
  38. {postbound-0.23.1 → postbound-0.23.2}/postbound/postgres/_config.py +0 -0
  39. {postbound-0.23.1 → postbound-0.23.2}/postbound/postgres/_ctl.py +0 -0
  40. {postbound-0.23.1 → postbound-0.23.2}/postbound/postgres/_explain.py +0 -0
  41. {postbound-0.23.1 → postbound-0.23.2}/postbound/postgres/_pg.py +0 -0
  42. {postbound-0.23.1 → postbound-0.23.2}/postbound/qal/__init__.py +0 -0
  43. {postbound-0.23.1 → postbound-0.23.2}/postbound/qal/_formatter.py +0 -0
  44. {postbound-0.23.1 → postbound-0.23.2}/postbound/qal/_qal.py +0 -0
  45. {postbound-0.23.1 → postbound-0.23.2}/postbound/relalg.py +0 -0
  46. {postbound-0.23.1 → postbound-0.23.2}/postbound/train/__init__.py +0 -0
  47. {postbound-0.23.1 → postbound-0.23.2}/postbound/train/_train.py +0 -0
  48. {postbound-0.23.1 → postbound-0.23.2}/postbound/transform.py +0 -0
  49. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/__init__.py +0 -0
  50. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/__init__.pyi +0 -0
  51. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/_df.py +0 -0
  52. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/_errors.py +0 -0
  53. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/collections.py +0 -0
  54. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/dicts.py +0 -0
  55. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/jsonize.py +0 -0
  56. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/logging.py +0 -0
  57. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/misc.py +0 -0
  58. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/num.py +0 -0
  59. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/nx.py +0 -0
  60. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/proc.py +0 -0
  61. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/stats.py +0 -0
  62. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/system.py +0 -0
  63. {postbound-0.23.1 → postbound-0.23.2}/postbound/util/typing.py +0 -0
  64. {postbound-0.23.1 → postbound-0.23.2}/postbound/validation.py +0 -0
  65. {postbound-0.23.1 → postbound-0.23.2}/postbound/vis/__init__.py +0 -0
  66. {postbound-0.23.1 → postbound-0.23.2}/postbound/vis/__init__.pyi +0 -0
  67. {postbound-0.23.1 → postbound-0.23.2}/postbound/vis/fdl.py +0 -0
  68. {postbound-0.23.1 → postbound-0.23.2}/postbound/vis/graphs.py +0 -0
  69. {postbound-0.23.1 → postbound-0.23.2}/postbound/vis/optimizer.py +0 -0
  70. {postbound-0.23.1 → postbound-0.23.2}/postbound/vis/plots.py +0 -0
  71. {postbound-0.23.1 → postbound-0.23.2}/postbound/vis/trees.py +0 -0
  72. {postbound-0.23.1 → postbound-0.23.2}/postbound/workloads.py +0 -0
  73. {postbound-0.23.1 → postbound-0.23.2}/setup.cfg +0 -0
  74. {postbound-0.23.1 → postbound-0.23.2}/tests/test_benchmarking.py +0 -0
  75. {postbound-0.23.1 → postbound-0.23.2}/tests/test_core.py +0 -0
  76. {postbound-0.23.1 → postbound-0.23.2}/tests/test_docs.py +0 -0
  77. {postbound-0.23.1 → postbound-0.23.2}/tests/test_postgres.py +0 -0
  78. {postbound-0.23.1 → postbound-0.23.2}/tests/test_relalg.py +0 -0
  79. {postbound-0.23.1 → postbound-0.23.2}/tests/test_util.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: PostBOUND
3
- Version: 0.23.1
3
+ Version: 0.23.2
4
4
  Summary: PostBOUND is a research framework to prototype and benchmark database query optimizers
5
5
  Author-email: Rico Bergmann <rico.bergmann1@tu-dresden.de>
6
6
  License-Expression: Apache-2.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: PostBOUND
3
- Version: 0.23.1
3
+ Version: 0.23.2
4
4
  Summary: PostBOUND is a research framework to prototype and benchmark database query optimizers
5
5
  Author-email: Rico Bergmann <rico.bergmann1@tu-dresden.de>
6
6
  License-Expression: Apache-2.0
@@ -35,6 +35,7 @@ from .._core import (
35
35
  TableReference,
36
36
  UnboundColumnError,
37
37
  VirtualTableError,
38
+ normalize,
38
39
  )
39
40
  from .._hints import (
40
41
  HintType,
@@ -1039,10 +1040,10 @@ class DatabaseSchema(ABC, Mapping[TableReference, TableInfo]):
1039
1040
  **Hint for implementors:** the default implementation of this method (transitively) relies on the
1040
1041
  *information_schema.columns* view.
1041
1042
  """
1042
- column = column.name if isinstance(column, ColumnReference) else column
1043
+ column = normalize(column.name if isinstance(column, ColumnReference) else column)
1043
1044
 
1044
1045
  for candidate in candidate_tables:
1045
- candidate_cols = {col.name for col in self.columns(candidate)}
1046
+ candidate_cols = {normalize(col.name) for col in self.columns(candidate)}
1046
1047
  if column in candidate_cols:
1047
1048
  return candidate
1048
1049
 
@@ -1104,8 +1105,8 @@ class DatabaseSchema(ABC, Mapping[TableReference, TableInfo]):
1104
1105
  result_set = cur.fetchall()
1105
1106
  assert result_set is not None
1106
1107
 
1107
- pk_cols = {row[0] for row in result_set}
1108
- requested_cols = {col.name for col in column}
1108
+ pk_cols = {normalize(row[0]) for row in result_set}
1109
+ requested_cols = {normalize(col.name) for col in column}
1109
1110
  return pk_cols == requested_cols
1110
1111
 
1111
1112
  def primary_key_of(self, table: TableReference | str) -> PrimaryKeyRef | None:
@@ -1636,9 +1637,9 @@ class DatabaseSchema(ABC, Mapping[TableReference, TableInfo]):
1636
1637
 
1637
1638
  unique_constraints: dict[str, set[str]] = collections.defaultdict(set)
1638
1639
  for constraint, col in result_set:
1639
- unique_constraints[constraint].add(col)
1640
+ unique_constraints[constraint].add(normalize(col))
1640
1641
 
1641
- target_cols = {col.name for col in column}
1642
+ target_cols = {normalize(col.name) for col in column}
1642
1643
  return any(target_cols == cols for cols in unique_constraints.values())
1643
1644
 
1644
1645
  def as_graph(self) -> nx.DiGraph:
@@ -47,6 +47,7 @@ from ..db import (
47
47
  MostCommonValues,
48
48
  OptimizerInterface,
49
49
  PreciseStatistics,
50
+ ResultRow,
50
51
  ResultSet,
51
52
  StatisticsCatalog,
52
53
  UnsupportedDatabaseFeatureError,
@@ -256,15 +257,11 @@ class DuckDBSchema(DatabaseSchema):
256
257
 
257
258
  def __init__(self, db: DuckDBDatabase) -> None:
258
259
  super().__init__(db, prep_placeholder="?", default_schema="main", default_catalog=db.database_name())
259
- self._tables: set[TableReference] = set()
260
260
 
261
261
  @override
262
262
  def tables(
263
263
  self, *, catalog: str = "", schema: str = "", include_system_tables: bool = False
264
264
  ) -> set[TableReference]:
265
- if self._tables:
266
- return self._tables
267
-
268
265
  catalog_placeholder = "?" if catalog else "current_database()"
269
266
  schema_placeholder = "?" if schema else "current_schema()"
270
267
  query_template = f"""
@@ -302,9 +299,9 @@ class DuckDBSchema(DatabaseSchema):
302
299
  referenced_table,
303
300
  unnest(referenced_column_names) as ref_col
304
301
  FROM duckdb_constraints()
305
- WHERE table_name = ?
306
- AND schema_name = {schema_placeholder}
307
- AND database_name = {catalog_placeholder}
302
+ WHERE lower(table_name) = lower(?)
303
+ AND lower(schema_name) = lower({schema_placeholder})
304
+ AND lower(database_name) = lower({catalog_placeholder})
308
305
  AND constraint_type = 'FOREIGN KEY'
309
306
  """
310
307
 
@@ -343,16 +340,16 @@ class DuckDBSchema(DatabaseSchema):
343
340
  query_template = f"""
344
341
  SELECT constraint_name, unnest(constraint_column_names)
345
342
  FROM duckdb_constraints()
346
- WHERE table_name = ?
347
- AND schema_name = {schema_placeholder}
348
- AND database_name = {catalog_placeholder}
343
+ WHERE lower(table_name) = lower(?)
344
+ AND lower(schema_name) = lower({schema_placeholder})
345
+ AND lower(database_name) = lower({catalog_placeholder})
349
346
  AND constraint_type IN ('PRIMARY KEY', 'FOREIGN KEY', 'UNIQUE')
350
347
  UNION
351
348
  SELECT index_name, unnest(expressions::varchar[])
352
349
  FROM duckdb_indexes()
353
- WHERE table_name = ?
354
- AND schema_name = {schema_placeholder}
355
- AND database_name = {catalog_placeholder}
350
+ WHERE lower(table_name) = lower(?)
351
+ AND lower(schema_name) = lower({schema_placeholder})
352
+ AND lower(database_name) = lower({catalog_placeholder})
356
353
  """
357
354
 
358
355
  params = [table.full_name]
@@ -536,10 +533,58 @@ def _bind_node_to_table(scan_node: dict, *, query: SqlQuery) -> TableReference |
536
533
  return best_match
537
534
 
538
535
 
539
- def parse_duckdb_plan(raw_plan: dict | str, *, query: SqlQuery | None = None) -> QueryPlan:
540
- """Parses a DuckDB query plan from its JSON representation.
536
+ _WRAPPER_OPERATORS = frozenset({"EXPLAIN", "EXPLAIN_ANALYZE", "RESULT_COLLECTOR"})
537
+ """Operators that DuckDB places above the actual plan in the EXPLAIN output."""
538
+
539
+
540
+ def _unwrap_explain_output(raw_plan: object) -> dict:
541
+ """Reduces the different representations of DuckDB's EXPLAIN output to the root operator of the actual plan.
541
542
 
542
- The query can be supplied to help with binding scan nodes to their corresponding tables (see Notes).
543
+ The representations are peeled off from the outside in: the raw result set of the EXPLAIN query (a list with a
544
+ single row), the simplified result set (the row itself, i.e. a pair of the EXPLAIN key and the JSON), the JSON
545
+ string, the parsed JSON (a list of root operators for EXPLAIN, a profiling summary for EXPLAIN ANALYZE), and finally
546
+ the `_WRAPPER_OPERATORS` above the actual plan.
547
+ """
548
+ while True:
549
+ match raw_plan:
550
+ case str():
551
+ raw_plan = json.loads(raw_plan)
552
+ case tuple() if len(raw_plan) == 2: # result set row: (explain key, JSON)
553
+ raw_plan = raw_plan[1]
554
+ case list() if len(raw_plan) == 1: # raw result set (one row), or the root operators of EXPLAIN
555
+ raw_plan = raw_plan[0]
556
+ case {"operator": [root]}: # profiling summary of EXPLAIN ANALYZE
557
+ raw_plan = root
558
+ case {"name": name, "children": [child]} if name in _WRAPPER_OPERATORS:
559
+ raw_plan = child
560
+ case dict():
561
+ return raw_plan
562
+ case _:
563
+ raise ValueError(f"Not a DuckDB EXPLAIN output: {textwrap.shorten(repr(raw_plan), width=200)}")
564
+
565
+
566
+ def parse_duckdb_plan(
567
+ raw_plan: ResultSet | ResultRow | list | dict | str, *, query: SqlQuery | None = None
568
+ ) -> QueryPlan:
569
+ """Parses a DuckDB query plan from the output of an EXPLAIN query.
570
+
571
+ Both the output of ``EXPLAIN (FORMAT JSON)`` and of ``EXPLAIN (ANALYZE, FORMAT JSON)`` are accepted, in the format
572
+ of DuckDB 2 (quacklab 2). For the latter, the actual cardinalities and execution times of the operators are included
573
+ in the plan.
574
+
575
+ Parameters
576
+ ----------
577
+ raw_plan : ResultSet | ResultRow | list | dict | str
578
+ The EXPLAIN output. Typically, this is just the result set of `DuckDBDatabase.execute_query`, either in its raw
579
+ or in its simplified form. The JSON column of the result set (as a string or as parsed JSON) is accepted as well,
580
+ as is the operator tree itself. The parser determines on its own which of these representations it receives.
581
+ query : SqlQuery | None, optional
582
+ The query that was explained. It is used to bind scan nodes to their corresponding tables (see Notes).
583
+
584
+ Returns
585
+ -------
586
+ QueryPlan
587
+ The parsed plan.
543
588
 
544
589
  Notes
545
590
  -----
@@ -547,38 +592,40 @@ def parse_duckdb_plan(raw_plan: dict | str, *, query: SqlQuery | None = None) ->
547
592
  referenced multiple times in a query, we cannot unambiguously bind scan nodes to their corresponding table. We employ
548
593
  some heuristics (e.g. trying to match filter predicates) to resolve this ambiguity, but there is no guarantee that this
549
594
  always works correctly.
595
+
596
+ Examples
597
+ --------
598
+ >>> explain_query = pb.transform.as_explain(query)
599
+ >>> plan = parse_duckdb_plan(duck_instance.execute_query(explain_query), query=query)
550
600
  """
551
- if isinstance(raw_plan, str):
552
- raw_plan = json.loads(raw_plan)
601
+ return _parse_plan_node(_unwrap_explain_output(raw_plan), query=query)
553
602
 
554
- node_type = raw_plan.get("name") or raw_plan.get("operator_name")
555
- if not node_type:
556
- assert len(raw_plan["children"]) == 1, "Expected a single child for the root operator"
557
- return parse_duckdb_plan(raw_plan["children"][0], query=query)
558
603
 
559
- if node_type == "EXPLAIN" or node_type == "EXPLAIN_ANALYZE":
560
- assert len(raw_plan["children"]) == 1, "Expected a single child for EXPLAIN operator"
561
- return parse_duckdb_plan(raw_plan["children"][0], query=query)
604
+ def _parse_plan_node(raw_plan: dict, *, query: SqlQuery | None) -> QueryPlan:
605
+ """Parses a single operator of a DuckDB plan, including all of its children."""
606
+ node_type = raw_plan.get("name")
607
+ if not node_type:
608
+ raise ValueError(f"Not a DuckDB plan node: the operator name is missing (keys: {sorted(raw_plan)})")
562
609
 
563
610
  extras: dict = raw_plan.get("extra_info", {})
564
611
  match node_type:
565
612
  case "HASH_JOIN":
566
- node_type = JoinOperator.HashJoin
613
+ operator = JoinOperator.HashJoin
567
614
  case (
568
615
  "SEQ_SCAN" | "SEQ_SCAN " # DuckDB has a weird typo in the SEQ_SCAN label
569
616
  ) if extras.get("Type", "") == "Sequential Scan":
570
- node_type = ScanOperator.SequentialScan
617
+ operator = ScanOperator.SequentialScan
571
618
  case (
572
619
  "SEQ_SCAN" | "SEQ_SCAN " # DuckDB has a weird typo in the SEQ_SCAN label
573
620
  ) if extras.get("Type", "") == "Index Scan":
574
- node_type = ScanOperator.IndexScan
621
+ operator = ScanOperator.IndexScan
575
622
  case "PROJECTION" | "FILTER" | "UNGROUPED_AGGREGATE" | "PERFECT_HASH_GROUP_BY":
576
- pass
623
+ operator = None
577
624
  case _:
578
625
  warnings.warn(f"Unknown node type: {node_type}, ({extras})", stacklevel=2)
579
- pass
626
+ operator = None
580
627
 
581
- if node_type in ScanOperator:
628
+ if operator in ScanOperator:
582
629
  base_table = _bind_node_to_table(raw_plan, query=query) if query is not None else None
583
630
  else:
584
631
  base_table = None
@@ -586,15 +633,16 @@ def parse_duckdb_plan(raw_plan: dict | str, *, query: SqlQuery | None = None) ->
586
633
  card_est = float(
587
634
  extras.get("Estimated Cardinality", math.nan)
588
635
  ) # Estimated Cardinality is a string for some reason..
589
- card_act = raw_plan.get("operator_cardinality", math.nan)
636
+ card_act = raw_plan.get("intermediate_rows", math.nan) # only present for EXPLAIN ANALYZE
590
637
 
591
- children = [parse_duckdb_plan(child, query=query) for child in raw_plan.get("children", [])]
638
+ children = [_parse_plan_node(child, query=query) for child in raw_plan.get("children", [])]
592
639
 
593
- own_runtime = raw_plan.get("operator_timing", math.nan)
640
+ own_runtime = raw_plan.get("timing", math.nan) # only present for EXPLAIN ANALYZE
594
641
  total_runtime = own_runtime + sum(child.execution_time for child in children)
595
642
 
596
643
  return QueryPlan(
597
644
  node_type,
645
+ operator=operator,
598
646
  children=children,
599
647
  base_table=base_table,
600
648
  estimated_cardinality=Cardinality(card_est),
@@ -632,9 +680,7 @@ class DuckDBOptimizer(OptimizerInterface):
632
680
  result_set = cur.fetchone()
633
681
  assert result_set is not None and len(result_set) == 2
634
682
 
635
- raw_explain = result_set[1]
636
- parsed = json.loads(raw_explain)
637
- return parse_duckdb_plan(parsed[0], query=query if isinstance(query, SqlQuery) else None)
683
+ return parse_duckdb_plan(result_set, query=query if isinstance(query, SqlQuery) else None)
638
684
 
639
685
  @overload
640
686
  def analyze_plan(self, query: SqlQuery) -> QueryPlan: ...
@@ -647,29 +693,21 @@ class DuckDBOptimizer(OptimizerInterface):
647
693
  query = transform.as_explain_analyze(query)
648
694
 
649
695
  try:
650
- result_set = self._db.execute_query(query, raw=True, timeout=timeout)[0]
696
+ result_set = self._db.execute_query(query, raw=True, timeout=timeout)
651
697
  except TimeoutError:
652
698
  return None
653
- assert len(result_set) == 2
654
699
 
655
- raw_explain = result_set[1]
656
- parsed = json.loads(raw_explain)
657
- return parse_duckdb_plan(parsed, query=query)
700
+ return parse_duckdb_plan(result_set, query=query)
658
701
 
659
702
  @override
660
703
  def parse_plan(self, plan: ResultSet, *, query: SqlQuery | None = None) -> QueryPlan | None:
661
- # Similar to the Postgres implementation of parse_plan(), we try to be graceful
662
- # and accept both simplified and raw versions of the execute_query() output.
663
- # In essence, we always try to break the input down to a plain dictionary and
664
- # unwrap the lists and tuples that come with a raw result set.
665
- if isinstance(plan, list):
666
- plan = plan[0]
667
- if isinstance(plan, tuple):
668
- plan = plan[1]
669
- if not isinstance(plan, dict):
670
- warnings.warn(f"Could not parse plan: {plan}. Expected a dictionary, but got {type(plan)}.", stacklevel=2)
704
+ # Similar to the Postgres implementation of parse_plan(), we try to be graceful and accept both simplified and
705
+ # raw versions of the execute_query() output. parse_duckdb_plan() already handles all of these representations.
706
+ try:
707
+ return parse_duckdb_plan(plan, query=query)
708
+ except ValueError as e: # includes malformed JSON
709
+ warnings.warn(f"Could not parse plan: {e}", stacklevel=2)
671
710
  return None
672
- return parse_duckdb_plan(plan, query=query)
673
711
 
674
712
  @override
675
713
  def cardinality_estimate(self, query: SqlQuery | str) -> Cardinality:
@@ -38,7 +38,7 @@ from typing import Literal, overload
38
38
  import pglast
39
39
 
40
40
  from . import util
41
- from ._core import ColumnReference, TableReference
41
+ from ._core import ColumnReference, TableReference, normalize
42
42
  from .db import DatabasePool, DatabaseSchema
43
43
  from .qal import (
44
44
  AbstractPredicate,
@@ -161,7 +161,7 @@ class SchemaCache:
161
161
  continue
162
162
 
163
163
  _, table_columns = self._inflate_cache(candidate)
164
- if colname in table_columns:
164
+ if normalize(colname) in table_columns:
165
165
  return candidate
166
166
 
167
167
  return None
@@ -187,7 +187,9 @@ class SchemaCache:
187
187
  Returns
188
188
  -------
189
189
  tuple[list[str], set[str]]
190
- The columns of the table in their defined order, as well as the same columns as a set.
190
+ The columns of the table in their defined order (spelled as in the catalog), as well as the normalized
191
+ column names as a set. Normalization makes the lookup case-insensitive, just like the database systems
192
+ resolve unquoted identifiers.
191
193
  """
192
194
  table = table.full_name if isinstance(table, TableReference) else table
193
195
  cached_res = self._lookup_cache.get(table)
@@ -197,7 +199,7 @@ class SchemaCache:
197
199
  return [], set()
198
200
 
199
201
  cols: list[str] = [col.name for col in self._schema.columns(table)]
200
- cols_set = set(cols)
202
+ cols_set = {normalize(col) for col in cols}
201
203
  self._lookup_cache[table] = cols, cols_set
202
204
  return cols, cols_set
203
205
 
@@ -402,7 +404,8 @@ class QueryNamespace:
402
404
 
403
405
  def provides_column(self, name: str) -> bool:
404
406
  """Checks, whether the current namespace has a specific column in its output relation."""
405
- return name in self._output_shape
407
+ name = normalize(name)
408
+ return any(normalize(col) == name for col in self._output_shape)
406
409
 
407
410
  def lookup_column(self, key: str) -> TableReference | None:
408
411
  """Searches for the table that provies a specific column.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "PostBOUND"
7
- version = "0.23.1"
7
+ version = "0.23.2"
8
8
  authors = [{ name = "Rico Bergmann", email = "rico.bergmann1@tu-dresden.de" }]
9
9
  description = "PostBOUND is a research framework to prototype and benchmark database query optimizers"
10
10
  readme = "README.md"
@@ -16,15 +16,20 @@ The pure formatting part of the EXPLAIN regression (the *duckdb* flavor of `form
16
16
 
17
17
  from __future__ import annotations
18
18
 
19
- from collections.abc import Iterator
19
+ import json
20
+ import math
21
+ import warnings
22
+ from collections.abc import Callable, Iterator
20
23
  from pathlib import Path
21
24
  from typing import Any, cast
22
25
 
23
26
  import pytest
24
27
 
25
- from postbound import ColumnReference, ScanOperator, TableReference, parser, transform
28
+ from postbound import ColumnReference, ScanOperator, TableReference, db, parser, transform
26
29
  from postbound.db import ForeignKeyRef, Histogram, PreciseStatistics, PrimaryKeyRef
27
- from postbound.duckdb import DuckDBDatabase
30
+ from postbound.duckdb import DuckDBDatabase, parse_duckdb_plan
31
+ from postbound.qal import SqlQuery
32
+ from tests.unit.test_duckdb import EXPLAIN_ANALYZE_FILTER_SCAN, EXPLAIN_FILTER_SCAN
28
33
 
29
34
  pytestmark = pytest.mark.embedded
30
35
 
@@ -221,6 +226,197 @@ def test_precise_histogram_of_a_single_distinct_value_on_duckdb(duckdb_instance:
221
226
  assert list(hist) == [(7, 2)]
222
227
 
223
228
 
229
+ # -- identifier casing ----------------------------------------------------------------------------------------
230
+
231
+ # DuckDB keeps the spelling of the DDL in its catalog but resolves identifiers case-insensitively. The schema mirrors
232
+ # the Stats database (``users.Id``, ``postHistory``), while the references use the lower-cased spelling that the parser
233
+ # produces for unquoted identifiers.
234
+ MIXED_CASE_DDL = [
235
+ "CREATE TABLE users (Id INTEGER PRIMARY KEY, DisplayName VARCHAR UNIQUE, Age INTEGER)",
236
+ "CREATE TABLE postHistory (Id INTEGER PRIMARY KEY, UserId INTEGER, FOREIGN KEY (UserId) REFERENCES users (Id))",
237
+ "CREATE INDEX postHistory_UserId_idx ON postHistory (UserId)",
238
+ "INSERT INTO users VALUES (1, 'a', 20), (2, 'b', NULL)",
239
+ "INSERT INTO postHistory VALUES (1, 1), (2, 1), (3, 2)",
240
+ ]
241
+ USERS, POST_HISTORY = TableReference("users"), TableReference("posthistory")
242
+ USERS_ID, USERS_AGE = ColumnReference("id", USERS), ColumnReference("age", USERS)
243
+ PH_USER_ID = ColumnReference("userid", POST_HISTORY)
244
+
245
+
246
+ @pytest.fixture
247
+ def mixed_case_instance(duckdb_instance: DuckDBDatabase) -> DuckDBDatabase:
248
+ for statement in MIXED_CASE_DDL:
249
+ duckdb_instance.execute_query(statement)
250
+ return duckdb_instance
251
+
252
+
253
+ @pytest.mark.parametrize(
254
+ ("lookup", "expected"),
255
+ [
256
+ (lambda d: d.schema().columns(POST_HISTORY), [ColumnReference("id", POST_HISTORY), PH_USER_ID]),
257
+ (lambda d: d.schema().lookup_column("age", [POST_HISTORY, USERS]), USERS),
258
+ (lambda d: d.schema().is_view(POST_HISTORY), False),
259
+ (lambda d: d.schema().is_primary_key(USERS_ID), True),
260
+ (lambda d: d.schema().primary_key_of(USERS), PrimaryKeyRef([USERS_ID])),
261
+ (lambda d: d.schema().is_unique(ColumnReference("displayname", USERS)), True),
262
+ (lambda d: d.schema().datatype(ColumnReference("displayname", USERS)), "varchar"),
263
+ (lambda d: d.schema().is_nullable(USERS_ID), False),
264
+ (lambda d: d.schema().foreign_keys_on(PH_USER_ID), {ForeignKeyRef([PH_USER_ID], [USERS_ID])}),
265
+ (lambda d: d.schema().indexes_on(PH_USER_ID), {"postHistory_UserId_idx", "postHistory_userid_id_fkey"}),
266
+ (lambda d: d.schema().has_secondary_index(PH_USER_ID), True),
267
+ (lambda d: d.schema()[USERS_ID].primary_key, True),
268
+ (lambda d: d.schema().as_graph().has_edge(POST_HISTORY, USERS), True),
269
+ (lambda d: d.schema().join_equivalence_keys()[PH_USER_ID], {PH_USER_ID, USERS_ID}),
270
+ (lambda d: d.statistics().total_rows(POST_HISTORY), 3),
271
+ (lambda d: d.statistics().num_distinct(PH_USER_ID), 2),
272
+ (lambda d: d.statistics().null_frac(USERS_AGE), 0.5),
273
+ (lambda d: d.statistics().min_max(USERS_AGE), (20, 20)),
274
+ (lambda d: list(d.statistics().most_common_values(PH_USER_ID)), [(1, 2), (2, 1)]),
275
+ ],
276
+ ids=[
277
+ "columns",
278
+ "lookup_column",
279
+ "is_view",
280
+ "is_primary_key",
281
+ "primary_key_of",
282
+ "is_unique",
283
+ "datatype",
284
+ "is_nullable",
285
+ "foreign_keys_on",
286
+ "indexes_on",
287
+ "has_secondary_index",
288
+ "column_info",
289
+ "as_graph",
290
+ "join_equivalence_keys",
291
+ "total_rows",
292
+ "num_distinct",
293
+ "null_frac",
294
+ "min_max",
295
+ "most_common_values",
296
+ ],
297
+ )
298
+ def test_duckdb_lookups_ignore_the_case_of_identifiers(
299
+ mixed_case_instance: DuckDBDatabase,
300
+ monkeypatch: pytest.MonkeyPatch,
301
+ lookup: Callable[[DuckDBDatabase], object],
302
+ expected: object,
303
+ ) -> None:
304
+ """Every schema and statistics lookup accepts the lower-cased spelling of a mixed-case DuckDB catalog.
305
+
306
+ The cases that used to fail have dedicated regression tests (tier 0 for the comparisons in the generic
307
+ `DatabaseSchema`, below for the SQL of `DuckDBSchema`); this guard checks the complete surface end to end.
308
+ """
309
+ monkeypatch.setattr(db, "enable_emulation_fallback", True)
310
+
311
+ assert lookup(mixed_case_instance) == expected
312
+
313
+
314
+ # -- plan format drift ----------------------------------------------------------------------------------------
315
+
316
+ # The plan parser is tested offline against captured EXPLAIN output (`tests/unit/test_duckdb.py`). These tests run it
317
+ # against the live engine instead, so that a DuckDB/quacklab upgrade that changes the JSON format fails here rather
318
+ # than silently producing plans without measurements. They use only operators that `parse_duckdb_plan` knows, so an
319
+ # "Unknown node type" warning signals a new wrapper or a renamed operator.
320
+ DRIFT_QUERIES = [
321
+ "SELECT * FROM r WHERE r.a = 1",
322
+ "SELECT * FROM r JOIN s ON r.a = s.a",
323
+ "SELECT count(*) FROM r JOIN s ON r.a = s.a WHERE r.b < 30",
324
+ "SELECT r.a, count(*) FROM r GROUP BY r.a",
325
+ ]
326
+ DRIFT_IDS = ["filter-scan", "join", "aggregate-over-join", "group-by"]
327
+
328
+
329
+ @pytest.fixture
330
+ def plan_instance(duckdb_instance: DuckDBDatabase) -> DuckDBDatabase:
331
+ """Extends `r` by a second table `s(a)` that joins with two of its rows."""
332
+ duckdb_instance.execute_query("CREATE TABLE s (a INTEGER)")
333
+ duckdb_instance.execute_query("INSERT INTO s VALUES (1), (2)")
334
+ return duckdb_instance
335
+
336
+
337
+ def json_shape(value: object) -> object:
338
+ """The structure of a JSON value -- its keys and nesting -- with every scalar replaced by its kind."""
339
+ match value:
340
+ case dict():
341
+ return {key: json_shape(item) for key, item in value.items()}
342
+ case list():
343
+ return [json_shape(item) for item in value]
344
+ case bool():
345
+ return "bool"
346
+ case int() | float():
347
+ return "number"
348
+ case _:
349
+ return type(value).__name__
350
+
351
+
352
+ @pytest.mark.parametrize("sql", DRIFT_QUERIES, ids=DRIFT_IDS)
353
+ def test_query_plan_parses_the_live_explain_format(plan_instance: DuckDBDatabase, sql: str) -> None:
354
+ query = parse(sql)
355
+
356
+ with warnings.catch_warnings():
357
+ warnings.simplefilter("error")
358
+ plan = plan_instance.optimizer().query_plan(query)
359
+
360
+ assert plan.tables() == query.tables()
361
+ assert all(node.estimated_cardinality.is_valid() for node in plan.iternodes())
362
+
363
+
364
+ @pytest.mark.parametrize("sql", DRIFT_QUERIES, ids=DRIFT_IDS)
365
+ def test_analyze_plan_parses_the_live_profiling_format(plan_instance: DuckDBDatabase, sql: str) -> None:
366
+ """Every operator carries its measurements, and the root produces exactly the rows of the query result."""
367
+ query = parse(sql)
368
+ n_result_rows = len(plan_instance.execute_query(query, raw=True))
369
+
370
+ with warnings.catch_warnings():
371
+ warnings.simplefilter("error")
372
+ plan = plan_instance.optimizer().analyze_plan(query)
373
+
374
+ assert plan.tables() == query.tables()
375
+ assert plan.actual_cardinality == n_result_rows
376
+ assert all(node.actual_cardinality.is_valid() for node in plan.iternodes())
377
+ assert all(math.isfinite(node.execution_time) for node in plan.iternodes())
378
+
379
+
380
+ EXPLAIN_RESULT_SETS = [
381
+ (transform.as_explain, True),
382
+ (transform.as_explain, False),
383
+ (transform.as_explain_analyze, True),
384
+ (transform.as_explain_analyze, False),
385
+ ]
386
+ EXPLAIN_RESULT_SET_IDS = ["explain-raw", "explain-simplified", "analyze-raw", "analyze-simplified"]
387
+
388
+
389
+ @pytest.mark.parametrize(("explain", "raw"), EXPLAIN_RESULT_SETS, ids=EXPLAIN_RESULT_SET_IDS)
390
+ def test_parse_duckdb_plan_accepts_the_result_set_of_execute_query(
391
+ duckdb_instance: DuckDBDatabase, explain: Callable[[SqlQuery], SqlQuery], raw: bool
392
+ ) -> None:
393
+ """The workflow of the DuckDB docs: the result set of an EXPLAIN query is passed to the parser as-is."""
394
+ result_set = duckdb_instance.execute_query(explain(FILTER_QUERY), raw=raw)
395
+
396
+ plan = parse_duckdb_plan(result_set, query=FILTER_QUERY)
397
+
398
+ assert plan == duckdb_instance.optimizer().query_plan(FILTER_QUERY)
399
+ assert plan.actual_cardinality.is_valid() is (explain is transform.as_explain_analyze)
400
+
401
+
402
+ def test_offline_explain_fixtures_match_the_live_format(duckdb_instance: DuckDBDatabase) -> None:
403
+ """The fixtures of `tests/unit/test_duckdb.py` still have the structure of the live EXPLAIN output.
404
+
405
+ If this fails after a DuckDB/quacklab upgrade, re-capture the fixtures (see the docstring of that module) and
406
+ check whether `parse_duckdb_plan` needs to follow. For the ANALYZE output, only the top-level sections of the
407
+ profiling summary are compared: the timings within them (e.g. one entry per optimizer pass) change with every
408
+ release but are irrelevant for parsing.
409
+ """
410
+ raw_explain = duckdb_instance.execute_query(transform.as_explain(FILTER_QUERY), raw=True)[0][1]
411
+ raw_analyze = duckdb_instance.execute_query(transform.as_explain_analyze(FILTER_QUERY), raw=True)[0][1]
412
+
413
+ live_explain, live_analyze = json.loads(raw_explain), json.loads(raw_analyze)
414
+
415
+ assert json_shape(live_explain) == json_shape(EXPLAIN_FILTER_SCAN)
416
+ assert set(live_analyze) == set(EXPLAIN_ANALYZE_FILTER_SCAN)
417
+ assert json_shape(live_analyze["operator"]) == json_shape(EXPLAIN_ANALYZE_FILTER_SCAN["operator"])
418
+
419
+
224
420
  # -- regression tests --------------------------------------------------------------------------------------------
225
421
 
226
422
 
@@ -330,3 +526,58 @@ def test_duckdb_database_describe_lists_keys_and_indexes_of_every_table(schema_i
330
526
  assert tables["s"]["primary_key"] is not None
331
527
  assert tables["ch"]["foreign_keys"]
332
528
  assert tables["s"]["indexes"]
529
+
530
+
531
+ def test_duckdb_schema_all_foreign_keys_on_finds_mixed_case_tables(mixed_case_instance: DuckDBDatabase) -> None:
532
+ """Regression guard for the DuckDB identifier casing bug: ``duckdb_constraints()`` was filtered with
533
+ ``table_name = ?``, which compares the catalog spelling (``postHistory``) to the lower-cased name of a parsed query
534
+ as raw strings. The table therefore seemed to have no foreign keys.
535
+ """
536
+ fks = mixed_case_instance.schema().all_foreign_keys_on(POST_HISTORY)
537
+
538
+ assert fks == {ForeignKeyRef([PH_USER_ID], [USERS_ID])}
539
+
540
+
541
+ def test_duckdb_schema_all_indexes_on_finds_mixed_case_tables(mixed_case_instance: DuckDBDatabase) -> None:
542
+ """Regression guard for the DuckDB identifier casing bug: see
543
+ `test_duckdb_schema_all_foreign_keys_on_finds_mixed_case_tables`. Both ``duckdb_constraints()`` and
544
+ ``duckdb_indexes()`` were filtered by the raw table name, so the table seemed to have no indexes.
545
+ """
546
+ indexes = mixed_case_instance.schema().all_indexes_on(POST_HISTORY)
547
+
548
+ assert {name: cols for name, cols in indexes.items()} == {
549
+ "postHistory_id_pkey": {ColumnReference("id", POST_HISTORY)},
550
+ "postHistory_userid_id_fkey": {PH_USER_ID},
551
+ "postHistory_UserId_idx": {PH_USER_ID},
552
+ }
553
+
554
+
555
+ def test_analyze_plan_reports_the_measurements_of_the_quacklab_2_profiling_format(
556
+ duckdb_instance: DuckDBDatabase,
557
+ ) -> None:
558
+ """Regression guard for the quacklab 2 profiling format: since quacklab 2.0, ``EXPLAIN (ANALYZE, FORMAT JSON)``
559
+ nests the operator tree below the ``operator`` key of a profiling summary, starts it with a ``RESULT_COLLECTOR``
560
+ operator, and renamed ``operator_cardinality``/``operator_timing`` to ``intermediate_rows``/``timing``.
561
+ `parse_duckdb_plan` looked for the plan below ``children`` and failed with a `KeyError`. Unwrapping alone was not
562
+ enough: the parser would have read the old keys and silently reported *NaN* for every measurement.
563
+ """
564
+ plan = duckdb_instance.optimizer().analyze_plan(FILTER_QUERY)
565
+
566
+ assert plan.operator == ScanOperator.SequentialScan
567
+ assert plan.actual_cardinality == 1
568
+ assert math.isfinite(plan.execution_time) and plan.execution_time > 0
569
+
570
+
571
+ @pytest.mark.parametrize(("explain", "raw"), EXPLAIN_RESULT_SETS, ids=EXPLAIN_RESULT_SET_IDS)
572
+ def test_parse_plan_accepts_the_result_set_of_an_explain_query(
573
+ duckdb_instance: DuckDBDatabase, explain: Callable[[SqlQuery], SqlQuery], raw: bool
574
+ ) -> None:
575
+ """Regression guard for `DuckDBOptimizer.parse_plan` rejecting every real result set: it unwrapped the result set
576
+ down to the JSON column, which is a *string*, and then gave up because it expected a parsed dictionary. Every
577
+ call therefore warned "Could not parse plan" and returned *None*.
578
+ """
579
+ result_set = duckdb_instance.execute_query(explain(FILTER_QUERY), raw=raw)
580
+
581
+ plan = duckdb_instance.optimizer().parse_plan(result_set, query=FILTER_QUERY)
582
+
583
+ assert plan == duckdb_instance.optimizer().query_plan(FILTER_QUERY)
@@ -18,6 +18,7 @@ import pytest
18
18
  import postbound as pb
19
19
  from postbound import BoundColumnReference, ColumnReference, TableReference
20
20
  from tests import regression_suite
21
+ from tests.doubles import StaticSchema
21
22
 
22
23
  pg_connect_dir = "."
23
24
 
@@ -404,3 +405,27 @@ def test_parser_handles_natural_right_join() -> None:
404
405
  [join] = query.from_clause.items
405
406
  assert isinstance(join, pb.qal.JoinTableSource)
406
407
  assert join.join_type == pb.qal.JoinType.NaturalRightJoin
408
+
409
+
410
+ def test_parser_binds_unqualified_columns_regardless_of_the_catalog_spelling() -> None:
411
+ """Regression guard for the DuckDB identifier casing bug: the parser's schema cache compared the lower-cased
412
+ column names of the query to the column names of the catalog as raw strings. DuckDB keeps the spelling of the DDL
413
+ (e.g. ``users.Age`` in the Stats database), so unqualified columns were left unbound.
414
+ """
415
+ schema = StaticSchema({"users": ["Id", "Age"]})
416
+
417
+ query = pb.parse_query("SELECT age FROM users", db_schema=schema, bind_columns=True)
418
+
419
+ assert query.columns() == {ColumnReference("age", TableReference("users"))}
420
+
421
+
422
+ def test_parser_binds_columns_of_a_star_subquery_regardless_of_the_catalog_spelling() -> None:
423
+ """Regression guard for the DuckDB identifier casing bug: the output shape of ``SELECT *`` is taken from the
424
+ catalog (``Id, Age``), and the lookup of an unqualified outer column compared it to the lower-cased name of the
425
+ query as a raw string. The column was therefore not attributed to the subquery.
426
+ """
427
+ schema = StaticSchema({"users": ["Id", "Age"]})
428
+
429
+ query = pb.parse_query("SELECT age FROM (SELECT * FROM users) s", db_schema=schema, bind_columns=True)
430
+
431
+ assert ColumnReference("age", TableReference.create_virtual("s")) in query.select_clause.columns()
File without changes
File without changes
File without changes
File without changes